Timon/SmartCane
1
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity
SmartCane/python_app/planet_download.ipynb
Timon 6efcc8cfec Fix CI report pipeline: update tmap v4 syntax, add continuous color scales, fix formatting 
- Updated all CI maps to use tm_scale_continuous() for proper tmap v4 compatibility
- Added fixed color scale limits (1-8 for CI, -3 to +3 for differences) for consistent field comparison
- Fixed YAML header formatting issues in CI_report_dashboard_planet.Rmd
- Positioned RGB map before CI overview map as requested
- Removed all obsolete use_breaks parameter references
- Enhanced error handling and logging throughout the pipeline
- Added new experimental analysis scripts and improvements to mosaic creation
2025-06-19 20:37:20 +02:00

659 lines
21 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"id": "0c18e312-8421-47d7-84f9-ed7d5e47e7ee",
"metadata": {
"tags": []
},
"source": [
"#### Load packages and connect to SentinelHub"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b7ca7102-5fd9-481f-90cd-3ba60e288649",
"metadata": {},
"outputs": [],
"source": [
"# $ pip install sentinelhub\n",
"# pip install gdal\n",
"\n",
"import os\n",
"import json\n",
"import datetime\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from pathlib import Path\n",
"from osgeo import gdal\n",
"\n",
"from sentinelhub import MimeType, CRS, BBox, SentinelHubRequest, SentinelHubDownloadClient, \\\n",
" DataCollection, bbox_to_dimensions, DownloadRequest, SHConfig, BBoxSplitter, read_data, Geometry, SentinelHubCatalog\n",
"\n",
"config = SHConfig()\n",
"catalog = SentinelHubCatalog(config=config)\n",
"\n",
"import time\n",
"import shutil\n",
"\n",
"import geopandas as gpd\n",
"from shapely.geometry import MultiLineString, MultiPolygon, Polygon, box, shape\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5491a840-779c-4f0c-8164-c3de738b3298",
"metadata": {},
"outputs": [],
"source": [
"config.sh_client_id = '1a72d811-4f0e-4447-8282-df09608cff44'\n",
"config.sh_client_secret = 'FcBlRL29i9ZmTzhmKTv1etSMFs5PxSos'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb1fb662-0e25-4ca9-8317-c6953290842b",
"metadata": {},
"outputs": [],
"source": [
"collection_id = 'c691479f-358c-46b1-b0f0-e12b70a9856c'\n",
"byoc = DataCollection.define_byoc(\n",
" collection_id,\n",
" name='planet_data2',\n",
" is_timeless=True)"
]
},
{
"cell_type": "markdown",
"id": "6adb603d-8182-48c6-a051-869e16ee7bba",
"metadata": {
"tags": []
},
"source": [
"#### Set some variables\n",
"The only place anything might need to be changed."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "060396e0-e5ee-4b54-b211-5d8bfcba167f",
"metadata": {},
"outputs": [],
"source": [
"#project = 'Mkulazi_trail' #or xinavane or chemba_test_8b\n",
"#project = 'xinavane' #or xinavane or chemba_test_8b\n",
"project = 'citrus_brazil_trial' #or xinavane or chemba_test_8b\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c9f79e81-dff8-4109-8d26-6c423142dcf2",
"metadata": {},
"outputs": [],
"source": [
"# Adjust the number of days needed\n",
"days = 30 #change back to 28 which is the default. 3 years is 1095 days.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e18bdf8f-be4b-44ab-baaa-de5de60d92cb",
"metadata": {},
"outputs": [],
"source": [
"#delete all the satellite outputs -> then True\n",
"empty_folder_question = True"
]
},
{
"cell_type": "markdown",
"id": "81bbb513-0bd2-4277-83e8-6f94051ce70b",
"metadata": {
"tags": []
},
"source": [
"#### Define functions\n",
"After this block, no manual changes to parameters are required. \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f7c8e04-4569-457b-b39d-283582c4ba36",
"metadata": {},
"outputs": [],
"source": [
"BASE_PATH = Path('../laravel_app/storage/app') / os.getenv('PROJECT_DIR',project) \n",
"BASE_PATH_SINGLE_IMAGES = Path(BASE_PATH / 'single_images')\n",
"folder_for_merged_tifs = str(BASE_PATH / 'merged_tif')\n",
"folder_for_virtual_raster = str(BASE_PATH / 'merged_virtual')\n",
"geojson_file = Path(BASE_PATH /'Data'/ 'pivot.geojson') #the geojsons should have the same name\n",
" \n",
"# Check if the folders exist, and if not, create them\n",
"if not os.path.exists(BASE_PATH_SINGLE_IMAGES):\n",
" os.makedirs(BASE_PATH_SINGLE_IMAGES)\n",
" \n",
"if not os.path.exists(folder_for_merged_tifs):\n",
" os.makedirs(folder_for_merged_tifs)\n",
"\n",
"if not os.path.exists(folder_for_virtual_raster):\n",
" os.makedirs(folder_for_virtual_raster)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "244b5752-4f02-4347-9278-f6a0a46b88f4",
"metadata": {},
"outputs": [],
"source": [
"evalscript_true_color = \"\"\"\n",
" //VERSION=3\n",
"\n",
" function setup() {\n",
" return {\n",
" input: [{\n",
" bands: [\"red\", \"green\", \"blue\", \"nir\", \"udm1\"]\n",
" }],\n",
" output: {\n",
" bands: 4,\n",
" sampleType: \"FLOAT32\"\n",
" }\n",
" };\n",
" }\n",
"\n",
" function evaluatePixel(sample) {\n",
" // Scale the bands\n",
" var scaledBlue = 2.5 * sample.blue / 10000;\n",
" var scaledGreen = 2.5 * sample.green / 10000;\n",
" var scaledRed = 2.5 * sample.red / 10000;\n",
" var scaledNIR = 2.5 * sample.nir / 10000;\n",
" \n",
" // Calculate indices for cloud and shadow detection\n",
" var brightness = (scaledBlue + scaledGreen + scaledRed) / 3;\n",
" var ndvi = (scaledNIR - scaledRed) / (scaledNIR + scaledRed);\n",
" var blue_ratio = scaledBlue / (scaledRed + 0.01); // Add 0.01 to prevent division by zero\n",
" \n",
" // CLOUD DETECTION\n",
" // Clouds are typically bright in all bands\n",
" var bright_pixels = (scaledBlue > 0.3) && (scaledGreen > 0.3) && (scaledRed > 0.3);\n",
" \n",
" // Clouds often have higher blue reflectance\n",
" var blue_dominant = scaledBlue > (scaledRed * 1.2);\n",
" \n",
" // Low NDVI areas that are bright are likely clouds\n",
" var low_ndvi = ndvi < 0.1;\n",
" \n",
" // Combine cloud criteria\n",
" var is_cloud = bright_pixels && (blue_dominant || low_ndvi);\n",
" \n",
" // SHADOW DETECTION\n",
" // Shadows are typically dark\n",
" var dark_pixels = brightness < 0.1;\n",
" \n",
" // Shadows have lower NIR reflectance\n",
" var low_nir = scaledNIR < 0.15;\n",
" \n",
" // Shadows often have higher blue proportion relative to NIR\n",
" var blue_enhanced = blue_ratio > 0.8;\n",
" \n",
" // Combine shadow criteria\n",
" var is_shadow = dark_pixels && (low_nir || blue_enhanced);\n",
" \n",
" // Calculate CI (Chlorophyll Index) using the scaled values\n",
" var CI = (scaledNIR / scaledRed) - 1;\n",
" \n",
" // Use built-in usable data mask (udm1) and our own cloud/shadow detection\n",
" // udm1 == 0 means pixel is usable according to Planet's metadata\n",
" if (sample.udm1 == 0 && !is_cloud && !is_shadow) {\n",
" return [scaledRed, scaledGreen, scaledBlue, scaledNIR];\n",
" } else {\n",
" return [NaN, NaN, NaN, NaN];\n",
" }\n",
" }\n",
"\"\"\"\n",
"\n",
"# Original evalscript without cloud/shadow detection (for comparison)\n",
"evalscript_original = \"\"\"\n",
" //VERSION=3\n",
"\n",
" function setup() {\n",
" return {\n",
" input: [{\n",
" bands: [\"red\", \"green\", \"blue\", \"nir\", \"udm1\"]\n",
" }],\n",
" output: {\n",
" bands: 4,\n",
" sampleType: \"FLOAT32\"\n",
" }\n",
" };\n",
" }\n",
"\n",
" function evaluatePixel(sample) {\n",
" // Scale the bands\n",
" var scaledBlue = 2.5 * sample.blue / 10000;\n",
" var scaledGreen = 2.5 * sample.green / 10000;\n",
" var scaledRed = 2.5 * sample.red / 10000;\n",
" var scaledNIR = 2.5 * sample.nir / 10000;\n",
" \n",
" // Only use udm1 mask (Planet's usable data mask)\n",
" if (sample.udm1 == 0) {\n",
" return [scaledRed, scaledGreen, scaledBlue, scaledNIR];\n",
" } else {\n",
" return [NaN, NaN, NaN, NaN];\n",
" }\n",
" }\n",
"\"\"\"\n",
"\n",
"def get_true_color_request_day(time_interval, bbox, size):\n",
" return SentinelHubRequest(\n",
" evalscript=evalscript_true_color,\n",
" input_data=[\n",
" SentinelHubRequest.input_data(\n",
" data_collection=DataCollection.planet_data2,\n",
" time_interval=(time_interval, time_interval)\n",
" )\n",
" ],\n",
" responses=[\n",
" SentinelHubRequest.output_response('default', MimeType.TIFF)\n",
" ],\n",
" bbox=bbox,\n",
" size=size,\n",
" config=config,\n",
" data_folder=str(BASE_PATH_SINGLE_IMAGES / time_interval),\n",
"\n",
" )\n",
"\n",
"# Added function to get original image for comparison\n",
"def get_original_request_day(time_interval, bbox, size):\n",
" return SentinelHubRequest(\n",
" evalscript=evalscript_original,\n",
" input_data=[\n",
" SentinelHubRequest.input_data(\n",
" data_collection=DataCollection.planet_data2,\n",
" time_interval=(time_interval, time_interval)\n",
" )\n",
" ],\n",
" responses=[\n",
" SentinelHubRequest.output_response('default', MimeType.TIFF)\n",
" ],\n",
" bbox=bbox,\n",
" size=size,\n",
" config=config,\n",
" )\n",
"\n",
"def download_function(slot, bbox, size):\n",
" # create a list of requests\n",
" list_of_requests = [get_true_color_request_day(slot, bbox, size)]\n",
" list_of_requests = [request.download_list[0] for request in list_of_requests]\n",
"\n",
" # download data chemba west with multiple threads\n",
" data = SentinelHubDownloadClient(config=config).download(list_of_requests, max_threads=15)\n",
" print(f' Image downloaded for ' +slot + ' and bbox ' + str(bbox))\n",
" \n",
" time.sleep(.1)\n",
" \n",
"\n",
"def merge_files(slot):\n",
" \n",
" # List the downloaded Tiffs in the different subfolders with pathlib (native library)\n",
" file_list = [f\"{x}/response.tiff\" for x in Path(BASE_PATH_SINGLE_IMAGES / slot).iterdir()]\n",
" \n",
" #print(file_list)\n",
"\n",
" folder_for_merged_tifs = str(BASE_PATH / 'merged_tif' / f\"{slot}.tif\")\n",
" folder_for_virtual_raster = str(BASE_PATH / 'merged_virtual' / f\"merged{slot}.vrt\")\n",
"\n",
" # Create a virtual raster\n",
" vrt_all = gdal.BuildVRT(folder_for_virtual_raster, file_list)\n",
" vrt_all = gdal.BuildVRT(folder_for_virtual_raster, file_list)\n",
"\n",
" # Convert to JPEG\n",
" gdal.Translate(folder_for_merged_tifs,folder_for_virtual_raster)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "848dc773-70d6-4ae6-b05c-d6ebfb41624d",
"metadata": {},
"outputs": [],
"source": [
"days_needed = int(os.environ.get(\"DAYS\", days))\n",
"date_str = os.environ.get(\"DATE\")\n",
"\n",
"if date_str:\n",
" # Parse de datumstring naar een datetime.date object\n",
" end = datetime.datetime.strptime(date_str, \"%Y-%m-%d\").date()\n",
"else:\n",
" # Gebruik de huidige datum als fallback\n",
" end = datetime.date.today() \n",
"\n",
"start = end - datetime.timedelta(days=days_needed - 1)\n",
"\n",
"slots = [(start + datetime.timedelta(days=i)).strftime('%Y-%m-%d') for i in range(days_needed)]\n",
"\n",
"print('Monthly time windows:\\n')\n",
"if len(slots) > 10:\n",
" for slot in slots[:3]:\n",
" print(slot)\n",
" print(\"...\")\n",
" for slot in slots[-3:]:\n",
" print(slot)\n",
"else:\n",
" for slot in slots:\n",
" print(slot)\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "f8ea846f-783b-4460-a951-7b522273555f",
"metadata": {},
"source": [
"#### Download images\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c803e373-2567-4233-af7d-0d2d6f7d4f8e",
"metadata": {},
"outputs": [],
"source": [
"geo_json = gpd.read_file(str(geojson_file))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dc24d54e-2272-4f30-bcf5-4d8fc381915c",
"metadata": {},
"outputs": [],
"source": [
"geometries = [Geometry(geometry, crs=CRS.WGS84) for geometry in geo_json.geometry]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cd071b42-d0cd-4e54-8f88-ad1a339748e3",
"metadata": {},
"outputs": [],
"source": [
"shapely_geometries = [geometry.geometry for geometry in geometries]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "301d12e4-e47a-4034-aec0-aa5673e64935",
"metadata": {},
"outputs": [],
"source": [
"bbox_splitter = BBoxSplitter(\n",
" shapely_geometries, CRS.WGS84, (1, 1), reduce_bbox_sizes=True\n",
") # bounding box will be split into a grid of 5x4 bounding boxes\n",
"\n",
"# based on https://github.com/sentinel-hub/sentinelhub-py/blob/master/examples/large_area_utilities.ipynb \n",
"\n",
"print(\"Area bounding box: {}\\n\".format(bbox_splitter.get_area_bbox().__repr__()))\n",
"\n",
"bbox_list = bbox_splitter.get_bbox_list()\n",
"info_list = bbox_splitter.get_info_list()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "431f6856-8d7e-4868-b627-20deeb47d77e",
"metadata": {},
"outputs": [],
"source": [
"geometry_list = bbox_splitter.get_geometry_list()\n",
"\n",
"geometry_list[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18655785",
"metadata": {},
"outputs": [],
"source": [
"# Function to check if images are available for a given date\n",
"def is_image_available(date):\n",
" for bbox in bbox_list:\n",
" search_iterator = catalog.search(\n",
" collection=byoc,\n",
" bbox=bbox, # Define your bounding box\n",
" time=(date, date)\n",
" )\n",
" if len(list(search_iterator)) > 0:\n",
" return True\n",
" return False\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a6fc418f",
"metadata": {},
"outputs": [],
"source": [
"# Filter slots to only include dates with available images\n",
"available_slots = [slot for slot in slots if is_image_available(slot)]\n",
"\n",
"# Store the first 5 available slots for comparison later (if available)\n",
"comparison_slots = available_slots[:min(5, len(available_slots))]\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ebc416be",
"metadata": {},
"outputs": [],
"source": [
"print(available_slots)\n",
"print(f\"Total slots: {len(slots)}\")\n",
"print(f\"Available slots: {len(available_slots)}\")\n",
"print(f\"Excluded slots due to empty dates: {len(slots) - len(available_slots)}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b0cabe8f-e1f2-4b18-8ac0-c2565d0ff16b",
"metadata": {},
"outputs": [],
"source": [
"def show_splitter(splitter, alpha=0.2, area_buffer=0.2, show_legend=False):\n",
" area_bbox = splitter.get_area_bbox()\n",
" minx, miny, maxx, maxy = area_bbox\n",
" lng, lat = area_bbox.middle\n",
" w, h = maxx - minx, maxy - miny\n",
" minx = minx - area_buffer * w\n",
" miny = miny - area_buffer * h\n",
" maxx = maxx + area_buffer * w\n",
" maxy = maxy + area_buffer * h\n",
"\n",
" fig = plt.figure(figsize=(10, 10))\n",
" ax = fig.add_subplot(111)\n",
"\n",
" base_map = Basemap(\n",
" projection=\"mill\",\n",
" lat_0=lat,\n",
" lon_0=lng,\n",
" llcrnrlon=minx,\n",
" llcrnrlat=miny,\n",
" urcrnrlon=maxx,\n",
" urcrnrlat=maxy,\n",
" resolution=\"l\",\n",
" epsg=4326,\n",
" )\n",
" base_map.drawcoastlines(color=(0, 0, 0, 0))\n",
"\n",
" area_shape = splitter.get_area_shape()\n",
"\n",
" if isinstance(area_shape, Polygon):\n",
" polygon_iter = [area_shape]\n",
" elif isinstance(area_shape, MultiPolygon):\n",
" polygon_iter = area_shape.geoms\n",
" else:\n",
" raise ValueError(f\"Geometry of type {type(area_shape)} is not supported\")\n",
"\n",
" for polygon in polygon_iter:\n",
" if isinstance(polygon.boundary, MultiLineString):\n",
" for linestring in polygon.boundary:\n",
" ax.add_patch(PltPolygon(np.array(linestring), closed=True, facecolor=(0, 0, 0, 0), edgecolor=\"red\"))\n",
" else:\n",
" ax.add_patch(\n",
" PltPolygon(np.array(polygon.boundary.coords), closed=True, facecolor=(0, 0, 0, 0), edgecolor=\"red\")\n",
" )\n",
"\n",
" bbox_list = splitter.get_bbox_list()\n",
" info_list = splitter.get_info_list()\n",
"\n",
" cm = plt.get_cmap(\"jet\", len(bbox_list))\n",
" legend_shapes = []\n",
" for i, bbox in enumerate(bbox_list):\n",
" wgs84_bbox = bbox.transform(CRS.WGS84).get_polygon()\n",
"\n",
" tile_color = tuple(list(cm(i))[:3] + [alpha])\n",
" ax.add_patch(PltPolygon(np.array(wgs84_bbox), closed=True, facecolor=tile_color, edgecolor=\"green\"))\n",
"\n",
" if show_legend:\n",
" legend_shapes.append(plt.Rectangle((0, 0), 1, 1, fc=cm(i)))\n",
"\n",
" if show_legend:\n",
" legend_names = []\n",
" for info in info_list:\n",
" legend_name = \"{},{}\".format(info[\"index_x\"], info[\"index_y\"])\n",
"\n",
" for prop in [\"grid_index\", \"tile\"]:\n",
" if prop in info:\n",
" legend_name = \"{},{}\".format(info[prop], legend_name)\n",
"\n",
" legend_names.append(legend_name)\n",
"\n",
" plt.legend(legend_shapes, legend_names)\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "41b7369c-f768-44ba-983e-eb8eae4f3afd",
"metadata": {},
"outputs": [],
"source": [
"# Load areas outside the loop if they remain constant\n",
"#bbox_area = json.dumps(chosen_area)\n",
"#areas = json.loads(os.getenv('BBOX', bbox_area))\n",
"resolution = 3\n",
"\n",
"for slot in available_slots:\n",
" for bbox in bbox_list:\n",
" bbox = BBox(bbox=bbox, crs=CRS.WGS84)\n",
" size = bbox_to_dimensions(bbox, resolution=resolution)\n",
" download_function(slot, bbox, size)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "68db3c15-6f94-432e-b315-c329e4251b21",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"for slot in available_slots:\n",
" merge_files(slot)"
]
},
{
"cell_type": "markdown",
"id": "4274d8e7-1ea3-46db-9528-069ede0b2132",
"metadata": {
"tags": []
},
"source": [
"#### Delete intermediate files\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb3fa856-a550-4899-844a-e69209bba3ad",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# List of folder names\n",
"\n",
"folders_to_empty = [BASE_PATH / 'merged_virtual', BASE_PATH_SINGLE_IMAGES]\n",
" \n",
"# Function to empty folders\n",
"\n",
"# Function to empty folders\n",
"def empty_folders(folders, run=True):\n",
" if not run:\n",
" print(\"Skipping empty_folders function.\")\n",
" return\n",
" \n",
" for folder in folders:\n",
" try:\n",
" for filename in os.listdir(folder):\n",
" file_path = os.path.join(folder, filename)\n",
" try:\n",
" if os.path.isfile(file_path):\n",
" os.unlink(file_path)\n",
" elif os.path.isdir(file_path):\n",
" shutil.rmtree(file_path)\n",
" except Exception as e:\n",
" print(f\"Error: {e}\")\n",
" print(f\"Emptied folder: {folder}\")\n",
" except OSError as e:\n",
" print(f\"Error: {e}\")\n",
"\n",
"# Call the function to empty folders only if the 'run' parameter is set to True\n",
"empty_folders(folders_to_empty, run=empty_folder_question)\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in a new issue View git blame Copy permalink