diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2042-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2042-1.png new file mode 100644 index 0000000..2d98817 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2042-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2657-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2657-1.png new file mode 100644 index 0000000..2fa7ee9 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_2657-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_29-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_29-1.png new file mode 100644 index 0000000..09f1046 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_29-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3011-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3011-1.png new file mode 100644 index 0000000..f24a574 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3011-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3236-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3236-1.png new file mode 100644 index 0000000..2d98817 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3236-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3488-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3488-1.png new file mode 100644 index 0000000..26c32eb Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_3488-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_402-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_402-1.png new file mode 100644 index 0000000..2fa7ee9 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_402-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_4816-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_4816-1.png new file mode 100644 index 0000000..fdd06ff Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_4816-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6272-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6272-1.png new file mode 100644 index 0000000..2fa7ee9 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6272-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6421-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6421-1.png new file mode 100644 index 0000000..cccb9fe Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6421-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6494-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6494-1.png new file mode 100644 index 0000000..2d98817 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_6494-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8719-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8719-1.png new file mode 100644 index 0000000..9342ec0 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8719-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8756-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8756-1.png new file mode 100644 index 0000000..fdd06ff Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_8756-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9297-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9297-1.png new file mode 100644 index 0000000..fdd06ff Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9297-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_983-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_983-1.png new file mode 100644 index 0000000..f24a574 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_983-1.png differ diff --git a/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9917-1.png b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9917-1.png new file mode 100644 index 0000000..f24a574 Binary files /dev/null and b/CI_report_dashboard_planet_long_files/figure-html/sub_chunk_9917-1.png differ diff --git a/python_app/planet_download.ipynb b/python_app/planet_download.ipynb new file mode 100644 index 0000000..de7b61d --- /dev/null +++ b/python_app/planet_download.ipynb @@ -0,0 +1,1251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0c18e312-8421-47d7-84f9-ed7d5e47e7ee", + "metadata": { + "tags": [] + }, + "source": [ + "#### Load packages and connect to SentinelHub" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b7ca7102-5fd9-481f-90cd-3ba60e288649", + "metadata": {}, + "outputs": [], + "source": [ + "# $ pip install sentinelhub\n", + "# pip install gdal\n", + "\n", + "import os\n", + "import json\n", + "import datetime\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from pathlib import Path\n", + "from osgeo import gdal\n", + "\n", + "from sentinelhub import MimeType, CRS, BBox, SentinelHubRequest, SentinelHubDownloadClient, \\\n", + " DataCollection, bbox_to_dimensions, DownloadRequest, SHConfig, BBoxSplitter, read_data\n", + "\n", + "config = SHConfig()\n", + "\n", + "import time\n", + "import shutil" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "330c967c-2742-4a7a-9a61-28bfdaf8eeca", + "metadata": {}, + "outputs": [], + "source": [ + "#pip install pipreqs" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "49f8496a-a267-4b74-9500-a168e031ed68", + "metadata": {}, + "outputs": [], + "source": [ + "#import pipreqs\n", + "#pipreqs Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/Chemba_download.ipynb" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5491a840-779c-4f0c-8164-c3de738b3298", + "metadata": {}, + "outputs": [], + "source": [ + "config.sh_client_id = '1a72d811-4f0e-4447-8282-df09608cff44'\n", + "config.sh_client_secret = 'FcBlRL29i9ZmTzhmKTv1etSMFs5PxSos'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "eb1fb662-0e25-4ca9-8317-c6953290842b", + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = 'c691479f-358c-46b1-b0f0-e12b70a9856c'\n", + "byoc = DataCollection.define_byoc(\n", + " collection_id,\n", + " name='planet_data2',\n", + " is_timeless=True)" + ] + }, + { + "cell_type": "markdown", + "id": "6adb603d-8182-48c6-a051-869e16ee7bba", + "metadata": { + "tags": [] + }, + "source": [ + "#### Set some variables\n", + "The only place anything might need to be changed." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "060396e0-e5ee-4b54-b211-5d8bfcba167f", + "metadata": {}, + "outputs": [], + "source": [ + "project = 'chemba' #or xinavane or chemba_test_8b\n", + "#project = 'xinavane' #or xinavane or chemba_test_8b\n", + "#project = 'chemba_test_8b' #or xinavane or chemba_test_8b\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c9f79e81-dff8-4109-8d26-6c423142dcf2", + "metadata": {}, + "outputs": [], + "source": [ + "# Adjust the number of days needed\n", + "days = 400 #change back to 28 which is the default. 3 years is 1095 days." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e18bdf8f-be4b-44ab-baaa-de5de60d92cb", + "metadata": {}, + "outputs": [], + "source": [ + "#delete all the satellite outputs -> then True\n", + "empty_folder_question = False" + ] + }, + { + "cell_type": "markdown", + "id": "81bbb513-0bd2-4277-83e8-6f94051ce70b", + "metadata": { + "tags": [] + }, + "source": [ + "#### Define functions\n", + "After this block, no manual changes to parameters are required. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3f7c8e04-4569-457b-b39d-283582c4ba36", + "metadata": {}, + "outputs": [], + "source": [ + "BASE_PATH = Path('../laravel_app/storage/app') / os.getenv('PROJECT_DIR',project) \n", + "BASE_PATH_SINGLE_IMAGES = Path(BASE_PATH / 'single_images')\n", + "folder_for_merged_tifs = str(BASE_PATH / 'merged_tif')\n", + "folder_for_virtual_raster = str(BASE_PATH / 'merged_virtual')\n", + " \n", + "# Check if the folders exist, and if not, create them\n", + "if not os.path.exists(BASE_PATH_SINGLE_IMAGES):\n", + " os.makedirs(BASE_PATH_SINGLE_IMAGES)\n", + " \n", + "if not os.path.exists(folder_for_merged_tifs):\n", + " os.makedirs(folder_for_merged_tifs)\n", + "\n", + "if not os.path.exists(folder_for_virtual_raster):\n", + " os.makedirs(folder_for_virtual_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "244b5752-4f02-4347-9278-f6a0a46b88f4", + "metadata": {}, + "outputs": [], + "source": [ + "evalscript_true_color = \"\"\"\n", + " //VERSION=3\n", + "\n", + " function setup() {\n", + " return {\n", + " input: [{\n", + " bands: [\"Red\", \"Green\", \"Blue\", \"NIR\", \"UDM2_Clear\"]\n", + " }],\n", + " output: {\n", + " bands: 4 \n", + " //sampleType: \"FLOAT32\"\n", + " }\n", + " };\n", + " }\n", + "\n", + " function evaluatePixel(sample) {\n", + " // Scale the bands\n", + " var scaledBlue = [2.5 * sample.Blue / 10000];\n", + " var scaledGreen = [2.5 * sample.Green / 10000];\n", + " var scaledRed = [2.5 * sample.Red / 10000];\n", + " var scaledNIR = [2.5 * sample.NIR / 10000];\n", + " \n", + " // Calculate the CI (Chlorophyll Index) using the scaled values\n", + " // var CI = [scaledNIR / scaledGreen - 1] ;\n", + "\n", + "// Output the scaled bands and CI\n", + " if (sample.UDM2_Clear != 0) { \n", + " return [\n", + " scaledRed,\n", + " scaledGreen,\n", + " scaledBlue,\n", + " scaledNIR\n", + " // sample.UDM,\n", + " // CI,\n", + " ]\n", + " } else {\n", + " return [NaN, NaN, NaN, NaN]}\n", + " }\n", + "\"\"\"\n", + "\n", + "def get_true_color_request_day(time_interval, bbox, size):\n", + " return SentinelHubRequest(\n", + " evalscript=evalscript_true_color,\n", + " input_data=[\n", + " SentinelHubRequest.input_data(\n", + " data_collection=DataCollection.planet_data2,\n", + " time_interval=(time_interval, time_interval)\n", + " )\n", + " ],\n", + " responses=[\n", + " SentinelHubRequest.output_response('default', MimeType.TIFF)\n", + " ],\n", + " bbox=bbox,\n", + " size=size,\n", + " config=config,\n", + " data_folder=str(BASE_PATH_SINGLE_IMAGES / time_interval),\n", + "\n", + " )\n", + "\n", + "def download_function(slot, bbox, size):\n", + " # create a list of requests\n", + " list_of_requests = [get_true_color_request_day(slot, bbox, size)]\n", + " list_of_requests = [request.download_list[0] for request in list_of_requests]\n", + "\n", + " # download data chemba west with multiple threads\n", + " data = SentinelHubDownloadClient(config=config).download(list_of_requests, max_threads=15)\n", + " print(f' Image downloaded for ' +slot)\n", + " \n", + " time.sleep(.1)\n", + "\n", + "def merge_files(slot):\n", + " \n", + " # List the downloaded Tiffs in the different subfolders with pathlib (native library)\n", + " file_list = [f\"{x}/response.tiff\" for x in Path(BASE_PATH_SINGLE_IMAGES / slot).iterdir()]\n", + " \n", + " #print(file_list)\n", + "\n", + " folder_for_merged_tifs = str(BASE_PATH / 'merged_tif' / f\"{slot}.tif\")\n", + " folder_for_virtual_raster = str(BASE_PATH / 'merged_virtual' / f\"merged{slot}.vrt\")\n", + "\n", + " # Create a virtual raster\n", + " vrt_all = gdal.BuildVRT(folder_for_virtual_raster, file_list)\n", + " vrt_all = gdal.BuildVRT(folder_for_virtual_raster, file_list)\n", + "\n", + " # Convert to JPEG\n", + " gdal.Translate(folder_for_merged_tifs,folder_for_virtual_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "848dc773-70d6-4ae6-b05c-d6ebfb41624d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Monthly time windows:\n", + "\n", + "2023-02-07\n", + "2023-02-08\n", + "2023-02-09\n", + "...\n", + "2024-03-10\n", + "2024-03-11\n", + "2024-03-12\n" + ] + } + ], + "source": [ + "days_needed = int(os.environ.get(\"DAYS\", days))\n", + "date_str = os.environ.get(\"DATE\")\n", + "if date_str:\n", + " # Parse de datumstring naar een datetime.date object\n", + " end = datetime.datetime.strptime(date_str, \"%Y-%m-%d\").date()\n", + "else:\n", + " # Gebruik de huidige datum als fallback\n", + " end = datetime.date.today() \n", + "\n", + "start = end - datetime.timedelta(days=days_needed - 1)\n", + "\n", + "slots = [(start + datetime.timedelta(days=i)).strftime('%Y-%m-%d') for i in range(days_needed)]\n", + "\n", + "print('Monthly time windows:\\n')\n", + "if len(slots) > 10:\n", + " for slot in slots[:3]:\n", + " print(slot)\n", + " print(\"...\")\n", + " for slot in slots[-3:]:\n", + " print(slot)\n", + "else:\n", + " for slot in slots:\n", + " print(slot)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "f8ea846f-783b-4460-a951-7b522273555f", + "metadata": {}, + "source": [ + "#### Download images\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "7c0f8f0e-b1bc-4b5a-939d-e4358a485c06", + "metadata": {}, + "outputs": [], + "source": [ + "if project == 'chemba':\n", + " chosen_area = [[34.9460, -17.3500, 34.9839, -17.3110], [34.8830, -17.3516, 34.9380, -17.2917]]\n", + "\n", + "if project == 'chemba_test_8b':\n", + " chosen_area = [[34.946, -17.3516, 34.938, -17.2917], [34.883, -17.3516, 34.938, -17.2917]]\n", + "\n", + "if project == 'xinavane':\n", + " chosen_area = [[32.6790, -25.0333, 32.7453, -25.0235], [32.6213, -25.0647, 32.6284, -25.0570]]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6c02d7de-cddf-4fc3-8d23-8431415d07b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Image downloaded for 2023-02-07\n", + " Image downloaded for 2023-02-07\n", + " Image downloaded for 2023-02-08\n", + " Image downloaded for 2023-02-08\n", + " Image downloaded for 2023-02-09\n", + " Image downloaded for 2023-02-09\n", + " Image downloaded for 2023-02-10\n", + " Image downloaded for 2023-02-10\n", + " Image downloaded for 2023-02-11\n", + " Image downloaded for 2023-02-11\n", + " Image downloaded for 2023-02-12\n", + " Image downloaded for 2023-02-12\n", + " Image downloaded for 2023-02-13\n", + " Image downloaded for 2023-02-13\n", + " Image downloaded for 2023-02-14\n", + " Image downloaded for 2023-02-14\n", + " Image downloaded for 2023-02-15\n", + " Image downloaded for 2023-02-15\n", + " Image downloaded for 2023-02-16\n", + " Image downloaded for 2023-02-16\n", + " Image downloaded for 2023-02-17\n", + " Image downloaded for 2023-02-17\n", + " Image downloaded for 2023-02-18\n", + " Image downloaded for 2023-02-18\n", + " Image downloaded for 2023-02-19\n", + " Image downloaded for 2023-02-19\n", + " Image downloaded for 2023-02-20\n", + " Image downloaded for 2023-02-20\n", + " Image downloaded for 2023-02-21\n", + " Image downloaded for 2023-02-21\n", + " Image downloaded for 2023-02-22\n", + " Image downloaded for 2023-02-22\n", + " Image downloaded for 2023-02-23\n", + " Image downloaded for 2023-02-23\n", + " Image downloaded for 2023-02-24\n", + " Image downloaded for 2023-02-24\n", + " Image downloaded for 2023-02-25\n", + " Image downloaded for 2023-02-25\n", + " Image downloaded for 2023-02-26\n", + " Image downloaded for 2023-02-26\n", + " Image downloaded for 2023-02-27\n", + " Image downloaded for 2023-02-27\n", + " Image downloaded for 2023-02-28\n", + " Image downloaded for 2023-02-28\n", + " Image downloaded for 2023-03-01\n", + " Image downloaded for 2023-03-01\n", + " Image downloaded for 2023-03-02\n", + " Image downloaded for 2023-03-02\n", + " Image downloaded for 2023-03-03\n", + " Image downloaded for 2023-03-03\n", + " Image downloaded for 2023-03-04\n", + " Image downloaded for 2023-03-04\n", + " Image downloaded for 2023-03-05\n", + " Image downloaded for 2023-03-05\n", + " Image downloaded for 2023-03-06\n", + " Image downloaded for 2023-03-06\n", + " Image downloaded for 2023-03-07\n", + " Image downloaded for 2023-03-07\n", + " Image downloaded for 2023-03-08\n", + " Image downloaded for 2023-03-08\n", + " Image downloaded for 2023-03-09\n", + " Image downloaded for 2023-03-09\n", + " Image downloaded for 2023-03-10\n", + " Image downloaded for 2023-03-10\n", + " Image downloaded for 2023-03-11\n", + " Image downloaded for 2023-03-11\n", + " Image downloaded for 2023-03-12\n", + " Image downloaded for 2023-03-12\n", + " Image downloaded for 2023-03-13\n", + " Image downloaded for 2023-03-13\n", + " Image downloaded for 2023-03-14\n", + " Image downloaded for 2023-03-14\n", + " Image downloaded for 2023-03-15\n", + " Image downloaded for 2023-03-15\n", + " Image downloaded for 2023-03-16\n", + " Image downloaded for 2023-03-16\n", + " Image downloaded for 2023-03-17\n", + " Image downloaded for 2023-03-17\n", + " Image downloaded for 2023-03-18\n", + " Image downloaded for 2023-03-18\n", + " Image downloaded for 2023-03-19\n", + " Image downloaded for 2023-03-19\n", + " Image downloaded for 2023-03-20\n", + " Image downloaded for 2023-03-20\n", + " Image downloaded for 2023-03-21\n", + " Image downloaded for 2023-03-21\n", + " Image downloaded for 2023-03-22\n", + " Image downloaded for 2023-03-22\n", + " Image downloaded for 2023-03-23\n", + " Image downloaded for 2023-03-23\n", + " Image downloaded for 2023-03-24\n", + " Image downloaded for 2023-03-24\n", + " Image downloaded for 2023-03-25\n", + " Image downloaded for 2023-03-25\n", + " Image downloaded for 2023-03-26\n", + " Image downloaded for 2023-03-26\n", + " Image downloaded for 2023-03-27\n", + " Image downloaded for 2023-03-27\n", + " Image downloaded for 2023-03-28\n", + " Image downloaded for 2023-03-28\n", + " Image downloaded for 2023-03-29\n", + " Image downloaded for 2023-03-29\n", + " Image downloaded for 2023-03-30\n", + " Image downloaded for 2023-03-30\n", + " Image downloaded for 2023-03-31\n", + " Image downloaded for 2023-03-31\n", + " Image downloaded for 2023-04-01\n", + " Image downloaded for 2023-04-01\n", + " Image downloaded for 2023-04-02\n", + " Image downloaded for 2023-04-02\n", + " Image downloaded for 2023-04-03\n", + " Image downloaded for 2023-04-03\n", + " Image downloaded for 2023-04-04\n", + " Image downloaded for 2023-04-04\n", + " Image downloaded for 2023-04-05\n", + " Image downloaded for 2023-04-05\n", + " Image downloaded for 2023-04-06\n", + " Image downloaded for 2023-04-06\n", + " Image downloaded for 2023-04-07\n", + " Image downloaded for 2023-04-07\n", + " Image downloaded for 2023-04-08\n", + " Image downloaded for 2023-04-08\n", + " Image downloaded for 2023-04-09\n", + " Image downloaded for 2023-04-09\n", + " Image downloaded for 2023-04-10\n", + " Image downloaded for 2023-04-10\n", + " Image downloaded for 2023-04-11\n", + " Image downloaded for 2023-04-11\n", + " Image downloaded for 2023-04-12\n", + " Image downloaded for 2023-04-12\n", + " Image downloaded for 2023-04-13\n", + " Image downloaded for 2023-04-13\n", + " Image downloaded for 2023-04-14\n", + " Image downloaded for 2023-04-14\n", + " Image downloaded for 2023-04-15\n", + " Image downloaded for 2023-04-15\n", + " Image downloaded for 2023-04-16\n", + " Image downloaded for 2023-04-16\n", + " Image downloaded for 2023-04-17\n", + " Image downloaded for 2023-04-17\n", + " Image downloaded for 2023-04-18\n", + " Image downloaded for 2023-04-18\n", + " Image downloaded for 2023-04-19\n", + " Image downloaded for 2023-04-19\n", + " Image downloaded for 2023-04-20\n", + " Image downloaded for 2023-04-20\n", + " Image downloaded for 2023-04-21\n", + " Image downloaded for 2023-04-21\n", + " Image downloaded for 2023-04-22\n", + " Image downloaded for 2023-04-22\n", + " Image downloaded for 2023-04-23\n", + " Image downloaded for 2023-04-23\n", + " Image downloaded for 2023-04-24\n", + " Image downloaded for 2023-04-24\n", + " Image downloaded for 2023-04-25\n", + " Image downloaded for 2023-04-25\n", + " Image downloaded for 2023-04-26\n", + " Image downloaded for 2023-04-26\n", + " Image downloaded for 2023-04-27\n", + " Image downloaded for 2023-04-27\n", + " Image downloaded for 2023-04-28\n", + " Image downloaded for 2023-04-28\n", + " Image downloaded for 2023-04-29\n", + " Image downloaded for 2023-04-29\n", + " Image downloaded for 2023-04-30\n", + " Image downloaded for 2023-04-30\n", + " Image downloaded for 2023-05-01\n", + " Image downloaded for 2023-05-01\n", + " Image downloaded for 2023-05-02\n", + " Image downloaded for 2023-05-02\n", + " Image downloaded for 2023-05-03\n", + " Image downloaded for 2023-05-03\n", + " Image downloaded for 2023-05-04\n", + " Image downloaded for 2023-05-04\n", + " Image downloaded for 2023-05-05\n", + " Image downloaded for 2023-05-05\n", + " Image downloaded for 2023-05-06\n", + " Image downloaded for 2023-05-06\n", + " Image downloaded for 2023-05-07\n", + " Image downloaded for 2023-05-07\n", + " Image downloaded for 2023-05-08\n", + " Image downloaded for 2023-05-08\n", + " Image downloaded for 2023-05-09\n", + " Image downloaded for 2023-05-09\n", + " Image downloaded for 2023-05-10\n", + " Image downloaded for 2023-05-10\n", + " Image downloaded for 2023-05-11\n", + " Image downloaded for 2023-05-11\n", + " Image downloaded for 2023-05-12\n", + " Image downloaded for 2023-05-12\n", + " Image downloaded for 2023-05-13\n", + " Image downloaded for 2023-05-13\n", + " Image downloaded for 2023-05-14\n", + " Image downloaded for 2023-05-14\n", + " Image downloaded for 2023-05-15\n", + " Image downloaded for 2023-05-15\n", + " Image downloaded for 2023-05-16\n", + " Image downloaded for 2023-05-16\n", + " Image downloaded for 2023-05-17\n", + " Image downloaded for 2023-05-17\n", + " Image downloaded for 2023-05-18\n", + " Image downloaded for 2023-05-18\n", + " Image downloaded for 2023-05-19\n", + " Image downloaded for 2023-05-19\n", + " Image downloaded for 2023-05-20\n", + " Image downloaded for 2023-05-20\n", + " Image downloaded for 2023-05-21\n", + " Image downloaded for 2023-05-21\n", + " Image downloaded for 2023-05-22\n", + " Image downloaded for 2023-05-22\n", + " Image downloaded for 2023-05-23\n", + " Image downloaded for 2023-05-23\n", + " Image downloaded for 2023-05-24\n", + " Image downloaded for 2023-05-24\n", + " Image downloaded for 2023-05-25\n", + " Image downloaded for 2023-05-25\n", + " Image downloaded for 2023-05-26\n", + " Image downloaded for 2023-05-26\n", + " Image downloaded for 2023-05-27\n", + " Image downloaded for 2023-05-27\n", + " Image downloaded for 2023-05-28\n", + " Image downloaded for 2023-05-28\n", + " Image downloaded for 2023-05-29\n", + " Image downloaded for 2023-05-29\n", + " Image downloaded for 2023-05-30\n", + " Image downloaded for 2023-05-30\n", + " Image downloaded for 2023-05-31\n", + " Image downloaded for 2023-05-31\n", + " Image downloaded for 2023-06-01\n", + " Image downloaded for 2023-06-01\n", + " Image downloaded for 2023-06-02\n", + " Image downloaded for 2023-06-02\n", + " Image downloaded for 2023-06-03\n", + " Image downloaded for 2023-06-03\n", + " Image downloaded for 2023-06-04\n", + " Image downloaded for 2023-06-04\n", + " Image downloaded for 2023-06-05\n", + " Image downloaded for 2023-06-05\n", + " Image downloaded for 2023-06-06\n", + " Image downloaded for 2023-06-06\n", + " Image downloaded for 2023-06-07\n", + " Image downloaded for 2023-06-07\n", + " Image downloaded for 2023-06-08\n", + " Image downloaded for 2023-06-08\n", + " Image downloaded for 2023-06-09\n", + " Image downloaded for 2023-06-09\n", + " Image downloaded for 2023-06-10\n", + " Image downloaded for 2023-06-10\n", + " Image downloaded for 2023-06-11\n", + " Image downloaded for 2023-06-11\n", + " Image downloaded for 2023-06-12\n", + " Image downloaded for 2023-06-12\n", + " Image downloaded for 2023-06-13\n", + " Image downloaded for 2023-06-13\n", + " Image downloaded for 2023-06-14\n", + " Image downloaded for 2023-06-14\n", + " Image downloaded for 2023-06-15\n", + " Image downloaded for 2023-06-15\n", + " Image downloaded for 2023-06-16\n", + " Image downloaded for 2023-06-16\n", + " Image downloaded for 2023-06-17\n", + " Image downloaded for 2023-06-17\n", + " Image downloaded for 2023-06-18\n", + " Image downloaded for 2023-06-18\n", + " Image downloaded for 2023-06-19\n", + " Image downloaded for 2023-06-19\n", + " Image downloaded for 2023-06-20\n", + " Image downloaded for 2023-06-20\n", + " Image downloaded for 2023-06-21\n", + " Image downloaded for 2023-06-21\n", + " Image downloaded for 2023-06-22\n", + " Image downloaded for 2023-06-22\n", + " Image downloaded for 2023-06-23\n", + " Image downloaded for 2023-06-23\n", + " Image downloaded for 2023-06-24\n", + " Image downloaded for 2023-06-24\n", + " Image downloaded for 2023-06-25\n", + " Image downloaded for 2023-06-25\n", + " Image downloaded for 2023-06-26\n", + " Image downloaded for 2023-06-26\n", + " Image downloaded for 2023-06-27\n", + " Image downloaded for 2023-06-27\n", + " Image downloaded for 2023-06-28\n", + " Image downloaded for 2023-06-28\n", + " Image downloaded for 2023-06-29\n", + " Image downloaded for 2023-06-29\n", + " Image downloaded for 2023-06-30\n", + " Image downloaded for 2023-06-30\n", + " Image downloaded for 2023-07-01\n", + " Image downloaded for 2023-07-01\n", + " Image downloaded for 2023-07-02\n", + " Image downloaded for 2023-07-02\n", + " Image downloaded for 2023-07-03\n", + " Image downloaded for 2023-07-03\n", + " Image downloaded for 2023-07-04\n", + " Image downloaded for 2023-07-04\n", + " Image downloaded for 2023-07-05\n", + " Image downloaded for 2023-07-05\n", + " Image downloaded for 2023-07-06\n", + " Image downloaded for 2023-07-06\n", + " Image downloaded for 2023-07-07\n", + " Image downloaded for 2023-07-07\n", + " Image downloaded for 2023-07-08\n", + " Image downloaded for 2023-07-08\n", + " Image downloaded for 2023-07-09\n", + " Image downloaded for 2023-07-09\n", + " Image downloaded for 2023-07-10\n", + " Image downloaded for 2023-07-10\n", + " Image downloaded for 2023-07-11\n", + " Image downloaded for 2023-07-11\n", + " Image downloaded for 2023-07-12\n", + " Image downloaded for 2023-07-12\n", + " Image downloaded for 2023-07-13\n", + " Image downloaded for 2023-07-13\n", + " Image downloaded for 2023-07-14\n", + " Image downloaded for 2023-07-14\n", + " Image downloaded for 2023-07-15\n", + " Image downloaded for 2023-07-15\n", + " Image downloaded for 2023-07-16\n", + " Image downloaded for 2023-07-16\n", + " Image downloaded for 2023-07-17\n", + " Image downloaded for 2023-07-17\n", + " Image downloaded for 2023-07-18\n", + " Image downloaded for 2023-07-18\n", + " Image downloaded for 2023-07-19\n", + " Image downloaded for 2023-07-19\n", + " Image downloaded for 2023-07-20\n", + " Image downloaded for 2023-07-20\n", + " Image downloaded for 2023-07-21\n", + " Image downloaded for 2023-07-21\n", + " Image downloaded for 2023-07-22\n", + " Image downloaded for 2023-07-22\n", + " Image downloaded for 2023-07-23\n", + " Image downloaded for 2023-07-23\n", + " Image downloaded for 2023-07-24\n", + " Image downloaded for 2023-07-24\n", + " Image downloaded for 2023-07-25\n", + " Image downloaded for 2023-07-25\n", + " Image downloaded for 2023-07-26\n", + " Image downloaded for 2023-07-26\n", + " Image downloaded for 2023-07-27\n", + " Image downloaded for 2023-07-27\n", + " Image downloaded for 2023-07-28\n", + " Image downloaded for 2023-07-28\n", + " Image downloaded for 2023-07-29\n", + " Image downloaded for 2023-07-29\n", + " Image downloaded for 2023-07-30\n", + " Image downloaded for 2023-07-30\n", + " Image downloaded for 2023-07-31\n", + " Image downloaded for 2023-07-31\n", + " Image downloaded for 2023-08-01\n", + " Image downloaded for 2023-08-01\n", + " Image downloaded for 2023-08-02\n", + " Image downloaded for 2023-08-02\n", + " Image downloaded for 2023-08-03\n", + " Image downloaded for 2023-08-03\n", + " Image downloaded for 2023-08-04\n", + " Image downloaded for 2023-08-04\n", + " Image downloaded for 2023-08-05\n", + " Image downloaded for 2023-08-05\n", + " Image downloaded for 2023-08-06\n", + " Image downloaded for 2023-08-06\n", + " Image downloaded for 2023-08-07\n", + " Image downloaded for 2023-08-07\n", + " Image downloaded for 2023-08-08\n", + " Image downloaded for 2023-08-08\n", + " Image downloaded for 2023-08-09\n", + " Image downloaded for 2023-08-09\n", + " Image downloaded for 2023-08-10\n", + " Image downloaded for 2023-08-10\n", + " Image downloaded for 2023-08-11\n", + " Image downloaded for 2023-08-11\n", + " Image downloaded for 2023-08-12\n", + " Image downloaded for 2023-08-12\n", + " Image downloaded for 2023-08-13\n", + " Image downloaded for 2023-08-13\n", + " Image downloaded for 2023-08-14\n", + " Image downloaded for 2023-08-14\n", + " Image downloaded for 2023-08-15\n", + " Image downloaded for 2023-08-15\n", + " Image downloaded for 2023-08-16\n", + " Image downloaded for 2023-08-16\n", + " Image downloaded for 2023-08-17\n", + " Image downloaded for 2023-08-17\n", + " Image downloaded for 2023-08-18\n", + " Image downloaded for 2023-08-18\n", + " Image downloaded for 2023-08-19\n", + " Image downloaded for 2023-08-19\n", + " Image downloaded for 2023-08-20\n", + " Image downloaded for 2023-08-20\n", + " Image downloaded for 2023-08-21\n", + " Image downloaded for 2023-08-21\n", + " Image downloaded for 2023-08-22\n", + " Image downloaded for 2023-08-22\n", + " Image downloaded for 2023-08-23\n", + " Image downloaded for 2023-08-23\n", + " Image downloaded for 2023-08-24\n", + " Image downloaded for 2023-08-24\n", + " Image downloaded for 2023-08-25\n", + " Image downloaded for 2023-08-25\n", + " Image downloaded for 2023-08-26\n", + " Image downloaded for 2023-08-26\n", + " Image downloaded for 2023-08-27\n", + " Image downloaded for 2023-08-27\n", + " Image downloaded for 2023-08-28\n", + " Image downloaded for 2023-08-28\n", + " Image downloaded for 2023-08-29\n", + " Image downloaded for 2023-08-29\n", + " Image downloaded for 2023-08-30\n", + " Image downloaded for 2023-08-30\n", + " Image downloaded for 2023-08-31\n", + " Image downloaded for 2023-08-31\n", + " Image downloaded for 2023-09-01\n", + " Image downloaded for 2023-09-01\n", + " Image downloaded for 2023-09-02\n", + " Image downloaded for 2023-09-02\n", + " Image downloaded for 2023-09-03\n", + " Image downloaded for 2023-09-03\n", + " Image downloaded for 2023-09-04\n", + " Image downloaded for 2023-09-04\n", + " Image downloaded for 2023-09-05\n", + " Image downloaded for 2023-09-05\n", + " Image downloaded for 2023-09-06\n", + " Image downloaded for 2023-09-06\n", + " Image downloaded for 2023-09-07\n", + " Image downloaded for 2023-09-07\n", + " Image downloaded for 2023-09-08\n", + " Image downloaded for 2023-09-08\n", + " Image downloaded for 2023-09-09\n", + " Image downloaded for 2023-09-09\n", + " Image downloaded for 2023-09-10\n", + " Image downloaded for 2023-09-10\n", + " Image downloaded for 2023-09-11\n", + " Image downloaded for 2023-09-11\n", + " Image downloaded for 2023-09-12\n", + " Image downloaded for 2023-09-12\n", + " Image downloaded for 2023-09-13\n", + " Image downloaded for 2023-09-13\n", + " Image downloaded for 2023-09-14\n", + " Image downloaded for 2023-09-14\n", + " Image downloaded for 2023-09-15\n", + " Image downloaded for 2023-09-15\n", + " Image downloaded for 2023-09-16\n", + " Image downloaded for 2023-09-16\n", + " Image downloaded for 2023-09-17\n", + " Image downloaded for 2023-09-17\n", + " Image downloaded for 2023-09-18\n", + " Image downloaded for 2023-09-18\n", + " Image downloaded for 2023-09-19\n", + " Image downloaded for 2023-09-19\n", + " Image downloaded for 2023-09-20\n", + " Image downloaded for 2023-09-20\n", + " Image downloaded for 2023-09-21\n", + " Image downloaded for 2023-09-21\n", + " Image downloaded for 2023-09-22\n", + " Image downloaded for 2023-09-22\n", + " Image downloaded for 2023-09-23\n", + " Image downloaded for 2023-09-23\n", + " Image downloaded for 2023-09-24\n", + " Image downloaded for 2023-09-24\n", + " Image downloaded for 2023-09-25\n", + " Image downloaded for 2023-09-25\n", + " Image downloaded for 2023-09-26\n", + " Image downloaded for 2023-09-26\n", + " Image downloaded for 2023-09-27\n", + " Image downloaded for 2023-09-27\n", + " Image downloaded for 2023-09-28\n", + " Image downloaded for 2023-09-28\n", + " Image downloaded for 2023-09-29\n", + " Image downloaded for 2023-09-29\n", + " Image downloaded for 2023-09-30\n", + " Image downloaded for 2023-09-30\n", + " Image downloaded for 2023-10-01\n", + " Image downloaded for 2023-10-01\n", + " Image downloaded for 2023-10-02\n", + " Image downloaded for 2023-10-02\n", + " Image downloaded for 2023-10-03\n", + " Image downloaded for 2023-10-03\n", + " Image downloaded for 2023-10-04\n", + " Image downloaded for 2023-10-04\n", + " Image downloaded for 2023-10-05\n", + " Image downloaded for 2023-10-05\n", + " Image downloaded for 2023-10-06\n", + " Image downloaded for 2023-10-06\n", + " Image downloaded for 2023-10-07\n", + " Image downloaded for 2023-10-07\n", + " Image downloaded for 2023-10-08\n", + " Image downloaded for 2023-10-08\n", + " Image downloaded for 2023-10-09\n", + " Image downloaded for 2023-10-09\n", + " Image downloaded for 2023-10-10\n", + " Image downloaded for 2023-10-10\n", + " Image downloaded for 2023-10-11\n", + " Image downloaded for 2023-10-11\n", + " Image downloaded for 2023-10-12\n", + " Image downloaded for 2023-10-12\n", + " Image downloaded for 2023-10-13\n", + " Image downloaded for 2023-10-13\n", + " Image downloaded for 2023-10-14\n", + " Image downloaded for 2023-10-14\n", + " Image downloaded for 2023-10-15\n", + " Image downloaded for 2023-10-15\n", + " Image downloaded for 2023-10-16\n", + " Image downloaded for 2023-10-16\n", + " Image downloaded for 2023-10-17\n", + " Image downloaded for 2023-10-17\n", + " Image downloaded for 2023-10-18\n", + " Image downloaded for 2023-10-18\n", + " Image downloaded for 2023-10-19\n", + " Image downloaded for 2023-10-19\n", + " Image downloaded for 2023-10-20\n", + " Image downloaded for 2023-10-20\n", + " Image downloaded for 2023-10-21\n", + " Image downloaded for 2023-10-21\n", + " Image downloaded for 2023-10-22\n", + " Image downloaded for 2023-10-22\n", + " Image downloaded for 2023-10-23\n", + " Image downloaded for 2023-10-23\n", + " Image downloaded for 2023-10-24\n", + " Image downloaded for 2023-10-24\n", + " Image downloaded for 2023-10-25\n", + " Image downloaded for 2023-10-25\n", + " Image downloaded for 2023-10-26\n", + " Image downloaded for 2023-10-26\n", + " Image downloaded for 2023-10-27\n", + " Image downloaded for 2023-10-27\n", + " Image downloaded for 2023-10-28\n", + " Image downloaded for 2023-10-28\n", + " Image downloaded for 2023-10-29\n", + " Image downloaded for 2023-10-29\n", + " Image downloaded for 2023-10-30\n", + " Image downloaded for 2023-10-30\n", + " Image downloaded for 2023-10-31\n", + " Image downloaded for 2023-10-31\n", + " Image downloaded for 2023-11-01\n", + " Image downloaded for 2023-11-01\n", + " Image downloaded for 2023-11-02\n", + " Image downloaded for 2023-11-02\n", + " Image downloaded for 2023-11-03\n", + " Image downloaded for 2023-11-03\n", + " Image downloaded for 2023-11-04\n", + " Image downloaded for 2023-11-04\n", + " Image downloaded for 2023-11-05\n", + " Image downloaded for 2023-11-05\n", + " Image downloaded for 2023-11-06\n", + " Image downloaded for 2023-11-06\n", + " Image downloaded for 2023-11-07\n", + " Image downloaded for 2023-11-07\n", + " Image downloaded for 2023-11-08\n", + " Image downloaded for 2023-11-08\n", + " Image downloaded for 2023-11-09\n", + " Image downloaded for 2023-11-09\n", + " Image downloaded for 2023-11-10\n", + " Image downloaded for 2023-11-10\n", + " Image downloaded for 2023-11-11\n", + " Image downloaded for 2023-11-11\n", + " Image downloaded for 2023-11-12\n", + " Image downloaded for 2023-11-12\n", + " Image downloaded for 2023-11-13\n", + " Image downloaded for 2023-11-13\n", + " Image downloaded for 2023-11-14\n", + " Image downloaded for 2023-11-14\n", + " Image downloaded for 2023-11-15\n", + " Image downloaded for 2023-11-15\n", + " Image downloaded for 2023-11-16\n", + " Image downloaded for 2023-11-16\n", + " Image downloaded for 2023-11-17\n", + " Image downloaded for 2023-11-17\n", + " Image downloaded for 2023-11-18\n", + " Image downloaded for 2023-11-18\n", + " Image downloaded for 2023-11-19\n", + " Image downloaded for 2023-11-19\n", + " Image downloaded for 2023-11-20\n", + " Image downloaded for 2023-11-20\n", + " Image downloaded for 2023-11-21\n", + " Image downloaded for 2023-11-21\n", + " Image downloaded for 2023-11-22\n", + " Image downloaded for 2023-11-22\n", + " Image downloaded for 2023-11-23\n", + " Image downloaded for 2023-11-23\n", + " Image downloaded for 2023-11-24\n", + " Image downloaded for 2023-11-24\n", + " Image downloaded for 2023-11-25\n", + " Image downloaded for 2023-11-25\n", + " Image downloaded for 2023-11-26\n", + " Image downloaded for 2023-11-26\n", + " Image downloaded for 2023-11-27\n", + " Image downloaded for 2023-11-27\n", + " Image downloaded for 2023-11-28\n", + " Image downloaded for 2023-11-28\n", + " Image downloaded for 2023-11-29\n", + " Image downloaded for 2023-11-29\n", + " Image downloaded for 2023-11-30\n", + " Image downloaded for 2023-11-30\n", + " Image downloaded for 2023-12-01\n", + " Image downloaded for 2023-12-01\n", + " Image downloaded for 2023-12-02\n", + " Image downloaded for 2023-12-02\n", + " Image downloaded for 2023-12-03\n", + " Image downloaded for 2023-12-03\n", + " Image downloaded for 2023-12-04\n", + " Image downloaded for 2023-12-04\n", + " Image downloaded for 2023-12-05\n", + " Image downloaded for 2023-12-05\n", + " Image downloaded for 2023-12-06\n", + " Image downloaded for 2023-12-06\n", + " Image downloaded for 2023-12-07\n", + " Image downloaded for 2023-12-07\n", + " Image downloaded for 2023-12-08\n", + " Image downloaded for 2023-12-08\n", + " Image downloaded for 2023-12-09\n", + " Image downloaded for 2023-12-09\n", + " Image downloaded for 2023-12-10\n", + " Image downloaded for 2023-12-10\n", + " Image downloaded for 2023-12-11\n", + " Image downloaded for 2023-12-11\n", + " Image downloaded for 2023-12-12\n", + " Image downloaded for 2023-12-12\n", + " Image downloaded for 2023-12-13\n", + " Image downloaded for 2023-12-13\n", + " Image downloaded for 2023-12-14\n", + " Image downloaded for 2023-12-14\n", + " Image downloaded for 2023-12-15\n", + " Image downloaded for 2023-12-15\n", + " Image downloaded for 2023-12-16\n", + " Image downloaded for 2023-12-16\n", + " Image downloaded for 2023-12-17\n", + " Image downloaded for 2023-12-17\n", + " Image downloaded for 2023-12-18\n", + " Image downloaded for 2023-12-18\n", + " Image downloaded for 2023-12-19\n", + " Image downloaded for 2023-12-19\n", + " Image downloaded for 2023-12-20\n", + " Image downloaded for 2023-12-20\n", + " Image downloaded for 2023-12-21\n", + " Image downloaded for 2023-12-21\n", + " Image downloaded for 2023-12-22\n", + " Image downloaded for 2023-12-22\n", + " Image downloaded for 2023-12-23\n", + " Image downloaded for 2023-12-23\n", + " Image downloaded for 2023-12-24\n", + " Image downloaded for 2023-12-24\n", + " Image downloaded for 2023-12-25\n", + " Image downloaded for 2023-12-25\n", + " Image downloaded for 2023-12-26\n", + " Image downloaded for 2023-12-26\n", + " Image downloaded for 2023-12-27\n", + " Image downloaded for 2023-12-27\n", + " Image downloaded for 2023-12-28\n", + " Image downloaded for 2023-12-28\n", + " Image downloaded for 2023-12-29\n", + " Image downloaded for 2023-12-29\n", + " Image downloaded for 2023-12-30\n", + " Image downloaded for 2023-12-30\n", + " Image downloaded for 2023-12-31\n", + " Image downloaded for 2023-12-31\n", + " Image downloaded for 2024-01-01\n", + " Image downloaded for 2024-01-01\n", + " Image downloaded for 2024-01-02\n", + " Image downloaded for 2024-01-02\n", + " Image downloaded for 2024-01-03\n", + " Image downloaded for 2024-01-03\n", + " Image downloaded for 2024-01-04\n", + " Image downloaded for 2024-01-04\n", + " Image downloaded for 2024-01-05\n", + " Image downloaded for 2024-01-05\n", + " Image downloaded for 2024-01-06\n", + " Image downloaded for 2024-01-06\n", + " Image downloaded for 2024-01-07\n", + " Image downloaded for 2024-01-07\n", + " Image downloaded for 2024-01-08\n", + " Image downloaded for 2024-01-08\n", + " Image downloaded for 2024-01-09\n", + " Image downloaded for 2024-01-09\n", + " Image downloaded for 2024-01-10\n", + " Image downloaded for 2024-01-10\n", + " Image downloaded for 2024-01-11\n", + " Image downloaded for 2024-01-11\n", + " Image downloaded for 2024-01-12\n", + " Image downloaded for 2024-01-12\n", + " Image downloaded for 2024-01-13\n", + " Image downloaded for 2024-01-13\n", + " Image downloaded for 2024-01-14\n", + " Image downloaded for 2024-01-14\n", + " Image downloaded for 2024-01-15\n", + " Image downloaded for 2024-01-15\n", + " Image downloaded for 2024-01-16\n", + " Image downloaded for 2024-01-16\n", + " Image downloaded for 2024-01-17\n", + " Image downloaded for 2024-01-17\n", + " Image downloaded for 2024-01-18\n", + " Image downloaded for 2024-01-18\n", + " Image downloaded for 2024-01-19\n", + " Image downloaded for 2024-01-19\n", + " Image downloaded for 2024-01-20\n", + " Image downloaded for 2024-01-20\n", + " Image downloaded for 2024-01-21\n", + " Image downloaded for 2024-01-21\n", + " Image downloaded for 2024-01-22\n", + " Image downloaded for 2024-01-22\n", + " Image downloaded for 2024-01-23\n", + " Image downloaded for 2024-01-23\n", + " Image downloaded for 2024-01-24\n", + " Image downloaded for 2024-01-24\n", + " Image downloaded for 2024-01-25\n", + " Image downloaded for 2024-01-25\n", + " Image downloaded for 2024-01-26\n", + " Image downloaded for 2024-01-26\n", + " Image downloaded for 2024-01-27\n", + " Image downloaded for 2024-01-27\n", + " Image downloaded for 2024-01-28\n", + " Image downloaded for 2024-01-28\n", + " Image downloaded for 2024-01-29\n", + " Image downloaded for 2024-01-29\n", + " Image downloaded for 2024-01-30\n", + " Image downloaded for 2024-01-30\n", + " Image downloaded for 2024-01-31\n", + " Image downloaded for 2024-01-31\n", + " Image downloaded for 2024-02-01\n", + " Image downloaded for 2024-02-01\n", + " Image downloaded for 2024-02-02\n", + " Image downloaded for 2024-02-02\n", + " Image downloaded for 2024-02-03\n", + " Image downloaded for 2024-02-03\n", + " Image downloaded for 2024-02-04\n", + " Image downloaded for 2024-02-04\n", + " Image downloaded for 2024-02-05\n", + " Image downloaded for 2024-02-05\n", + " Image downloaded for 2024-02-06\n", + " Image downloaded for 2024-02-06\n", + " Image downloaded for 2024-02-07\n", + " Image downloaded for 2024-02-07\n", + " Image downloaded for 2024-02-08\n", + " Image downloaded for 2024-02-08\n", + " Image downloaded for 2024-02-09\n", + " Image downloaded for 2024-02-09\n", + " Image downloaded for 2024-02-10\n", + " Image downloaded for 2024-02-10\n", + " Image downloaded for 2024-02-11\n", + " Image downloaded for 2024-02-11\n", + " Image downloaded for 2024-02-12\n", + " Image downloaded for 2024-02-12\n", + " Image downloaded for 2024-02-13\n", + " Image downloaded for 2024-02-13\n", + " Image downloaded for 2024-02-14\n", + " Image downloaded for 2024-02-14\n", + " Image downloaded for 2024-02-15\n", + " Image downloaded for 2024-02-15\n", + " Image downloaded for 2024-02-16\n", + " Image downloaded for 2024-02-16\n", + " Image downloaded for 2024-02-17\n", + " Image downloaded for 2024-02-17\n", + " Image downloaded for 2024-02-18\n", + " Image downloaded for 2024-02-18\n", + " Image downloaded for 2024-02-19\n", + " Image downloaded for 2024-02-19\n", + " Image downloaded for 2024-02-20\n", + " Image downloaded for 2024-02-20\n", + " Image downloaded for 2024-02-21\n", + " Image downloaded for 2024-02-21\n", + " Image downloaded for 2024-02-22\n", + " Image downloaded for 2024-02-22\n", + " Image downloaded for 2024-02-23\n", + " Image downloaded for 2024-02-23\n", + " Image downloaded for 2024-02-24\n", + " Image downloaded for 2024-02-24\n", + " Image downloaded for 2024-02-25\n", + " Image downloaded for 2024-02-25\n", + " Image downloaded for 2024-02-26\n", + " Image downloaded for 2024-02-26\n", + " Image downloaded for 2024-02-27\n", + " Image downloaded for 2024-02-27\n", + " Image downloaded for 2024-02-28\n", + " Image downloaded for 2024-02-28\n", + " Image downloaded for 2024-02-29\n", + " Image downloaded for 2024-02-29\n", + " Image downloaded for 2024-03-01\n", + " Image downloaded for 2024-03-01\n", + " Image downloaded for 2024-03-02\n", + " Image downloaded for 2024-03-02\n", + " Image downloaded for 2024-03-03\n", + " Image downloaded for 2024-03-03\n", + " Image downloaded for 2024-03-04\n", + " Image downloaded for 2024-03-04\n", + " Image downloaded for 2024-03-05\n", + " Image downloaded for 2024-03-05\n", + " Image downloaded for 2024-03-06\n", + " Image downloaded for 2024-03-06\n", + " Image downloaded for 2024-03-07\n", + " Image downloaded for 2024-03-07\n", + " Image downloaded for 2024-03-08\n", + " Image downloaded for 2024-03-08\n", + " Image downloaded for 2024-03-09\n", + " Image downloaded for 2024-03-09\n", + " Image downloaded for 2024-03-10\n", + " Image downloaded for 2024-03-10\n", + " Image downloaded for 2024-03-11\n", + " Image downloaded for 2024-03-11\n", + " Image downloaded for 2024-03-12\n", + " Image downloaded for 2024-03-12\n" + ] + } + ], + "source": [ + "# Load areas outside the loop if they remain constant\n", + "bbox_area = json.dumps(chosen_area)\n", + "areas = json.loads(os.getenv('BBOX', bbox_area))\n", + "resolution = 3\n", + "\n", + "for slot in slots:\n", + " for area in areas:\n", + " bbox = BBox(bbox=area, crs=CRS.WGS84)\n", + " size = bbox_to_dimensions(bbox, resolution=resolution)\n", + " download_function(slot, bbox, size)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "68db3c15-6f94-432e-b315-c329e4251b21", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for slot in slots:\n", + " merge_files(slot)" + ] + }, + { + "cell_type": "markdown", + "id": "4274d8e7-1ea3-46db-9528-069ede0b2132", + "metadata": { + "tags": [] + }, + "source": [ + "#### Delete intermediate files\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "cb3fa856-a550-4899-844a-e69209bba3ad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Skipping empty_folders function.\n" + ] + } + ], + "source": [ + "# List of folder names\n", + "\n", + "folders_to_empty = [BASE_PATH / 'merged_virtual', BASE_PATH_SINGLE_IMAGES]\n", + " \n", + "# Function to empty folders\n", + "\n", + "# Function to empty folders\n", + "def empty_folders(folders, run=True):\n", + " if not run:\n", + " print(\"Skipping empty_folders function.\")\n", + " return\n", + " \n", + " for folder in folders:\n", + " try:\n", + " for filename in os.listdir(folder):\n", + " file_path = os.path.join(folder, filename)\n", + " try:\n", + " if os.path.isfile(file_path):\n", + " os.unlink(file_path)\n", + " elif os.path.isdir(file_path):\n", + " shutil.rmtree(file_path)\n", + " except Exception as e:\n", + " print(f\"Error: {e}\")\n", + " print(f\"Emptied folder: {folder}\")\n", + " except OSError as e:\n", + " print(f\"Error: {e}\")\n", + "\n", + "# Call the function to empty folders only if the 'run' parameter is set to True\n", + "empty_folders(folders_to_empty, run=empty_folder_question)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/r_app/2_CI_data_prep.R b/r_app/2_CI_data_prep.R index 9db7db9..1706267 100644 --- a/r_app/2_CI_data_prep.R +++ b/r_app/2_CI_data_prep.R @@ -11,6 +11,7 @@ library(lubridate) library(exactextractr) library(CIprep) +source(here("r_app", "parameters_project.R")) # Vang alle command line argumenten op args <- commandArgs(trailingOnly = TRUE) @@ -55,6 +56,8 @@ cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") daily_vrt <- here(data_dir, "vrt") harvest_dir <- here(data_dir, "HarvestData") + +dir.create(here(laravel_storage_dir)) dir.create(here(data_dir)) dir.create(here(extracted_CI_dir)) dir.create(here(daily_CI_vals_dir)) @@ -70,7 +73,7 @@ dir.create(harvest_dir) dates <- date_list(weeks_ago) #load pivot geojson -pivot_sf_q <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% vect() +# pivot_sf_q <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% vect() raster_files <- list.files(planet_tif_folder,full.names = T, pattern = ".tif") head(raster_files) @@ -79,65 +82,72 @@ filtered_files <- map(dates$days_filter, ~ raster_files[grepl(pattern = .x, x = compact() %>% flatten_chr() +# filtered_files <- raster_files #for first CI extraction -#rasters_masked <- map(filtered_files, mask_raster, fields = pivot_sf_q) %>% set_names(filtered_files) - - -# rasters_masked <- list() # Creëer een lege lijst om de resultaten op te slaan - -# for (i in seq_along(filtered_files[1])) { -# file_name <- filtered_files[i] -# result <- mask_raster(file_name, fields = pivot_sf_q) -# rasters_masked[[file_name]] <- result +# create_mask_and_crop <- function(file, field_boundaries) { +# message("starting ", file) +# CI <- rast(file) +# # names(CI) <- c("green","nir") +# message("raster loaded") +# +# CI <- CI[[2]]/CI[[1]]-1 +# # CI <- CI$nir/CI$green-1 +# message("CI calculated") +# CI <- terra::crop(CI, field_boundaries, mask = TRUE) #%>% CI_func() +# +# new_file <- here(merged_final, paste0(tools::file_path_sans_ext(basename(file)), ".tif")) +# writeRaster(CI, new_file, overwrite = TRUE) +# +# vrt_file <- here(daily_vrt, paste0(tools::file_path_sans_ext(basename(file)), ".vrt")) +# terra::vrt(new_file, vrt_file, overwrite = TRUE) +# +# return(CI) # } - -create_mask_and_crop <- function(file, pivot_sf_q) { -message("starting ", file) - CI <- rast(file) -# names(CI) <- c("green","nir") -message("raster loaded") - - CI <- CI[[2]]/CI[[1]]-1 -# CI <- CI$nir/CI$green-1 -message("CI calculated") - CI <- terra::crop(CI, pivot_sf_q, mask = TRUE) #%>% CI_func() - - # v_crop[v_crop == 0] <- NA +create_mask_and_crop <- function(file, field_boundaries) { + # file <- filtered_files[5] + message("starting ", file) + loaded_raster <- rast(file) + names(loaded_raster) <- c("Red", "Green", "Blue", "NIR") + # names(CI) <- c("green","nir") + message("raster loaded") + + # CI <- CI[[2]] / CI[[1]] - 1 + CI <- loaded_raster$NIR / loaded_raster$Green - 1 + + loaded_raster$CI <- CI + # CI <- CI$nir/CI$green-1 + message("CI calculated") + loaded_raster <- terra::crop(loaded_raster, field_boundaries, mask = TRUE) #%>% CI_func() + + loaded_raster[loaded_raster == 0] <- NA # names(v_crop) <- c("red", "green", "blue","nir", "cloud" ,"CI") # v_crop$CI <- v_crop$CI - 1 - new_file <- here(merged_final, paste0(tools::file_path_sans_ext(basename(file)), ".tif")) - writeRaster(CI, new_file, overwrite = TRUE) - + new_file <- here(merged_final, paste0(tools::file_path_sans_ext(basename(file)), ".tif")) + writeRaster(loaded_raster, new_file, overwrite = TRUE) + vrt_file <- here(daily_vrt, paste0(tools::file_path_sans_ext(basename(file)), ".vrt")) terra::vrt(new_file, vrt_file, overwrite = TRUE) - + # v_crop <- mask_raster(v, pivot_sf_q) - return(CI) + return(loaded_raster) } -# rasters_masked <- map(filtered_files, create_mask_and_crop, pivot_sf_q) -# list_global <- list() vrt_list <- list() for (file in filtered_files) { - v_crop <- create_mask_and_crop(file, pivot_sf_q) + v_crop <- create_mask_and_crop(file, field_boundaries) emtpy_or_full <- global(v_crop, "notNA") vrt_file <- here(daily_vrt, paste0(tools::file_path_sans_ext(basename(file)), ".vrt")) if(emtpy_or_full[1,] > 10000){ - # list_global[file] <- file vrt_list[vrt_file] <- vrt_file }else{ file.remove(vrt_file) - # file.remove(file) - # message(file, " removed") + } - # Save the processed raster to a new file - # output_file <- here(data_dir, "vrt", paste0(basename(file), "_processed")) - # terra::writeRaster(v_crop, output_file, overwrite = TRUE) message(file, " processed") gc() } @@ -145,38 +155,9 @@ for (file in filtered_files) { # list_global <- list_global %>% flatten_chr() vrt_list <- vrt_list %>% flatten_chr() -#testing writing raster -# for(i in seq_along(filtered_files)){ -# message("starting ", i) -# x <- mask_raster(filtered_files[i], fields = pivot_sf_q) -# writeRaster(x, filtered_files[i], overwrite=TRUE) -# message("writing ", i) -# } - - -# rasters_masked[sapply(rasters_masked, is.null)] <- NULL - -# rasters_masked <- setNames(list_global, map_chr(names(list_global), date_extract)) - -# }) - - total_pix_area <- rast(vrt_list[1]) %>% terra::subset(1) %>% setValues(1) %>% - crop(pivot_sf_q, mask = TRUE) %>% + crop(field_boundaries, mask = TRUE) %>% global(., fun="notNA") #%>% -# as.matrix() %>% -# `[`(1, 1) - -# total_pix_area <- rast(filtered_files[1]) %>% subset(1) %>% crop(pivot_sf_q, mask = TRUE) %>% freq(., usenames = TRUE) - -# list_global -# rast(list_global[1])[[5]] %>% plot() - -# vrt_files <- list.files(here(data_dir, "vrt"),full.names = T) -# vrt_days_filter <- tools::file_path_sans_ext(basename(list_global)) -# vrt_list <- map(vrt_days_filter, ~ vrt_files[grepl(pattern = .x, x = vrt_files)]) %>% -# compact() %>% -# flatten_chr() layer_5_list <- purrr::map(vrt_list, function(vrt_list) { rast(vrt_list[1]) %>% terra::subset(1) @@ -190,11 +171,6 @@ missing_pixels_count <- layer_5_list %>% global(., fun="notNA") %>% thres_40perc = as.integer(missing_pixels_percentage < 45) ) -# cloud_perc_list <- freq(layer_5_list, usenames = TRUE) %>% -# mutate(cloud_perc = (100 -((count/sum(total_pix_area$notNA))*100)), -# cloud_thres_5perc = as.integer(cloud_perc < 5), -# cloud_thres_40perc = as.integer(cloud_perc < 40)) %>% -# rename(Date = layer) %>% select(-value, -count) index_5perc <- which(missing_pixels_count$thres_5perc == max(missing_pixels_count$thres_5perc) ) index_40perc <- which(missing_pixels_count$thres_40perc == max(missing_pixels_count$thres_40perc)) @@ -208,14 +184,14 @@ if(sum(missing_pixels_count$thres_5perc)>1){ rsrc <- sprc(cloudy_rasters_list) x <- mosaic(rsrc, fun = "max") - names(x) <- "CI" - + # names(x) <- "CI" + names(x) <- c("Red", "Green", "Blue", "NIR", "CI") }else if(sum(missing_pixels_count$thres_5perc)==1){ message("Only 1 raster without clouds (<5%)") x <- rast(vrt_list[index_5perc[1]]) - names(x) <- c("CI") - + # names(x) <- c("CI") + names(x) <- c("Red", "Green", "Blue", "NIR", "CI") }else if(sum(missing_pixels_count$thres_40perc)>1){ message("More than 1 image contains clouds, composite made of <40% cloud cover images") @@ -223,26 +199,26 @@ if(sum(missing_pixels_count$thres_5perc)>1){ rsrc <- sprc(cloudy_rasters_list) x <- mosaic(rsrc, fun = "max") - names(x) <- "CI" - + # names(x) <- "CI" + names(x) <- c("Red", "Green", "Blue", "NIR", "CI") }else if(sum(missing_pixels_count$thres_40perc)==1){ message("Only 1 image available but contains clouds") x <- rast(vrt_list[index_40perc[1]]) - names(x) <- c("CI") - + # names(x) <- c("CI") + names(x) <- c("Red", "Green", "Blue", "NIR", "CI") }else{ message("No cloud free images available, all images combined") rsrc <- sprc(vrt_list) x <- mosaic(rsrc, fun = "max") # x <- rast(vrt_list[1]) %>% setValues(NA) - names(x) <- c("CI") - + # names(x) <- c("CI") + names(x) <- c("Red", "Green", "Blue", "NIR", "CI") } -plot(x$CI, main = paste("CI map", dates$week)) -#plotRGB(x, main = paste("RGB image week", dates$week)) +plot(x$CI, main = paste("CI map ", dates$week)) +plotRGB(x, main = paste("RGB map ", dates$week)) file_path_tif <- here(weekly_CI_mosaic ,paste0("week_", sprintf("%02d", dates$week), "_", dates$year, ".tif")) writeRaster(x, file_path_tif, overwrite=TRUE) @@ -257,7 +233,7 @@ extract_rasters_daily <- function(file, field_geojson, quadrants = TRUE, save_di field_geojson <- sf::st_as_sf(field_geojson) x <- rast(file[1]) date <- date_extract(file) - pivot_stats <- cbind(field_geojson, mean_CI = round(exactextractr::exact_extract(x, field_geojson, fun = "mean"), 2)) %>% + pivot_stats <- cbind(field_geojson, mean_CI = round(exactextractr::exact_extract(x$CI, field_geojson, fun = "mean"), 2)) %>% st_drop_geometry() %>% rename("{date}" := mean_CI) save_suffix <- if (quadrants){"quadrant"} else {"whole_field"} save_path <- here(save_dir, paste0("extracted_", date, "_", save_suffix, ".rds")) @@ -265,33 +241,31 @@ extract_rasters_daily <- function(file, field_geojson, quadrants = TRUE, save_di } # pivot_sf_q <- st_read(here("..", "Data", "pivot_20210625.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% vect() -pivot_sf <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% group_by(pivot) %>% summarise() %>% vect() -message("pivot loaded") +# pivot_sf <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% group_by(pivot) %>% summarise() %>% vect() +# message("pivot loaded") raster_files_NEW <- list.files(merged_final,full.names = T, pattern = ".tif") -pivots_dates0 <- readRDS(here(harvest_dir, "harvest_data_new")) %>% filter( - pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", - "1.14" , "1.16" , "1.17" , "1.18" ,"2.1", "2.2", "2.3" , "2.4", "2.5", "3.1", "3.2", "3.3", - "4.1", "4.2", "4.3", "4.4", "4.5", "4.6", "5.1" ,"5.2", "5.3", "5.4", "6.1", "6.2", "DL1.1", "DL1.3") -) -harvesting_data <- pivots_dates0 %>% - select(c("pivot_quadrant", "season_start_2021", "season_end_2021", "season_start_2022", "season_end_2022", "season_start_2023", "season_end_2023", "season_start_2024", "season_end_2024")) %>% - pivot_longer(cols = starts_with("season"), names_to = "Year", values_to = "value") %>% - separate(Year, into = c("name", "Year"), sep = "(?<=season_start|season_end)\\_", remove = FALSE) %>% - mutate(name = str_to_title(name)) %>% - pivot_wider(names_from = name, values_from = value) %>% - rename(Field = pivot_quadrant) +# pivots_dates0 <- readRDS(here(harvest_dir, "harvest_data_new")) %>% filter( +# pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", +# "1.14" , "1.16" , "1.17" , "1.18" ,"2.1", "2.2", "2.3" , "2.4", "2.5", "3.1", "3.2", "3.3", +# "4.1", "4.2", "4.3", "4.4", "4.5", "4.6", "5.1" ,"5.2", "5.3", "5.4", "6.1", "6.2", "DL1.1", "DL1.3") +# ) -pivots_data_present <- unique(pivots_dates0$pivot_quadrant) -quadrant_list <- pivots_data_present +# harvesting_data <- pivots_dates0 %>% +# select(c("pivot_quadrant", "season_start_2021", "season_end_2021", "season_start_2022", "season_end_2022", "season_start_2023", "season_end_2023", "season_start_2024", "season_end_2024")) %>% +# pivot_longer(cols = starts_with("season"), names_to = "Year", values_to = "value") %>% +# separate(Year, into = c("name", "Year"), sep = "(?<=season_start|season_end)\\_", remove = FALSE) %>% +# mutate(name = str_to_title(name)) %>% +# pivot_wider(names_from = name, values_from = value) %>% +# rename(Field = pivot_quadrant) #If run for the firsttime, it will extract all data since the start and put into a table.rds. otherwise it will only add on to the existing table. if (new_project_question == TRUE) { print("combined_CI_data.rds does not exist. Preparing combined_CI_data.rds file for all available images.") - walk(raster_files_NEW, extract_rasters_daily, field_geojson= pivot_sf_q, quadrants = TRUE, daily_CI_vals_dir) + walk(raster_files_NEW, extract_rasters_daily, field_geojson= field_boundaries, quadrants = F, daily_CI_vals_dir) extracted_values <- list.files(here(daily_CI_vals_dir), full.names = TRUE) @@ -313,7 +287,7 @@ if (new_project_question == TRUE) { compact() %>% flatten_chr() - walk(filtered_files, extract_rasters_daily, field_geojson= pivot_sf_q, quadrants = TRUE, daily_CI_vals_dir) + walk(filtered_files, extract_rasters_daily, field_geojson= field_boundaries, quadrants = TRUE, daily_CI_vals_dir) extracted_values <- list.files(daily_CI_vals_dir, full.names = TRUE) @@ -326,34 +300,13 @@ if (new_project_question == TRUE) { group_by(pivot_quadrant) %>% summarise(across(everything(), ~ first(na.omit(.)))) - combined_CI_data <- readRDS(here(cumulative_CI_vals_dir,"combined_CI_data.rds")) %>% drop_na(pivot_quadrant) + combined_CI_data <- readRDS(here(cumulative_CI_vals_dir,"combined_CI_data.rds")) #%>% drop_na(pivot_quadrant) pivot_stats2 <- bind_rows(pivot_stats, combined_CI_data) + # pivot_stats2 <- combined_CI_data print("All CI values extracted from latest 7 images.") } - - - -#extracted_values <- map(dates$days_filter, ~ extracted_values[grepl(pattern = .x, x = extracted_values)]) %>% -# compact() %>% -# flatten_chr() - -#combine them into one df - - - - #saveRDS(pivot_stats, here(cumulative_CI_vals_dir,"combined_CI_data.rds")) #used to save the rest of the data into one file - - -#load historic CI data and update it with last week of CI data -# combined_CI_data <- readRDS(here(cumulative_CI_vals_dir,"combined_CI_data.rds")) %>% drop_na(pivot_quadrant) -# pivot_stats2 <- bind_rows(pivot_stats, combined_CI_data) -# pivot_stats2 <- purrr::map(list.files(here(daily_CI_vals_dir), full.names = TRUE, pattern = "quadrant"), readRDS) %>% list_rbind() %>% group_by(pivot_quadrant) %>% -# summarise(across(everything(), ~ first(na.omit(.)))) - - - # gather data into long format for easier calculation and visualisation pivot_stats_long <- pivot_stats2 %>% tidyr::gather("Date", value, -pivot_quadrant, -pivot ) %>% @@ -362,36 +315,73 @@ pivot_stats_long <- pivot_stats2 %>% ) %>% drop_na(c("value","Date")) %>% mutate(value = as.numeric(value))%>% - filter_all(all_vars(!is.infinite(.)))%>% - rename(Field = pivot_quadrant) %>% + filter_all(all_vars(!is.infinite(.))) %>% + rename(Field = pivot_quadrant, + subField = Field) %>% unique() -# #2021 -pivots_dates_Data_2021 <- pivots_dates0 %>% filter(!is.na(season_start_2021)) -pivot_select_model_Data_2021 <- unique(pivots_dates_Data_2021$pivot_quadrant) -# #2022 -pivots_dates_Data_2022 <- pivots_dates0 %>% filter(!is.na(season_end_2022)) -pivot_select_model_Data_2022 <- unique(pivots_dates_Data_2022$pivot_quadrant ) -# #2023 -pivots_dates_Data_2023 <- pivots_dates0 %>% filter(!is.na(season_start_2023)) -pivot_select_model_Data_2023 <- unique(pivots_dates_Data_2023$pivot_quadrant) -# #2024 -pivots_dates_Data_2024 <- pivots_dates0 %>% filter(!is.na(season_start_2024)) -pivot_select_model_Data_2024 <- unique(pivots_dates_Data_2024$pivot_quadrant) +# #2021 +# pivot_select_model_Data_2021 <- harvesting_data %>% filter(Year == 2021) %>% pull(Field) +# +# pivot_select_model_Data_2022 <- harvesting_data %>% filter(Year == 2022) %>% pull(Field) + +pivot_select_model_Data_2023 <- harvesting_data %>% filter(Year == 2023) %>% pull(Field) + +pivot_select_model_Data_2024 <- harvesting_data %>% filter(Year == 2024) %>% pull(Field) + +# pivots_dates_Data_2022 <- pivots_dates0 %>% filter(!is.na(season_end_2022)) +# pivot_select_model_Data_2022 <- unique(pivots_dates_Data_2022$pivot_quadrant ) +# +# pivots_dates_Data_2023 <- pivots_dates0 %>% filter(!is.na(season_start_2023)) +# pivot_select_model_Data_2023 <- unique(pivots_dates_Data_2023$pivot_quadrant) +# +# pivots_dates_Data_2024 <- pivots_dates0 %>% filter(!is.na(season_start_2024)) +# pivot_select_model_Data_2024 <- unique(pivots_dates_Data_2024$pivot_quadrant) + +extract_CI_data <- function(field_names, harvesting_data, field_CI_data, season) { + # field_names = "AG1D06P" + # field_names = "1.1A" + # harvesting_data = harvesting_data + # field_CI_data = pivot_stats_long + # season= 2023 + + filtered_harvesting_data <- harvesting_data %>% + filter(Year == season, subField %in% field_names) + + filtered_field_CI_data <- field_CI_data %>% + filter(subField %in% field_names) + + # CI <- map_df(field_names, ~ { + ApproxFun <- approxfun(x = filtered_field_CI_data$Date, y = filtered_field_CI_data$value) + Dates <- seq.Date(ymd(min(filtered_field_CI_data$Date)), ymd(max(filtered_field_CI_data$Date)), by = 1) + LinearFit <- ApproxFun(Dates) + + CI <- data.frame(Date = Dates, FitData = LinearFit) %>% + left_join(., filtered_field_CI_data, by = "Date") %>% + filter(Date > filtered_harvesting_data$Season_start & Date < filtered_harvesting_data$Season_end) %>% + mutate(DOY = seq(1, n(), 1), + model = paste0("Data", season, " : ", field_names), + season = season, + subField = field_names) + # }) #%>% + #{if (length(field_names) > 0) message("Done!")} + + return(CI) +} ## Extracting the correct CI values -#Data_2021 <- map(pivot_select_model_Data_2021, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2021)) %>% list_rbind() -message('2021') -Data_2022 <- map(pivot_select_model_Data_2022, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2022)) %>% list_rbind() -message('2022') +# Data_2021 <- map(pivot_select_model_Data_2021, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2021)) %>% list_rbind() +# message('2021') +# Data_2022 <- map(pivot_select_model_Data_2022, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2022)) %>% list_rbind() +# message('2022') Data_2023 <- map(pivot_select_model_Data_2023, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2023)) %>% list_rbind() message('2023') Data_2024 <- map(pivot_select_model_Data_2024, ~ extract_CI_data(.x, harvesting_data = harvesting_data, field_CI_data = pivot_stats_long, season = 2024)) %>% list_rbind() message('2024') -CI_all <- rbind(Data_2022, Data_2023, Data_2024) +CI_all <- rbind(Data_2023, Data_2024) message('CI_all created') #CI_all <- Data_2023 @@ -403,4 +393,4 @@ head(CI_all) message('show head') saveRDS(CI_all, here(cumulative_CI_vals_dir,"All_pivots_Cumulative_CI_quadrant_year_v2.rds")) -message('rds saved') \ No newline at end of file +message('rds saved') diff --git a/r_app/CI_report_dashboard_planet.Rmd b/r_app/CI_report_dashboard_planet.Rmd index fe09d0c..381705f 100644 --- a/r_app/CI_report_dashboard_planet.Rmd +++ b/r_app/CI_report_dashboard_planet.Rmd @@ -43,13 +43,17 @@ library(rsample) library(caret) library(randomForest) library(CAST) + + + ``` ```{r directories, message=FALSE, warning=FALSE, include=FALSE} laravel_storage_dir <- here("laravel_app/storage/app/",params$data_dir) +laravel_storage_dir <- here("laravel_app/storage/app",project_dir) data_dir_project <- here(laravel_storage_dir, "Data") -message('DATA_DIR',data_dir_project) +# message('DATA_DIR',data_dir_project) extracted_CI_dir <- here(data_dir_project, "extracted_ci") daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") @@ -57,8 +61,8 @@ harvest_dir <- here(data_dir_project, "HarvestData") weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") - -s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/chemba_S2/" +source(here("r_app", "parameters_project.R")) +# s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/chemba_S2/" ``` @@ -119,21 +123,22 @@ CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_3, last_week_dif_raster_abs <- (CI - CI_m1) three_week_dif_raster_abs <- (CI - CI_m3) -AllPivots0 <-st_read(here(data_dir_project, "pivot.geojson")) +# AllPivots0 <-st_read(here(data_dir_project, "pivot.geojson")) + # AllPivots0$pivot <- factor(AllPivots0$pivot, levels = c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14" , "1.16" , "1.17" , "1.18" ,"2.1", "2.2", "2.3" , "2.4", "2.5", "3.1", "3.2", "3.3", "4.1", "4.2", "4.3", "4.4", "4.5", "4.6", "5.1" ,"5.2", "5.3", "5.4", "6.1", "6.2", "DL1.1", "DL1.3")) -AllPivots0 +AllPivots0 <- field_boundaries_sf -joined_spans <-st_read(here(data_dir_project, "span.geojson")) %>% st_transform(crs(AllPivots0)) +# joined_spans <-st_read(here(data_dir_project, "span.geojson")) %>% st_transform(crs(AllPivots0)) -pivots_dates <- readRDS(here(harvest_dir, "harvest_data_new")) +# pivots_dates <- readRDS(here(harvest_dir, "harvest_data_new")) # pivots_dates$pivot <- factor(pivots_dates$pivot, levels = c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14" , "1.16" , "1.17" , "1.18" ,"2.1", "2.2", "2.3" , "2.4", "2.5", "3.1", "3.2", "3.3", "4.1", "4.2", "4.3", "4.4", "4.5", "4.6", "5.1" ,"5.2", "5.3", "5.4", "6.1", "6.2", "DL1.1", "DL1.3")) -AllPivots <- merge(AllPivots0, pivots_dates, by = "pivot_quadrant") %>% - rename(pivot = pivot.x) #%>% select(-pivot.y) +AllPivots <- merge(AllPivots0, harvesting_data, by = c("Field", "subField")) #%>% + # rename(pivot = pivot.x) #%>% select(-pivot.y) head(AllPivots) -AllPivots_merged <- AllPivots %>% - group_by(pivot) %>% summarise() +AllPivots_merged <- AllPivots %>% + group_by(Field) %>% summarise() AllPivots_merged <- st_transform(AllPivots_merged, crs = proj4string(CI)) @@ -208,7 +213,7 @@ create_CI_map <- function(pivot_raster, pivot_shape, pivot_spans, show_legend = tm_layout(main.title = paste0("\nMax CI week ", week,"\n", age, " weeks old"), main.title.size = 0.7, legend.show = show_legend) + tm_shape(pivot_shape) + - tm_borders(lwd = 3) + tm_text("pivot_quadrant", size = 1/2) + + tm_borders(lwd = 3) + tm_text("subField", size = 1/2) + tm_shape(pivot_spans) + tm_borders(lwd = 0.5, alpha=0.5) } @@ -218,16 +223,16 @@ create_CI_diff_map <- function(pivot_raster, pivot_shape, pivot_spans, show_lege tm_layout(main.title = paste0("CI change week ", week_1, "- week ",week_2, "\n", age," weeks old"), main.title.size = 0.7, legend.show = show_legend) + tm_shape(pivot_shape) + - tm_borders(lwd = 3) + tm_text("pivot_quadrant", size = 1/2) + + tm_borders(lwd = 3) + tm_text("subField", size = 1/2) + tm_shape(pivot_spans) + tm_borders(lwd = 0.5, alpha=0.5) } ci_plot <- function(pivotName){ # pivotName = "1.1" - pivotShape <- AllPivots_merged %>% terra::subset(pivot %in% pivotName) %>% st_transform(crs(CI)) - age <- AllPivots %>% dplyr::filter(pivot %in% pivotName) %>% st_drop_geometry() %>% dplyr::select(Age) %>% unique() + pivotShape <- AllPivots_merged %>% terra::subset(Field %in% pivotName) %>% st_transform(crs(CI)) + age <- AllPivots %>% dplyr::filter(Field %in% pivotName) %>% st_drop_geometry() %>% dplyr::select(Age) %>% unique() - AllPivots2 <- AllPivots %>% dplyr::filter(pivot %in% pivotName) + AllPivots2 <- AllPivots %>% dplyr::filter(Field %in% pivotName) singlePivot <- CI %>% crop(., pivotShape) %>% mask(., pivotShape) @@ -238,9 +243,9 @@ ci_plot <- function(pivotName){ abs_CI_last_week <- last_week_dif_raster_abs %>% crop(., pivotShape) %>% mask(., pivotShape) abs_CI_three_week <- three_week_dif_raster_abs %>% crop(., pivotShape) %>% mask(., pivotShape) - planting_date <- pivots_dates %>% dplyr::filter(pivot %in% pivotName) %>% ungroup() %>% dplyr::select(planting_date) %>% unique() + planting_date <- pivots_dates %>% dplyr::filter(Field %in% pivotName) %>% ungroup() %>% dplyr::select(planting_date) %>% unique() - joined_spans2 <- joined_spans %>% st_transform(crs(pivotShape)) %>% dplyr::filter(pivot %in% pivotName) %>% st_crop(., pivotShape) + joined_spans2 <- joined_spans %>% st_transform(crs(pivotShape)) %>% dplyr::filter(Field %in% pivotName) %>% unique() %>% st_crop(., pivotShape) CImap_m2 <- create_CI_map(singlePivot_m2, AllPivots2, joined_spans2, show_legend= T, legend_is_portrait = T, week = week_minus_2, age = age -2) CImap_m1 <- create_CI_map(singlePivot_m1, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age -1) @@ -266,7 +271,7 @@ ci_plot <- function(pivotName){ cum_ci_plot <- function(pivotName){ # pivotName = "1.17" - data_ci <- CI_quadrant %>% filter(pivot == pivotName) + data_ci <- CI_quadrant %>% filter(Field == pivotName) data_ci2 <- data_ci %>% mutate(CI_rate = cumulative_CI/DOY, week = week(Date))%>% group_by(Field) %>% mutate(mean_rolling10 = rollapplyr(CI_rate , width = 10, FUN = mean, partial = TRUE)) #%>% @@ -275,38 +280,38 @@ cum_ci_plot <- function(pivotName){ max_date = max(Date), days = max_date - min_date) - perfect_pivot_raw <- CI_quadrant %>% group_by(pivot) %>% filter(pivot == "5.1" & season == "2022") %>% - group_by(DOY) %>% summarise(cumulative_CI = mean(cumulative_CI)) %>% mutate(CI_rate = cumulative_CI/DOY) %>% - mutate(mean_rolling10 = rollapplyr(CI_rate , width = 10, FUN = mean, partial = TRUE)) + # perfect_pivot_raw <- CI_quadrant %>% group_by(Field) %>% filter(Field == "5.1" & season == "2022") %>% + # group_by(DOY) %>% summarise(cumulative_CI = mean(cumulative_CI)) %>% mutate(CI_rate = cumulative_CI/DOY) %>% + # mutate(mean_rolling10 = rollapplyr(CI_rate , width = 10, FUN = mean, partial = TRUE)) unique_seasons <- unique(date_preperation_perfect_pivot$season) - if(length(unique_seasons) == 3) { - unique_seasons <- unique_seasons[c(2,3)] - } else if(length(unique_seasons) == 4) { - unique_seasons <- unique_seasons[c(3,4)] - } else { - unique_seasons <- unique_seasons - } + # if(length(unique_seasons) == 3) { + # unique_seasons <- unique_seasons[c(2,3)] + # } else if(length(unique_seasons) == 4) { + # unique_seasons <- unique_seasons[c(3,4)] + # } else { + # unique_seasons <- unique_seasons + # } - perfect_pivot <- perfect_pivot_raw + # perfect_pivot <- perfect_pivot_raw + # + # for (s in unique_seasons) { + # season_dates <- seq(from = date_preperation_perfect_pivot$min_date[date_preperation_perfect_pivot$season == s], + # to = date_preperation_perfect_pivot$min_date[date_preperation_perfect_pivot$season == s] + nrow(perfect_pivot_raw) - 1, + # by = "1 day") + # col_name <- as.character(s) + # perfect_pivot <- dplyr::bind_cols(perfect_pivot, tibble(!!col_name := season_dates)) + # } + # + # perfect_pivot <- perfect_pivot %>% + # pivot_longer(cols = -c("DOY", "cumulative_CI", "CI_rate", "mean_rolling10"), + # names_to = "season", values_to = "Date") - for (s in unique_seasons) { - season_dates <- seq(from = date_preperation_perfect_pivot$min_date[date_preperation_perfect_pivot$season == s], - to = date_preperation_perfect_pivot$min_date[date_preperation_perfect_pivot$season == s] + nrow(perfect_pivot_raw) - 1, - by = "1 day") - col_name <- as.character(s) - perfect_pivot <- dplyr::bind_cols(perfect_pivot, tibble(!!col_name := season_dates)) - } - - perfect_pivot <- perfect_pivot %>% - pivot_longer(cols = -c("DOY", "cumulative_CI", "CI_rate", "mean_rolling10"), - names_to = "season", values_to = "Date") - - g <- ggplot() + + g <- ggplot(data= data_ci2 %>% filter(season %in% unique_seasons)) + facet_wrap(~season, scales = "free_x") + - geom_line(data= data_ci2 %>% filter(season %in% unique_seasons), aes(Date, mean_rolling10, col = Field)) + - geom_line(data= perfect_pivot, aes(Date , mean_rolling10, col = "Model CI (p5.1 Data 2022, \n date x axis is fictive)"), lty="11",size=1) + + geom_line( aes(Date, mean_rolling10, col = Field)) + + # geom_line(data= perfect_pivot, aes(Date , mean_rolling10, col = "Model CI (p5.1 Data 2022, \n date x axis is fictive)"), lty="11",size=1) + labs(title = paste("14 day rolling MEAN CI rate - Pivot ", pivotName))+ # scale_y_continuous(limits=c(0.5,3), breaks = seq(0.5, 3, 0.5))+ scale_x_date(date_breaks = "1 month", date_labels = "%m-%Y") + @@ -348,7 +353,7 @@ tm_shape(CI, unit = "m")+ tm_compass(position = c("right", "top"), text.color = "black") + tm_shape(AllPivots)+ tm_borders( col = "black") + - tm_text("pivot_quadrant", size = .6, col = "black") + tm_text("subField", size = .6, col = "black") ``` \newpage @@ -362,7 +367,7 @@ tm_shape(CI, unit = "m")+ tm_compass(position = c("right", "top"), text.color = "black") + tm_shape(AllPivots)+ tm_borders( col = "black") + - tm_text("pivot_quadrant", size = .6, col = "black") + tm_text("subField", size = .6, col = "black") ``` @@ -371,7 +376,9 @@ tm_shape(CI, unit = "m")+ ```{r plots_ci_estate, echo=FALSE, fig.height=3.8, fig.width=10, message=FALSE, warning=FALSE, results='asis'} # # pivots <- AllPivots_merged %>% filter(pivot != c("1.1", "1.17")) -pivots_estate <- AllPivots_merged # %>% filter(pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14" , "1.16" , "1.17" , "1.18" , "6.1", "6.2", "DL1.1", "DL1.3")) %>% filter(pivot != "1.17") +pivots_estate <- AllPivots_merged %>% filter(pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14" , "1.16" , "1.17" , "1.18" , "6.1", "6.2", "DL1.1", "DL1.3")) %>% filter(pivot != "1.17") + + # pivots <- AllPivots_merged %>% filter(pivot != c("1.1", "1.17")) # pivots_estate <- AllPivots_merged %>% filter(pivot %in% c("1.1", "1.2", "1.7")) %>% filter(pivot != "1.17") @@ -383,6 +390,22 @@ walk(pivots_estate$pivot, ~ { ``` +```{r looping over sub_area, echo=FALSE, fig.height=3.8, fig.width=10, message=FALSE, warning=FALSE, results='asis'} +pivots_grouped <- AllPivots_merged %>% + group_by(sub_area) %>% + arrange(sub_area) # Optional: arrange the groups alphabetically by sub_area + +# Iterate over each subgroup +for (subgroup in unique(pivots_grouped$sub_area)) { + cat("\n## Subgroup:", subgroup, "\n") # Add a title for the subgroup + subset_data <- filter(pivots_grouped, sub_area == subgroup) + walk(subset_data$Field, ~ { + cat("\n") # Add an empty line for better spacing + ci_plot(.x) + cum_ci_plot(.x) + }) +} +``` ```{r eval=FALSE, fig.height=10, fig.width=14, include=FALSE} CI_all2 <- readRDS(here(cumulative_CI_vals_dir, "All_pivots_Cumulative_CI_whole_pivot_year.rds")) %>% diff --git a/r_app/parameters_project.R b/r_app/parameters_project.R new file mode 100644 index 0000000..394cd88 --- /dev/null +++ b/r_app/parameters_project.R @@ -0,0 +1,93 @@ + + +#chemba +if(project_dir == "chemba"){ + message("Yield data for Chemba") + + + field_boundaries_sf <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant)%>% + mutate(sub_area = case_when(pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", + "1.11", "1.12", "1.13", "1.14" , "1.16" , "1.17" , "1.18" , "6.1", "6.2", "DL1.1", "DL1.3") ~ "estate", + TRUE ~ "Cooperative")) + names(field_boundaries_sf) <- c("Field", "subField", "geometry", "sub_area") + + field_boundaries <- field_boundaries_sf %>% vect() + names(field_boundaries) <- c("Field", "subField", "sub_area") + + # field_boundaries_merged <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% group_by(pivot) %>% summarise() %>% vect() + + joined_spans <-st_read(here(data_dir_project, "span.geojson")) %>% st_transform(crs(field_boundaries_sf)) + names(joined_spans) <- c("Field", "area", "radius", "spans", "span", "geometry") + + + pivots_dates0 <- readRDS(here(harvest_dir, "harvest_data_new")) %>% filter( + pivot %in% c("1.1", "1.2", "1.3", "1.4", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", + "1.14" , "1.16" , "1.17" , "1.18" ,"2.1", "2.2", "2.3" , "2.4", "2.5", "3.1", "3.2", "3.3", + "4.1", "4.2", "4.3", "4.4", "4.5", "4.6", "5.1" ,"5.2", "5.3", "5.4", "6.1", "6.2", "DL1.1", "DL1.3") + ) + + harvesting_data <- pivots_dates0 %>% + dplyr::select(c("pivot_quadrant", "season_start_2021", "season_end_2021", "season_start_2022", + "season_end_2022", "season_start_2023", "season_end_2023", "season_start_2024", "season_end_2024", "Age")) %>% + pivot_longer(cols = starts_with("season"), names_to = "Year", values_to = "value") %>% + separate(Year, into = c("name", "Year"), sep = "(?<=season_start|season_end)\\_", remove = FALSE) %>% + mutate(name = str_to_title(name)) %>% + pivot_wider(names_from = name, values_from = value) %>% + rename(subField = pivot_quadrant) %>% + mutate(Field = substr(subField, 1, 3)) + + + + + + + + +} else if (project_dir == "xinavane"){ + library(readxl) + message("Yield data for Xinavane") + + field_boundaries <- st_read(here(data_dir, "Xinavane_demo.geojson")) %>% vect() + field_boundaries_sf <- st_read(here(data_dir, "Xinavane_demo.geojson")) + + joined_spans <- field_boundaries + + pivots_dates0 <- read_excel(here(harvest_dir, "Yield data.xlsx"), + skip = 3, + col_types = c("numeric", "text", "skip", "text", "numeric", "numeric", "numeric", "numeric", "date", + "numeric", "skip", "numeric")) %>% + rename( + Year = 1, + Field = 2, + sub_area = 3, + hectares = 4, + tons = 5, + tcha = 6, + tchy = 7, + Season_end = 8, + age = 9, + ratoon = 10 + ) %>% + mutate(Season_end = ymd(Season_end), + Season_start = as.Date(Season_end - (duration(months = age))), + subField = Field) #don't forget to add 2022 as a year for the 'curent' season + + pivots_dates0 <- pivots_dates0 %>% + mutate(Year = Year + 6) + + # Add 6 years to Season_end column + pivots_dates0 <- pivots_dates0 %>% + mutate(Season_end = Season_end + years(6)) + + # Add 6 years to Season_start column + pivots_dates0 <- pivots_dates0 %>% + mutate(Season_start = Season_start + years(6)) + + + harvesting_data <- pivots_dates0 %>% select(c("Field","subField", "Year", "Season_start","Season_end", "Age" )) + + +} else { + message("No yield data supplied") +} +