wip

build_report.sh

@@ -48,4 +48,4 @@ cd /Users/mfolkerts/smartCane/r_app
 #Rscript -e "rmarkdown::render('/Users/mfolkerts/smartCane/r_app/CI_Report_dashboard_planet.Rmd', 'all')"
 #Rscript 1_harvest_data_EcoFarm_v2.R
 #Rscript 2_CI_data_prep.R $weeks_ago
-Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))"
+Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))"

laravel_app/app/Jobs/ProjectDownloadTiffJob.php

@@ -4,6 +4,7 @@
 
 use App\Models\Project;
 use App\Models\ProjectDownload;
+use Illuminate\Bus\Batchable;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -16,7 +17,7 @@
 
 class ProjectDownloadTiffJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     protected Carbon $date;
     protected ProjectDownload $download;

laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php

@@ -6,6 +6,7 @@
 use App\Models\ProjectDownload;
 use App\Models\ProjectMosaic;
 use App\Models\ProjectReport;
+use Illuminate\Bus\Batchable;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -16,7 +17,7 @@
 
 class ProjectMosiacGeneratorJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public  ProjectMosaic $mosaic;
 

laravel_app/app/Jobs/ProjectReportGeneratorJob.php

@@ -3,6 +3,7 @@
 namespace App\Jobs;
 
 use App\Models\ProjectReport;
+use Illuminate\Bus\Batchable;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -13,7 +14,7 @@
 
 class ProjectReportGeneratorJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public $timeout = 220;
     private ProjectReport $projectReport;

laravel_app/app/Models/Project.php

@@ -4,11 +4,13 @@
 
 use App\Jobs\ProjectDownloadTiffJob;
 use App\Jobs\ProjectMosiacGeneratorJob;
+use App\Jobs\ProjectReportGeneratorJob;
 use Carbon\CarbonPeriod;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
 use Carbon\Carbon;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
 
@@ -266,18 +268,19 @@ public function startDownload(Carbon $date)
 
     public function scheduleReport($year, $week)
     {
-        if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) {
+//        if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) {
-            return;
+//            return;
-        }
+//        }
 
         Bus::chain([
             Bus::batch(self::getFileDownloadsFor($year, $week)),
-            Bus::batch(self::getMosiacsFor($year, $week)),
+            Bus::batch(self::getMosaicsFor($year, $week)),
-            Bus::batch(self::getReport($year, $week)),
+            Bus::batch(self::getReportFor($year, $week)),
         ])->dispatch();
+        return  "done";
     }
 
-    public function getReport($year, $week)
+    public function getReportFor($year, $week)
     {
 
         $report = $this->reports()->create([
@@ -290,21 +293,24 @@ public function getReport($year, $week)
         return new ProjectReportGeneratorJob($report);
     }
 
-    public function getFileDownloadsFor($year, $startWeekNumber): Collection
+    public function getFileDownloadsFor($year, $startWeekNumber)
     {
         $endOfRange = \Illuminate\Support\Carbon::now()->setISODate($year, $startWeekNumber)->endOfWeek();
-        $startOfRange = (clone $endOfRange)->subWeeks(2)->startOfWeek();
+        $startOfRange = (clone $endOfRange)->subWeeks(3)->startOfWeek();
 
         $dateRange = CarbonPeriod::create($startOfRange, $endOfRange);
 
         return collect($dateRange)
             ->map(fn($date) => ProjectDownloadTiffJob::handleForDate($this, $date))
-            ->filter();
+            ->filter()
+            ->toArray();
     }
 
     public function getMosaicsFor($year, $startWeekNumber)
     {
-        return collect(range(0, 2))
+        return collect(range(0, 3))
-            ->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff));
+            ->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff))
+            ->filter()
+            ->toArray();
     }
 }

laravel_app/database/migrations/2024_01_10_194728_create_job_batches_table.php

@@ -0,0 +1,35 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('job_batches', function (Blueprint $table) {
+            $table->string('id')->primary();
+            $table->string('name');
+            $table->integer('total_jobs');
+            $table->integer('pending_jobs');
+            $table->integer('failed_jobs');
+            $table->longText('failed_job_ids');
+            $table->mediumText('options')->nullable();
+            $table->integer('cancelled_at')->nullable();
+            $table->integer('created_at');
+            $table->integer('finished_at')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('job_batches');
+    }
+};

laravel_app/database/seeders/DatabaseSeeder.php

@@ -270,13 +270,11 @@ private function createKakiraProject()
             [
                 'subject'    => 'Kakira 2021-01-01',
                 'message'    => 'Kakira 2021-01-01',
-                'status'     => 'completed',
                 'created_at' => '2021-01-01 00:00:00',
                 'updated_at' => '2021-01-01 00:00:00',
             ], [
                 'subject'    => 'Kakira 2021-01-08',
                 'message'    => 'Kakira 2021-01-08',
-                'status'     => 'completed',
                 'created_at' => '2021-01-08 00:00:00',
                 'updated_at' => '2021-01-08 00:00:00',
             ],
@@ -287,13 +285,11 @@ private function createKakiraProject()
                 [
                     'name'       => 'Martin Folkerts',
                     'email'      => 'martin@sobit.nl',
-                    'status'     => 'completed',
                     'created_at' => $mailing->created_at,
                     'updated_at' => $mailing->updated_at,
                 ], [
                     'name'       => 'Timon Weitkamp',
                     'email'      => 'timon@smartfarmingtech.com',
-                    'status'     => 'completed',
                     'created_at' => $mailing->created_at,
                     'updated_at' => $mailing->updated_at,
                 ],

laravel_app/tests/Unit/Models/ProjectTest.php

@@ -4,10 +4,13 @@
 
 use App\Jobs\ProjectDownloadTiffJob;
 use App\Jobs\ProjectMosiacGeneratorJob;
+use App\Jobs\ProjectReportGeneratorJob;
 use App\Models\Project;
 
+use Illuminate\Bus\PendingBatch;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Bus;
 use Mockery;
 use Tests\TestCase;
 
@@ -174,6 +177,19 @@ public function when_getMosaicsFor_is_called_it_returns_a_collection_of_seven_do
         $mosaics->each(fn($job) => $this->assertInstanceOf(ProjectMosiacGeneratorJob::class, $job));
     }
 
+    /** @test */
+    public function when_getReport_is_called_it_returns_a_jobs()
+    {
+        $project = Project::create([
+            'name'          => 'project_name',
+            'download_path' => 'project_download_path',
+        ]);
+
+        $job = $project->getReportFor(2023, 2);
+
+        $this->assertInstanceOf(ProjectReportGeneratorJob::class, $job);
+    }
+
     /** @test */
     public function it_can_create_a_chain_of_batches_that_result_in_a_report()
     {
@@ -182,10 +198,18 @@ public function it_can_create_a_chain_of_batches_that_result_in_a_report()
             'download_path' => 'project_download_path',
         ]);
 
-
         Bus::fake();
         $project->scheduleReport(2023, 50);
-
+        Bus::assertChained([
-
+            Bus::chainedBatch(function (PendingBatch $batch) {
+                return $batch->jobs->count() === 28;
+            }),
+            Bus::chainedBatch(function (PendingBatch $batch) {
+                return $batch->jobs->count() === 4;
+            }),
+            Bus::chainedBatch(function (PendingBatch $batch) {
+                return $batch->jobs->count() === 1;
+            }),
+        ]);
     }
 }

r_app/CI_report_dashboard_planet.Rmd

@@ -43,14 +43,15 @@ library(CAST)
 ```
 
 ```{r directories, message=FALSE, warning=FALSE, include=FALSE}
-laravel_storage_dir <- here("../laravel_app/storage/app")
+laravel_storage_dir <- here("../laravel_app/storage/app/chemba")
-data_dir <- here(laravel_storage_dir, "Data")
+data_dir <- here(laravel_storage_dir, "../Data")
 extracted_CI_dir <- here(data_dir, "extracted_ci")
 daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
 cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
 harvest_dir <-  here(data_dir, "HarvestData")
 
-weekly_CI_mosaic <- here(data_dir, "weekly_mosaic")
+weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
+
 
 s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/chemba_S2/"
 ```
@@ -61,7 +62,7 @@ s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/400
 #today = "2023-12-12"
 today <- as.character(report_date)
 week <- week(today)
- #today = "2022-08-18"
+
 
 #today = as.character(Sys.Date())
 #week = lubridate::week(Sys.time())
@@ -82,25 +83,30 @@ This PDF-dashboard shows the status of your fields on a weekly basis. We will sh
 
 
 
-```{r data, message=FALSE, warning=FALSE, include=FALSE}
+```{r data, message=TRUE, warning=TRUE, include=FALSE}
 # get latest CI index
-week_minus_1 <- as.numeric(week) -1
-week_minus_2 <- as.numeric(week) -2
-week_minus_3 <- as.numeric(week) -3
-
 today_minus_1 <- as.character(ymd(today) - 7)
 today_minus_2 <- as.character(ymd(today) - 14)
 today_minus_3 <- as.character(ymd(today) - 21)
 
+week_minus_1 <- week(today_minus_1)
+week_minus_2 <- week(today_minus_2)
+week_minus_3 <- week(today_minus_3)
+
+year = year(today)
+year_2 = year(today_minus_1)
+year_3 = year(today_minus_2)
+year_4 = year(today_minus_3)
+
 # remove_pivots <- c("1.1", "1.12", "1.8", "1.9", "1.11", "1.14")
 CI_quadrant <- readRDS(here(cumulative_CI_vals_dir,"All_pivots_Cumulative_CI_quadrant_year_v2.rds"))# %>%
  # rename(pivot_quadrant = Field)
 
 message("STOP - check ci name in layer")
-CI    <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_2023.tif"))) %>% subset("CI")
+CI    <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_", year, ".tif"))) %>% subset("CI")
-CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_2023.tif"))) %>% subset("CI")   #%>% subset("CI")
+CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_", year_2, ".tif"))) %>% subset("CI")
-CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_2023.tif"))) %>% subset("CI")   #%>% subset("CI")
+CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_", year_3, ".tif"))) %>% subset("CI")
-CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_2023.tif"))) %>% subset("CI")   #%>% subset("CI")
+CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_4, ".tif"))) %>% subset("CI")
 
 last_week_dif_raster <- ((CI - CI_m1) / CI_m1) * 100
 last_week_dif_raster_abs <- (CI - CI_m1)
@@ -214,7 +220,7 @@ ci_plot <- function(pivotName){
 
   CImap_m2 <- create_CI_map(singlePivot_m2, AllPivots2,  joined_spans2, show_legend= T, legend_is_portrait = T, week = week_minus_2, age = age  -2)
   CImap_m1 <- create_CI_map(singlePivot_m1, AllPivots2,  joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age  -1)
-  CImap <- create_CI_map(singlePivot_m1, AllPivots2,  joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age )
+  CImap <- create_CI_map(singlePivot, AllPivots2,  joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age )
 
 
   CI_max_abs_last_week <- create_CI_diff_map(abs_CI_last_week,AllPivots2,  joined_spans2,  show_legend = T, legend_is_portrait = T, week_1 = week, week_2 = week_minus_1, age = age)