diff --git a/build_report.sh b/build_report.sh index 125b122..a63e381 100755 --- a/build_report.sh +++ b/build_report.sh @@ -48,4 +48,4 @@ cd /Users/mfolkerts/smartCane/r_app #Rscript -e "rmarkdown::render('/Users/mfolkerts/smartCane/r_app/CI_Report_dashboard_planet.Rmd', 'all')" #Rscript 1_harvest_data_EcoFarm_v2.R #Rscript 2_CI_data_prep.R $weeks_ago -Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))" \ No newline at end of file +Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))" diff --git a/laravel_app/app/Jobs/ProjectDownloadTiffJob.php b/laravel_app/app/Jobs/ProjectDownloadTiffJob.php index 752c25e..b587002 100644 --- a/laravel_app/app/Jobs/ProjectDownloadTiffJob.php +++ b/laravel_app/app/Jobs/ProjectDownloadTiffJob.php @@ -4,6 +4,7 @@ use App\Models\Project; use App\Models\ProjectDownload; +use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; @@ -16,7 +17,7 @@ class ProjectDownloadTiffJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels; protected Carbon $date; protected ProjectDownload $download; diff --git a/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php b/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php index 1192dec..e84eb53 100644 --- a/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php +++ b/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php @@ -6,6 +6,7 @@ use App\Models\ProjectDownload; use App\Models\ProjectMosaic; use App\Models\ProjectReport; +use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; @@ -16,7 +17,7 @@ class ProjectMosiacGeneratorJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public ProjectMosaic $mosaic; diff --git a/laravel_app/app/Jobs/ProjectReportGeneratorJob.php b/laravel_app/app/Jobs/ProjectReportGeneratorJob.php index 87bc48b..f4ac9fa 100644 --- a/laravel_app/app/Jobs/ProjectReportGeneratorJob.php +++ b/laravel_app/app/Jobs/ProjectReportGeneratorJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\ProjectReport; +use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; @@ -13,7 +14,7 @@ class ProjectReportGeneratorJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $timeout = 220; private ProjectReport $projectReport; diff --git a/laravel_app/app/Models/Project.php b/laravel_app/app/Models/Project.php index 62337cc..f7aafbf 100644 --- a/laravel_app/app/Models/Project.php +++ b/laravel_app/app/Models/Project.php @@ -4,11 +4,13 @@ use App\Jobs\ProjectDownloadTiffJob; use App\Jobs\ProjectMosiacGeneratorJob; +use App\Jobs\ProjectReportGeneratorJob; use Carbon\CarbonPeriod; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Carbon\Carbon; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; @@ -266,18 +268,19 @@ public function startDownload(Carbon $date) public function scheduleReport($year, $week) { - if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) { - return; - } +// if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) { +// return; +// } Bus::chain([ Bus::batch(self::getFileDownloadsFor($year, $week)), - Bus::batch(self::getMosiacsFor($year, $week)), - Bus::batch(self::getReport($year, $week)), + Bus::batch(self::getMosaicsFor($year, $week)), + Bus::batch(self::getReportFor($year, $week)), ])->dispatch(); + return "done"; } - public function getReport($year, $week) + public function getReportFor($year, $week) { $report = $this->reports()->create([ @@ -290,21 +293,24 @@ public function getReport($year, $week) return new ProjectReportGeneratorJob($report); } - public function getFileDownloadsFor($year, $startWeekNumber): Collection + public function getFileDownloadsFor($year, $startWeekNumber) { $endOfRange = \Illuminate\Support\Carbon::now()->setISODate($year, $startWeekNumber)->endOfWeek(); - $startOfRange = (clone $endOfRange)->subWeeks(2)->startOfWeek(); + $startOfRange = (clone $endOfRange)->subWeeks(3)->startOfWeek(); $dateRange = CarbonPeriod::create($startOfRange, $endOfRange); return collect($dateRange) ->map(fn($date) => ProjectDownloadTiffJob::handleForDate($this, $date)) - ->filter(); + ->filter() + ->toArray(); } public function getMosaicsFor($year, $startWeekNumber) { - return collect(range(0, 2)) - ->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff)); + return collect(range(0, 3)) + ->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff)) + ->filter() + ->toArray(); } } diff --git a/laravel_app/database/migrations/2024_01_10_194728_create_job_batches_table.php b/laravel_app/database/migrations/2024_01_10_194728_create_job_batches_table.php new file mode 100644 index 0000000..50e38c2 --- /dev/null +++ b/laravel_app/database/migrations/2024_01_10_194728_create_job_batches_table.php @@ -0,0 +1,35 @@ +string('id')->primary(); + $table->string('name'); + $table->integer('total_jobs'); + $table->integer('pending_jobs'); + $table->integer('failed_jobs'); + $table->longText('failed_job_ids'); + $table->mediumText('options')->nullable(); + $table->integer('cancelled_at')->nullable(); + $table->integer('created_at'); + $table->integer('finished_at')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('job_batches'); + } +}; diff --git a/laravel_app/database/seeders/DatabaseSeeder.php b/laravel_app/database/seeders/DatabaseSeeder.php index ab4b640..cb55d91 100644 --- a/laravel_app/database/seeders/DatabaseSeeder.php +++ b/laravel_app/database/seeders/DatabaseSeeder.php @@ -270,13 +270,11 @@ private function createKakiraProject() [ 'subject' => 'Kakira 2021-01-01', 'message' => 'Kakira 2021-01-01', - 'status' => 'completed', 'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00', ], [ 'subject' => 'Kakira 2021-01-08', 'message' => 'Kakira 2021-01-08', - 'status' => 'completed', 'created_at' => '2021-01-08 00:00:00', 'updated_at' => '2021-01-08 00:00:00', ], @@ -287,13 +285,11 @@ private function createKakiraProject() [ 'name' => 'Martin Folkerts', 'email' => 'martin@sobit.nl', - 'status' => 'completed', 'created_at' => $mailing->created_at, 'updated_at' => $mailing->updated_at, ], [ 'name' => 'Timon Weitkamp', 'email' => 'timon@smartfarmingtech.com', - 'status' => 'completed', 'created_at' => $mailing->created_at, 'updated_at' => $mailing->updated_at, ], diff --git a/laravel_app/tests/Unit/Models/ProjectTest.php b/laravel_app/tests/Unit/Models/ProjectTest.php index 174775c..c8cb531 100644 --- a/laravel_app/tests/Unit/Models/ProjectTest.php +++ b/laravel_app/tests/Unit/Models/ProjectTest.php @@ -4,10 +4,13 @@ use App\Jobs\ProjectDownloadTiffJob; use App\Jobs\ProjectMosiacGeneratorJob; +use App\Jobs\ProjectReportGeneratorJob; use App\Models\Project; +use Illuminate\Bus\PendingBatch; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\Bus; use Mockery; use Tests\TestCase; @@ -174,6 +177,19 @@ public function when_getMosaicsFor_is_called_it_returns_a_collection_of_seven_do $mosaics->each(fn($job) => $this->assertInstanceOf(ProjectMosiacGeneratorJob::class, $job)); } + /** @test */ + public function when_getReport_is_called_it_returns_a_jobs() + { + $project = Project::create([ + 'name' => 'project_name', + 'download_path' => 'project_download_path', + ]); + + $job = $project->getReportFor(2023, 2); + + $this->assertInstanceOf(ProjectReportGeneratorJob::class, $job); + } + /** @test */ public function it_can_create_a_chain_of_batches_that_result_in_a_report() { @@ -182,10 +198,18 @@ public function it_can_create_a_chain_of_batches_that_result_in_a_report() 'download_path' => 'project_download_path', ]); - Bus::fake(); $project->scheduleReport(2023, 50); - - + Bus::assertChained([ + Bus::chainedBatch(function (PendingBatch $batch) { + return $batch->jobs->count() === 28; + }), + Bus::chainedBatch(function (PendingBatch $batch) { + return $batch->jobs->count() === 4; + }), + Bus::chainedBatch(function (PendingBatch $batch) { + return $batch->jobs->count() === 1; + }), + ]); } } diff --git a/r_app/CI_report_dashboard_planet.Rmd b/r_app/CI_report_dashboard_planet.Rmd index bd781c4..fe9f597 100644 --- a/r_app/CI_report_dashboard_planet.Rmd +++ b/r_app/CI_report_dashboard_planet.Rmd @@ -43,14 +43,15 @@ library(CAST) ``` ```{r directories, message=FALSE, warning=FALSE, include=FALSE} -laravel_storage_dir <- here("../laravel_app/storage/app") -data_dir <- here(laravel_storage_dir, "Data") +laravel_storage_dir <- here("../laravel_app/storage/app/chemba") +data_dir <- here(laravel_storage_dir, "../Data") extracted_CI_dir <- here(data_dir, "extracted_ci") daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") harvest_dir <- here(data_dir, "HarvestData") -weekly_CI_mosaic <- here(data_dir, "weekly_mosaic") +weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") + s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/chemba_S2/" ``` @@ -61,7 +62,7 @@ s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/400 #today = "2023-12-12" today <- as.character(report_date) week <- week(today) - #today = "2022-08-18" + #today = as.character(Sys.Date()) #week = lubridate::week(Sys.time()) @@ -82,25 +83,30 @@ This PDF-dashboard shows the status of your fields on a weekly basis. We will sh -```{r data, message=FALSE, warning=FALSE, include=FALSE} +```{r data, message=TRUE, warning=TRUE, include=FALSE} # get latest CI index -week_minus_1 <- as.numeric(week) -1 -week_minus_2 <- as.numeric(week) -2 -week_minus_3 <- as.numeric(week) -3 - today_minus_1 <- as.character(ymd(today) - 7) today_minus_2 <- as.character(ymd(today) - 14) today_minus_3 <- as.character(ymd(today) - 21) +week_minus_1 <- week(today_minus_1) +week_minus_2 <- week(today_minus_2) +week_minus_3 <- week(today_minus_3) + +year = year(today) +year_2 = year(today_minus_1) +year_3 = year(today_minus_2) +year_4 = year(today_minus_3) + # remove_pivots <- c("1.1", "1.12", "1.8", "1.9", "1.11", "1.14") CI_quadrant <- readRDS(here(cumulative_CI_vals_dir,"All_pivots_Cumulative_CI_quadrant_year_v2.rds"))# %>% # rename(pivot_quadrant = Field) message("STOP - check ci name in layer") -CI <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_2023.tif"))) %>% subset("CI") -CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_2023.tif"))) %>% subset("CI") #%>% subset("CI") -CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_2023.tif"))) %>% subset("CI") #%>% subset("CI") -CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_2023.tif"))) %>% subset("CI") #%>% subset("CI") +CI <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_", year, ".tif"))) %>% subset("CI") +CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_", year_2, ".tif"))) %>% subset("CI") +CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_", year_3, ".tif"))) %>% subset("CI") +CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_4, ".tif"))) %>% subset("CI") last_week_dif_raster <- ((CI - CI_m1) / CI_m1) * 100 last_week_dif_raster_abs <- (CI - CI_m1) @@ -214,7 +220,7 @@ ci_plot <- function(pivotName){ CImap_m2 <- create_CI_map(singlePivot_m2, AllPivots2, joined_spans2, show_legend= T, legend_is_portrait = T, week = week_minus_2, age = age -2) CImap_m1 <- create_CI_map(singlePivot_m1, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age -1) - CImap <- create_CI_map(singlePivot_m1, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age ) + CImap <- create_CI_map(singlePivot, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age ) CI_max_abs_last_week <- create_CI_diff_map(abs_CI_last_week,AllPivots2, joined_spans2, show_legend = T, legend_is_portrait = T, week_1 = week, week_2 = week_minus_1, age = age) diff --git a/r_app/Rplots.pdf b/r_app/Rplots.pdf index 7b37390..2a4ceb7 100644 Binary files a/r_app/Rplots.pdf and b/r_app/Rplots.pdf differ