Timon/SmartCane
1
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

wip

Browse source
This commit is contained in:
Martin Folkerts 2024-01-10 21:42:32 +01:00
parent a1bba242cb
commit 3eaf471f05
10 changed files with 106 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build_report.sh
View file

@ -48,4 +48,4 @@ cd /Users/mfolkerts/smartCane/r_app
#Rscript -e "rmarkdown::render('/Users/mfolkerts/smartCane/r_app/CI_Report_dashboard_planet.Rmd', 'all')"
#Rscript 1_harvest_data_EcoFarm_v2.R
#Rscript 2_CI_data_prep.R $weeks_ago
Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))"
Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))"

3
laravel_app/app/Jobs/ProjectDownloadTiffJob.php
View file

@ -4,6 +4,7 @@
use App\Models\Project;
use App\Models\ProjectDownload;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -16,7 +17,7 @@
class ProjectDownloadTiffJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected Carbon $date;
protected ProjectDownload $download;

3
laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php
View file

@ -6,6 +6,7 @@
use App\Models\ProjectDownload;
use App\Models\ProjectMosaic;
use App\Models\ProjectReport;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -16,7 +17,7 @@
class ProjectMosiacGeneratorJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public ProjectMosaic $mosaic;

3
laravel_app/app/Jobs/ProjectReportGeneratorJob.php
View file

@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\ProjectReport;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -13,7 +14,7 @@
class ProjectReportGeneratorJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 220;
private ProjectReport $projectReport;

28
laravel_app/app/Models/Project.php
View file

@ -4,11 +4,13 @@
use App\Jobs\ProjectDownloadTiffJob;
use App\Jobs\ProjectMosiacGeneratorJob;
use App\Jobs\ProjectReportGeneratorJob;
use Carbon\CarbonPeriod;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Carbon\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
@ -266,18 +268,19 @@ public function startDownload(Carbon $date)
public function scheduleReport($year, $week)
{
if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) {
return;
}
// if ($this->reports()->where(['year' => $year, 'week' => $week])->count() > 0) {
// return;
// }
Bus::chain([
Bus::batch(self::getFileDownloadsFor($year, $week)),
Bus::batch(self::getMosiacsFor($year, $week)),
Bus::batch(self::getReport($year, $week)),
Bus::batch(self::getMosaicsFor($year, $week)),
Bus::batch(self::getReportFor($year, $week)),
])->dispatch();
return "done";
}
public function getReport($year, $week)
public function getReportFor($year, $week)
{
$report = $this->reports()->create([
@ -290,21 +293,24 @@ public function getReport($year, $week)
return new ProjectReportGeneratorJob($report);
}
public function getFileDownloadsFor($year, $startWeekNumber): Collection
public function getFileDownloadsFor($year, $startWeekNumber)
{
$endOfRange = \Illuminate\Support\Carbon::now()->setISODate($year, $startWeekNumber)->endOfWeek();
$startOfRange = (clone $endOfRange)->subWeeks(2)->startOfWeek();
$startOfRange = (clone $endOfRange)->subWeeks(3)->startOfWeek();
$dateRange = CarbonPeriod::create($startOfRange, $endOfRange);
return collect($dateRange)
->map(fn($date) => ProjectDownloadTiffJob::handleForDate($this, $date))
->filter();
->filter()
->toArray();
}
public function getMosaicsFor($year, $startWeekNumber)
{
return collect(range(0, 2))
->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff));
return collect(range(0, 3))
->map(fn($weekDiff) => ProjectMosiacGeneratorJob::handleFor($this, $year, $startWeekNumber - $weekDiff))
->filter()
->toArray();
}
}

35
laravel_app/database/migrations/2024_01_10_194728_create_job_batches_table.php Normal file
View file

@ -0,0 +1,35 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('job_batches', function (Blueprint $table) {
$table->string('id')->primary();
$table->string('name');
$table->integer('total_jobs');
$table->integer('pending_jobs');
$table->integer('failed_jobs');
$table->longText('failed_job_ids');
$table->mediumText('options')->nullable();
$table->integer('cancelled_at')->nullable();
$table->integer('created_at');
$table->integer('finished_at')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('job_batches');
}
};

4
laravel_app/database/seeders/DatabaseSeeder.php
View file

@ -270,13 +270,11 @@ private function createKakiraProject()
[
'subject' => 'Kakira 2021-01-01',
'message' => 'Kakira 2021-01-01',
'status' => 'completed',
'created_at' => '2021-01-01 00:00:00',
'updated_at' => '2021-01-01 00:00:00',
], [
'subject' => 'Kakira 2021-01-08',
'message' => 'Kakira 2021-01-08',
'status' => 'completed',
'created_at' => '2021-01-08 00:00:00',
'updated_at' => '2021-01-08 00:00:00',
],
@ -287,13 +285,11 @@ private function createKakiraProject()
[
'name' => 'Martin Folkerts',
'email' => 'martin@sobit.nl',
'status' => 'completed',
'created_at' => $mailing->created_at,
'updated_at' => $mailing->updated_at,
], [
'name' => 'Timon Weitkamp',
'email' => 'timon@smartfarmingtech.com',
'status' => 'completed',
'created_at' => $mailing->created_at,
'updated_at' => $mailing->updated_at,
],

30
laravel_app/tests/Unit/Models/ProjectTest.php
View file

@ -4,10 +4,13 @@
use App\Jobs\ProjectDownloadTiffJob;
use App\Jobs\ProjectMosiacGeneratorJob;
use App\Jobs\ProjectReportGeneratorJob;
use App\Models\Project;
use Illuminate\Bus\PendingBatch;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Bus;
use Mockery;
use Tests\TestCase;
@ -174,6 +177,19 @@ public function when_getMosaicsFor_is_called_it_returns_a_collection_of_seven_do
$mosaics->each(fn($job) => $this->assertInstanceOf(ProjectMosiacGeneratorJob::class, $job));
}
/** @test */
public function when_getReport_is_called_it_returns_a_jobs()
{
$project = Project::create([
'name' => 'project_name',
'download_path' => 'project_download_path',
]);
$job = $project->getReportFor(2023, 2);
$this->assertInstanceOf(ProjectReportGeneratorJob::class, $job);
}
/** @test */
public function it_can_create_a_chain_of_batches_that_result_in_a_report()
{
@ -182,10 +198,18 @@ public function it_can_create_a_chain_of_batches_that_result_in_a_report()
'download_path' => 'project_download_path',
]);
Bus::fake();
$project->scheduleReport(2023, 50);
Bus::assertChained([
Bus::chainedBatch(function (PendingBatch $batch) {
return $batch->jobs->count() === 28;
}),
Bus::chainedBatch(function (PendingBatch $batch) {
return $batch->jobs->count() === 4;
}),
Bus::chainedBatch(function (PendingBatch $batch) {
return $batch->jobs->count() === 1;
}),
]);
}
}

34
r_app/CI_report_dashboard_planet.Rmd
View file

@ -43,14 +43,15 @@ library(CAST)
```
```{r directories, message=FALSE, warning=FALSE, include=FALSE}
laravel_storage_dir <- here("../laravel_app/storage/app")
data_dir <- here(laravel_storage_dir, "Data")
laravel_storage_dir <- here("../laravel_app/storage/app/chemba")
data_dir <- here(laravel_storage_dir, "../Data")
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
harvest_dir <- here(data_dir, "HarvestData")
weekly_CI_mosaic <- here(data_dir, "weekly_mosaic")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/4002 TechnicalData/04 WP2 technical/python/chemba_S2/"
```
@ -61,7 +62,7 @@ s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/400
#today = "2023-12-12"
today <- as.character(report_date)
week <- week(today)
#today = "2022-08-18"
#today = as.character(Sys.Date())
#week = lubridate::week(Sys.time())
@ -82,25 +83,30 @@ This PDF-dashboard shows the status of your fields on a weekly basis. We will sh
```{r data, message=FALSE, warning=FALSE, include=FALSE}
```{r data, message=TRUE, warning=TRUE, include=FALSE}
# get latest CI index
week_minus_1 <- as.numeric(week) -1
week_minus_2 <- as.numeric(week) -2
week_minus_3 <- as.numeric(week) -3
today_minus_1 <- as.character(ymd(today) - 7)
today_minus_2 <- as.character(ymd(today) - 14)
today_minus_3 <- as.character(ymd(today) - 21)
week_minus_1 <- week(today_minus_1)
week_minus_2 <- week(today_minus_2)
week_minus_3 <- week(today_minus_3)
year = year(today)
year_2 = year(today_minus_1)
year_3 = year(today_minus_2)
year_4 = year(today_minus_3)
# remove_pivots <- c("1.1", "1.12", "1.8", "1.9", "1.11", "1.14")
CI_quadrant <- readRDS(here(cumulative_CI_vals_dir,"All_pivots_Cumulative_CI_quadrant_year_v2.rds"))# %>%
# rename(pivot_quadrant = Field)
message("STOP - check ci name in layer")
CI <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_2023.tif"))) %>% subset("CI")
CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_2023.tif"))) %>% subset("CI") #%>% subset("CI")
CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_2023.tif"))) %>% subset("CI") #%>% subset("CI")
CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_2023.tif"))) %>% subset("CI") #%>% subset("CI")
CI <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_", year, ".tif"))) %>% subset("CI")
CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_", year_2, ".tif"))) %>% subset("CI")
CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_", year_3, ".tif"))) %>% subset("CI")
CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_4, ".tif"))) %>% subset("CI")
last_week_dif_raster <- ((CI - CI_m1) / CI_m1) * 100
last_week_dif_raster_abs <- (CI - CI_m1)
@ -214,7 +220,7 @@ ci_plot <- function(pivotName){
CImap_m2 <- create_CI_map(singlePivot_m2, AllPivots2, joined_spans2, show_legend= T, legend_is_portrait = T, week = week_minus_2, age = age -2)
CImap_m1 <- create_CI_map(singlePivot_m1, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age -1)
CImap <- create_CI_map(singlePivot_m1, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age )
CImap <- create_CI_map(singlePivot, AllPivots2, joined_spans2, show_legend= F, legend_is_portrait = F, week = week_minus_1, age = age )
CI_max_abs_last_week <- create_CI_diff_map(abs_CI_last_week,AllPivots2, joined_spans2, show_legend = T, legend_is_portrait = T, week_1 = week, week_2 = week_minus_1, age = age)

BIN
r_app/Rplots.pdf
View file

Binary file not shown.