Timon/SmartCane
1
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

wip

Browse source
This commit is contained in:
Martin Folkerts 2024-07-04 21:16:12 +02:00
parent 8f2e6e7a6c
commit 1de6dd4edf
14 changed files with 364 additions and 353 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
build_mosaic.sh
View file

@ -3,6 +3,7 @@
end_date="2024-06-08"
offset=7
data_dir="chemba"
file_name_tif="week_03_2024.tif"
# Parse command line arguments
for arg in "$@"; do
@ -16,6 +17,9 @@ for arg in "$@"; do
--data_dir=*)
data_dir="${arg#*=}"
;;
--file_name_tif=*)
file_name_tif="${arg#*=}"
;;
*)
echo "Unknown option: $arg"
exit 1
@ -28,12 +32,12 @@ echo "offset: $offset"
echo "end_date: $end_date"
# Check if required arguments are set
if [ -z "$end_date" ] || [ -z "$data_dir" ] || [ -z "$offset" ]; then
echo "Missing arguments. Use: build_mosiac.sh --endate=2024-01-01 --offset=7 --data_dir=chemba"
if [ -z "$end_date" ] || [ -z "$data_dir" ] || [ -z "$offset" ] || [ -z "$file_name_tif" ]; then
echo "Missing arguments. Use: build_mosiac.sh --endate=2024-01-01 --offset=7 --data_dir=chemba --file_name_tif=week_03_2024.tif"
exit 1
fi
echo mosaic_creation.R $end_date $offset $data_dir
echo mosaic_creation.R $end_date $offset $data_dir $file_name_tif
cd ../r_app
Rscript mosaic_creation.R $end_date $offset $data_dir
Rscript mosaic_creation.R $end_date $offset $data_dir $file_name_tif

4
laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php
View file

@ -51,6 +51,7 @@ public function handle(): void
sprintf('--end_date=%s', $this->mosaic->end_date->format('Y-m-d')),
sprintf('--offset=%s', $this->mosaic->offset),
sprintf('--data_dir=%s', $this->mosaic->project->download_path),
sprintf('--file_name_tif=%s', basename($this->mosaic->path)),
];
$currentPath = '/usr/bin:/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin/Users/mfolkerts/anaconda3/bin:/Library/Apple/usr/bin';
@ -89,8 +90,7 @@ public static function handleFor(Project $project,Carbon $endDate, int $offset):
*/
$mosaic = $project->mosaics()->updateOrCreate(
[
'end_date' => $endDate,
'offset' => $offset,
'name' => sprintf('Week_%s_%s', $week, $year),
],
[
'name' => sprintf('Week_%s_%s', $week, $year),

2
laravel_app/app/Mail/ReportMailer.php
View file

@ -49,7 +49,7 @@ public function content(): Content
markdown: 'emails.scheduled-report',
with: [
'mailingContent' => $this->mailing->message,
'logoPath'=> Storage::disk('local')->path('images/smartcane.png'),
'logoPath'=> resource_path('images/smartcane.png'),
'subject' => $this->mailing->subject,
'mailing' => $this->mailing,
],

43
laravel_app/app/Models/Project.php
View file

@ -57,12 +57,14 @@ public static function saveWithFormData(mixed $formData)
$project->update(['span_json_path' => $baseTo.'span.geojson']);
}
if ($formData['harvest_file']) {
Storage::copy($baseFrom.$formData['harvest_file']['tmpFilename'],$baseTo.'harvest.'.$formData['harvest_file']['extension']);
Storage::copy($baseFrom.$formData['harvest_file']['tmpFilename'],
$baseTo.'harvest.'.$formData['harvest_file']['extension']);
if ($project->update(['harvest_json_path' => $baseTo.'harvest.'.$formData['harvest_file']['extension']])) {
$project->setMinHarvestDate();
}
}
}
private function upsertMailRecipients($formData)
{
$mailRecipientsData = array_map(function ($mailRecipient) {
@ -237,6 +239,7 @@ public function hasPendingDownload(): bool
{
return $this->downloads()->statusPending()->count() > 0;
}
public function hasPendingReport(): bool
{
return $this->reports()->statusPending()->count() > 0;
@ -342,13 +345,40 @@ public function getFileDownloadsFor(Carbon $endDate, int $offset=7):array
public function getMosaicsFor(Carbon $endDate, int $offset = 7): array
{
logger(sprintf('in Get Mosaics for Period %s with %d offset', $endDate->format('Y-m-d'),$offset));
return collect(range(0, 3))
->map(function ($periodDiff) use ($endDate, $offset) {
$periodEndDate = $endDate->copy()->subDays($offset*$periodDiff);
// $periodStartDate = $periodEndDate->copy()->subDays($offset-1);
return ProjectMosiacGeneratorJob::handleFor($this, $periodEndDate, $offset);
// ->map(function ($periodDiff) use ($endDate, $offset) {
// $calculatedEndDate = $endDate->clone();
// $calculatedOffset = (int) $endDate->clone()->previous($this->mail_day)->diffInDays($endDate);
// if ($calculatedOffset === 7) {
// $calculatedOffset = 1;
// }
// if ($periodDiff !== 0) {
// if ($calculatedOffset === 1) {
// for ($i = 1; $i < $periodDiff; $i++) {
// $calculatedEndDate->previous($this->mail_day);
// }
// } else {
// for ($i = 0; $i < $periodDiff; $i++) {
// $calculatedEndDate->previous($this->mail_day);
// }
// }
// $calculatedOffset = 7;
// $calculatedEndDate->subDay();
// }
//
// return ProjectMosiacGeneratorJob::handleFor($this, $calculatedEndDate, $calculatedOffset);
// })
->map(function() use ($endDate, $offset){
$currentEndDate = $endDate->clone();
if (!$currentEndDate->isDayOfWeek($this->mail_day)) {
$endDate->previous($this->mail_day);
}
$endDate->subDay();
$calculatedOffSet = (int) $endDate->clone()->diffInDays($currentEndDate);
return ProjectMosiacGeneratorJob::handleFor($this, $currentEndDate, $calculatedOffSet);
})
->filter()
->toArray();
}
@ -383,6 +413,7 @@ public function setMinHarvestDate():bool
}
return $this->update(['min_harvest_date' => $this->getMinimumDateFromHarvestExcelFile()->format('Y-m-d')]);
}
private function getMinimumDateFromHarvestExcelFile(): Carbon
{
$value = Storage::disk('local')->path($this->harvest_json_path);

6
laravel_app/app/Models/ProjectMosaic.php
View file

@ -29,13 +29,15 @@ class ProjectMosaic extends Model
public static function getFilenameByPeriod(Carbon $endDate, int $offset)
{
return sprintf('week_%s_%s.tif', (clone $endDate)->subdays($offset)->week, $endDate->year);
return sprintf('%s.tif', strtolower(self::projectMosaicNameFormat($endDate, $offset)));
}
public static function projectMosaicNameFormat(Carbon $endDate, int $offset): string
{
$paddedWeek = str_pad($endDate->clone()->subDays($offset)->week, 2, '0', STR_PAD_LEFT);
return sprintf('Week_%s_%s',
$endDate->clone()->subDays($offset)->week,
$paddedWeek,
$endDate->clone()->subDays($offset)->year
);
}

445
laravel_app/composer.lock generated
View file

File diff suppressed because it is too large Load diff

29
laravel_app/tests/Unit/Imports/HarvestDataImportTest.php
View file

@ -11,25 +11,32 @@
class HarvestDataImportTest extends TestCase
{
private $base_path;
private $excelData;
protected function setUp(): void
{
parent::setUp(); // TODO: Change the autogenerated stub
$this->base_path = base_path('tests/__fixtures__/harvest.xlsx');
$this->excelData = Excel::toCollection(new \App\Imports\ExcelFileImport(), $this->base_path);
}
public function test_it_gets_data_from_excel_file()
{
$value = Storage::disk('local')->path('Chemba/Data/harvest.xlsx');
$data = Excel::toCollection(new \App\Imports\ExcelFileImport(),$value);
$season_start_index = $data->first()->first()->search('season_start');
$season_start_index = $this->excelData->first()->first()->search('season_start');
$min = now();
collect($data->first()->slice(1))->each(function($value) use (&$min,$season_start_index){
collect($this->excelData->first()->slice(1))->each(function ($value) use (&$min, $season_start_index) {
$min = min($min, Carbon::instance(SharedDate::excelToDateTimeObject($value[$season_start_index])));
});
$this->assertTrue($min->equalTo('2023-01-12'));
}
public function test_it_gets_the_min_value()
{
$value = Storage::disk('local')->path('Chemba/Data/harvest.xlsx');
$data = Excel::toCollection(new \App\Imports\ExcelFileImport(),$value);
$season_start_index = $data->first()->first()->search('season_start');
$season_start_index = $this->excelData->first()->first()->search('season_start');
$min = now();
collect($data->first()->slice(1))->each(function($value) use (&$min,$season_start_index){
collect($this->excelData->first()->slice(1))->each(function ($value) use (&$min, $season_start_index) {
$min = min($min, Carbon::instance(SharedDate::excelToDateTimeObject($value[$season_start_index])));
});
@ -38,10 +45,8 @@ public function test_it_gets_the_min_value()
public function test_it_gets_the_min_value_using_reduce()
{
$value = Storage::disk('local')->path('chemba/Data/harvest.xlsx');
$data = Excel::toCollection(new \App\Imports\ExcelFileImport(),$value);
$season_start_index = $data->first()->first()->search('season_start');
$min = collect($data->first()->slice(1))->reduce(function($carry,$value,$key) use ($season_start_index){
$season_start_index = $this->excelData->first()->first()->search('season_start');
$min = collect($this->excelData->first()->slice(1))->reduce(function ($carry, $value, $key) use ($season_start_index) {
return min($carry, Carbon::instance(SharedDate::excelToDateTimeObject($value[$season_start_index])));
}, now());
assertEquals(Carbon::parse('2023-01-12'), $min);

5
laravel_app/tests/Unit/Models/ProjectMosaicTest.php
View file

@ -29,15 +29,14 @@ public function it_should_return_the_correct_attachment_path($endDate,$offset,
$this->assertEquals(
$expected,
ProjectMosaic::getFilenameByPeriod($endDate,$offset)
);
}
public static function filenameProvider(){
return [
[new Carbon('2022-01-01'), 10, 'period_2021-12-22_2022-01-01.tif'],
[new Carbon('2022-10-25'),7, 'period_2022-10-18_2022-10-25.tif'],
[new Carbon('2022-01-01'), 7, 'week_52_2021.tif'],
[new Carbon('2022-10-25'),7, 'week_43_2022.tif'],
];
}

56
laravel_app/tests/Unit/Models/ProjectTest.php
View file

@ -77,10 +77,9 @@ public function when_running_the_seeder_their_are_three_projects_with_the_correc
/** @test */
public function when_not_all_mosaics_are_present_it_should_return_an_exception()
{
$this->seed();
$this->expectException(\Exception::class);
$this->expectExceptionMessage('Missing mosaics: period_2021-12-26_2022-01-01.tif, period_2021-12-19_2021-12-25.tif, period_2021-12-12_2021-12-18.tif, period_2021-12-05_2021-12-11.tif');
$this->expectExceptionMessage('Missing mosaics: week_52_2021.tif, week_51_2021.tif, week_50_2021.tif, week_49_2021.tif');
$project = Project::find(1);
$lastDate = Carbon::parse('2022-01-01');
$project->allMosaicsPresent($lastDate);
@ -89,15 +88,15 @@ public function when_not_all_mosaics_are_present_it_should_return_an_exception()
/** @test */
public function when_all_mosaics_are_present_it_should_return_true()
{
// TODO CHeck with Martin the Leading Zero test
$project = Mockery::mock(Project::class)->makePartial();
$project->mail_day = 'Friday';
$lastDate = Carbon::parse('2022-01-01');
$project->shouldReceive('getMosaicList')->andReturn(
collect([
"chemba/weekly_mosaic/period_2021-12-26_2022-01-01.tif",
"chemba/weekly_mosaic/period_2021-12-19_2021-12-25.tif",
"chemba/weekly_mosaic/period_2021-12-12_2021-12-18.tif",
"chemba/weekly_mosaic/period_2021-12-05_2021-12-11.tif",
"chemba/weekly_mosaic/week_52_2021.tif",
"chemba/weekly_mosaic/week_51_2021.tif",
"chemba/weekly_mosaic/week_50_2021.tif",
"chemba/weekly_mosaic/week_49_2021.tif",
]));
$this->assertTrue($project->allMosaicsPresent($lastDate));
@ -107,14 +106,14 @@ public function when_all_mosaics_are_present_it_should_return_true()
public function when_not_mosaics_are_present_it_should_throw_an_exception_listing_the_missing_mosiacs()
{
$this->expectException(\Exception::class);
$this->expectExceptionMessage('Missing mosaics: period_2020-12-05_2020-12-11.tif');
$this->expectExceptionMessage('Missing mosaics: week_49_2020.tif');
$project = Mockery::mock(Project::class)->makePartial();
$lastDate = Carbon::parse('2021-01-01');
$project->shouldReceive('getMosaicList')->andReturn(
collect([
"chemba/weekly_mosaic/period_2020-12-26_2021-01-01.tif",
"chemba/weekly_mosaic/period_2020-12-19_2020-12-25.tif",
"chemba/weekly_mosaic/period_2020-12-12_2020-12-18.tif",
"chemba/weekly_mosaic/week_52_2020.tif",
"chemba/weekly_mosaic/week_51_2020.tif",
"chemba/weekly_mosaic/week_50_2020.tif",
]));
$project->allMosaicsPresent($lastDate);
@ -143,10 +142,10 @@ public function getMosiacFileListByEndDate_should_return_four_filenames()
$list = iterator_to_array($project::getMosaicFilenameListByEndDate($lastDate));
$this->assertCount(4, $list);
$this->assertEquals([
"period_2020-12-26_2021-01-01.tif",
"period_2020-12-19_2020-12-25.tif",
"period_2020-12-12_2020-12-18.tif",
"period_2020-12-05_2020-12-11.tif",
'week_52_2020.tif',
'week_51_2020.tif',
'week_50_2020.tif',
'week_49_2020.tif',
], $list);
}
@ -202,9 +201,34 @@ public function when_getMosaicsFor_is_called_it_returns_a_collection_of_4_jobs_w
return $job->mosaic->name;
})->toArray());
}
/** @test */
public function when_getMosaicsFor_is_called_it_returns_a_collection_of_4_jobs_with_correct_week_numbers_when_end_date_is_mail_day()
{
/* @var Project $project */
$project = Project::create([
'name' => 'project_name',
'download_path' => 'project_download_path',
'mail_day' => 'Friday',
]);
$mosaics = $project->getMosaicsFor(new Carbon('2024-07-05'));
$this->assertCount(4, $mosaics);
collect($mosaics)->each(fn($job) => $this->assertInstanceOf(ProjectMosiacGeneratorJob::class, $job));
$this->assertEquals([
"Week_28_2024",
"Week_27_2024",
"Week_26_2024",
"Week_25_2024",
],
collect($mosaics)->map(function ($job) {
return $job->mosaic->name;
})->toArray());
}
/** @test */
public function when_getMosaicsFor_is_called_it_returns_a_collection_of_4_jobs_with_where_current_day_is_in_week_26_but_the_mail_day_is_in_week_27()
public function when_getMosaicsFor_is_called_it_returns_a_collection_of_4_jobs_with_where_current_day_is_in_week_26_but_the_mail_day_is_in_week_27(
)
{
/* @var Project $project */
$project = Project::create([

BIN
laravel_app/tests/__fixtures__/harvest.xlsx Normal file
View file

Binary file not shown.

24
r_app/CI_report_dashboard_planet.Rmd
View file

@ -72,8 +72,8 @@ source(here("r_app", "parameters_project.R"))
```{r week, message=FALSE, warning=FALSE, include=FALSE}
today <- as.character(report_date)
week <- week(today)
week <- 25
today = "2024-06-21"
# week <- 25
# today = "2024-06-21"
#today = as.character(Sys.Date())
@ -117,10 +117,22 @@ This PDF-dashboard shows the status of your fields on a weekly basis. We will sh
CI_quadrant <- readRDS(here(cumulative_CI_vals_dir,"All_pivots_Cumulative_CI_quadrant_year_v2.rds"))# %>%
# rename(pivot_quadrant = field)
CI <- brick(here(weekly_CI_mosaic, paste0("week_",week, "_", year, ".tif"))) %>% subset("CI")
CI_m1 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_", year_1, ".tif"))) %>% subset("CI")
CI_m2 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_", year_2, ".tif"))) %>% subset("CI")
CI_m3 <- brick(here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_3, ".tif"))) %>% subset("CI")
path_to_week_current = here(weekly_CI_mosaic, paste0("week_",week, "_", year, ".tif"))
path_to_week_minus_1 = here(weekly_CI_mosaic, paste0("week_",week_minus_1, "_", year_1, ".tif"))
path_to_week_minus_2 = here(weekly_CI_mosaic, paste0("week_",week_minus_2, "_", year_2, ".tif"))
path_to_week_minus_3 = here(weekly_CI_mosaic, paste0("week_",week_minus_3, "_", year_3, ".tif"))
print("required mosaic paths")
print(path_to_week_current)
print(path_to_week_minus_1)
print(path_to_week_minus_2)
print(path_to_week_minus_3)
CI <- brick(path_to_week_current) %>% subset("CI")
CI_m1 <- brick(path_to_week_minus_1) %>% subset("CI")
CI_m2 <- brick(path_to_week_minus_2) %>% subset("CI")
CI_m3 <- brick(path_to_week_minus_3) %>% subset("CI")
# last_week_dif_raster <- ((CI - CI_m1) / CI_m1) * 100
last_week_dif_raster_abs <- (CI - CI_m1)

BIN
r_app/Rplots.pdf
View file

Binary file not shown.

2
r_app/ci_extraction_utils.R
View file

@ -1,6 +1,6 @@
# Utils for ci extraction
date_list <- function(end_date, offset){
date_list <- function(end_date, offset, week){
offset <- as.numeric(offset) - 1
end_date <- as.Date(end_date)
start_date <- end_date - lubridate::days(offset)

13
r_app/mosaic_creation.R
View file

@ -45,6 +45,7 @@ if (!is.character(project_dir)) {
laravel_storage_dir <- here("laravel_app/storage/app", project_dir)
#preparing directories
reports_dir <- here(laravel_storage_dir, "reports")
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
@ -65,6 +66,7 @@ source("parameters_project.R")
source("mosaic_creation_utils.R")
dir.create(here(laravel_storage_dir))
dir.create(here(reports_dir))
dir.create(here(data_dir))
dir.create(here(extracted_CI_dir))
dir.create(here(daily_CI_vals_dir))
@ -83,7 +85,14 @@ week <- week(end_date)
# Creating weekly mosaic
#dates <- date_list(weeks_ago)
dates <- date_list(end_date, offset)
file_name_tif <- as.character(args[4])
if (is.na(file_name_tif)) {
file_name_tif <- paste0("week_", sprintf("%02d", dates$week), "_", dates$year, ".tif")
}
print(dates)
print(file_name_tif)
#load pivot geojson
# pivot_sf_q <- st_read(here(data_dir, "pivot.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% vect()
@ -146,7 +155,7 @@ if(sum(missing_pixels_count$thres_5perc)>1){
names(x) <- c("Red", "Green", "Blue", "NIR", "CI")
}else{
message("No cloud free images available, all images combined")
message(vrt_list)
rsrc <- sprc(vrt_list)
x <- mosaic(rsrc, fun = "max")
# x <- rast(vrt_list[1]) %>% setValues(NA)
@ -157,6 +166,6 @@ if(sum(missing_pixels_count$thres_5perc)>1){
plot(x$CI, main = paste("CI map ", dates$week))
plotRGB(x, main = paste("RGB map ", dates$week))
file_path_tif <- here(weekly_CI_mosaic ,paste0("week_", sprintf("%02d", dates$week), "_", dates$year, ".tif"))
file_path_tif <- here(weekly_CI_mosaic ,file_name_tif)
writeRaster(x, file_path_tif, overwrite=TRUE)
message("Raster written/made at: ", file_path_tif)