diff --git a/laravel_app/app/Jobs/ProjectDownloadRDSJob.php b/laravel_app/app/Jobs/ProjectDownloadRDSJob.php index 557c54f..9115de7 100644 --- a/laravel_app/app/Jobs/ProjectDownloadRDSJob.php +++ b/laravel_app/app/Jobs/ProjectDownloadRDSJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\Project; +use App\ProjectLogger; use Carbon\Carbon; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; @@ -44,10 +45,10 @@ public function handle(): void $process->run(); if (!$process->isSuccessful()) { - logger('error', [$process->getErrorOutput()]); + ProjectLogger::log($this->project, $process->getErrorOutput()); return; } - logger($process->getOutput()); + ProjectLogger::log($this->project, $process->getOutput()); } @@ -58,7 +59,7 @@ public static function fromDate(Project $project,Carbon $date,int $offset = 1):N { $filename = $date->format('Y-m-d') . '.tif'; if (!$project->downloads()->statusSuccess()->where(['name' => $filename])->exists()) { - Log::warning("The download $filename can't be processed. Please check it's status."); + ProjectLogger::log($project, "The download $filename can't be processed. Please check it's status."); return new NullJob(); } return new self($project,$date,$offset); diff --git a/laravel_app/app/Jobs/ProjectDownloadTiffJob.php b/laravel_app/app/Jobs/ProjectDownloadTiffJob.php index 8f0486e..4e9784c 100644 --- a/laravel_app/app/Jobs/ProjectDownloadTiffJob.php +++ b/laravel_app/app/Jobs/ProjectDownloadTiffJob.php @@ -4,6 +4,7 @@ use App\Models\Project; use App\Models\ProjectDownload; +use App\ProjectLogger; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; @@ -49,13 +50,15 @@ public function handle(): void $process->setTimeout(600); $process->run(); + $project = $this->download->project; + if (!$process->isSuccessful()) { - logger('error', [$process->getErrorOutput()]); + ProjectLogger::log($project, $process->getErrorOutput()); $this->download->setStatusFailed(); return; } // Success - logger($process->getOutput()); + ProjectLogger::log($project, $process->getOutput()); $this->download->setStatusSuccess(); dispatch(ProjectDownloadRDSJob::fromDate($this->download->project, $this->date, $this->days)); } diff --git a/laravel_app/app/Jobs/ProjectInterpolateGrowthModelJob.php b/laravel_app/app/Jobs/ProjectInterpolateGrowthModelJob.php index c2e838c..073ed5a 100644 --- a/laravel_app/app/Jobs/ProjectInterpolateGrowthModelJob.php +++ b/laravel_app/app/Jobs/ProjectInterpolateGrowthModelJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\Project; +use App\ProjectLogger; use Carbon\Carbon; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; @@ -23,7 +24,7 @@ class ProjectInterpolateGrowthModelJob implements ShouldQueue public function __construct(Project $project) { - logger(__CLASS__ . __METHOD__); + ProjectLogger::log($project, __CLASS__ . __METHOD__); $this->project = $project; } @@ -32,7 +33,7 @@ public function __construct(Project $project) */ public function handle(): void { - logger('start interpolate growth model', [$this->project->name]); + ProjectLogger::log($this->project,'start interpolate growth model', [$this->project->name]); $command = [ sprintf('%s/interpolate_growth_model.sh', base_path('../')), sprintf('--project_dir=%s', $this->project->download_path), @@ -43,10 +44,10 @@ public function handle(): void $process->run(); if (!$process->isSuccessful()) { - logger('error', [$process->getErrorOutput()]); + ProjectLogger::log($this->project, $process->getErrorOutput()); return; } - logger($process->getOutput()); + ProjectLogger::log($this->project, $process->getOutput()); } } diff --git a/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php b/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php index 60b70b4..8956765 100644 --- a/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php +++ b/laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php @@ -4,6 +4,7 @@ use App\Models\Project; use App\Models\ProjectMosaic; +use App\ProjectLogger; use Carbon\Carbon; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; @@ -45,6 +46,7 @@ public function __construct(ProjectMosaic $mosaic) public function handle(): void { $projectFolder = base_path('../'); + $project = $this->mosaic->project; $command = [ sprintf('%sbuild_mosaic.sh', $projectFolder), @@ -53,20 +55,23 @@ public function handle(): void sprintf('--data_dir=%s', $this->mosaic->project->download_path), sprintf('--file_name_tif=%s', basename($this->mosaic->path)), ]; + + ProjectLogger::log($project, 'command:'. print_r($command, true)); + $currentPath = '/usr/bin:/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin/Users/mfolkerts/anaconda3/bin:/Library/Apple/usr/bin'; try { $process = ProcessNew::timeout(220) ->env(['PATH' => $currentPath.':/usr/local/Cellar/pandoc/3.1.8/bin/pandoc']) - ->start($command, function (string $type, string $output) { - logger($output); + ->start($command, function (string $type, string $output) use ($project) { + ProjectLogger::log($project, $output); }); $results = $process->wait(); if ($results->successful()) { $this->mosaic->setStatusSuccess(); } } catch (\RuntimeException|ProcessTimedOutException|ProcessFailedException $e) { - echo $e->getMessage(); + ProjectLogger::log($project, $e->getMessage()); $this->mosaic->setStatusFailed(); } } @@ -77,13 +82,13 @@ public function handle(): void public static function handleFor(Project $project, Carbon $endDate, int $offset): NullJob|ProjectMosiacGeneratorJob { $endDate = $endDate->clone(); - logger("ProjectMosiacGeneratorJob::handleFor($endDate, $offset)"); + ProjectLogger::log($project, "ProjectMosiacGeneratorJob::handleFor($endDate, $offset)"); if ($project->hasInvalidMosaicFor($endDate, $offset)) { - logger("ProjecMosaicGeneratorJob::handleFor(end_date: $endDate, offset: $offset): InvalidMosaic."); + ProjectLogger::log($project,"ProjecMosaicGeneratorJob::handleFor(end_date: $endDate, offset: $offset): InvalidMosaic."); return new NullJob(); } - logger(__CLASS__."::".__METHOD__."::Project->mail_day::".$project->mail_day); + ProjectLogger::log($project, __CLASS__."::".__METHOD__."::Project->mail_day::".$project->mail_day); if (Carbon::parse($project->mail_day)->dayOfWeek < $endDate->dayOfWeek) { $endDate->next($project->mail_day); diff --git a/laravel_app/app/Jobs/ProjectReportGeneratorJob.php b/laravel_app/app/Jobs/ProjectReportGeneratorJob.php index 94a5a53..55dcc1d 100644 --- a/laravel_app/app/Jobs/ProjectReportGeneratorJob.php +++ b/laravel_app/app/Jobs/ProjectReportGeneratorJob.php @@ -4,6 +4,7 @@ use App\Livewire\Forms\MailingForm; use App\Models\ProjectReport; +use App\ProjectLogger; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; @@ -50,7 +51,7 @@ public function handle() sprintf('--data_dir=%s', $this->projectReport->project->download_path), sprintf('--borders=%s', $this->projectReport->project->borders ? 'True' : 'False'), ]; - logger('command:'. print_r($command, true)); + ProjectLogger::log($this->projectReport->project, 'command:'. print_r($command, true)); // Convert commands array to a single string @@ -65,7 +66,7 @@ public function handle() $process->wait(function ($type, $buffer) use (&$myOutput) { // $this->stream(to: 'processOutput', content: $buffer); $myOutput[] = $buffer; - logger($buffer); + ProjectLogger::log($this->projectReport->project, $buffer); }); $this->processOutput = collect($myOutput)->join('\n'); $this->projectReport->setStatusSuccess(); diff --git a/laravel_app/app/Jobs/newDownloadsUploaded.php b/laravel_app/app/Jobs/newDownloadsUploaded.php index ced2e88..1d63111 100644 --- a/laravel_app/app/Jobs/newDownloadsUploaded.php +++ b/laravel_app/app/Jobs/newDownloadsUploaded.php @@ -2,6 +2,7 @@ namespace App\Jobs; +use App\Models\Project; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; diff --git a/laravel_app/app/Models/Project.php b/laravel_app/app/Models/Project.php index 4dfaf7a..075a279 100644 --- a/laravel_app/app/Models/Project.php +++ b/laravel_app/app/Models/Project.php @@ -477,7 +477,6 @@ public function newDownloadsUploaded() public function getRdsAsDownload() { - $path = $this->download_path.'/Data/extracted_ci/cumulative_vals/combined_CI_data.rds'; return Storage::download( diff --git a/laravel_app/app/ProjectLogger.php b/laravel_app/app/ProjectLogger.php new file mode 100644 index 0000000..e4b4e8e --- /dev/null +++ b/laravel_app/app/ProjectLogger.php @@ -0,0 +1,16 @@ +download_path . '/logs/' . now()->format('Ymd') . '.log'; + Storage::makeDirectory(dirname($filePath)); + file_put_contents(Storage::path($filePath), now()->format('Y-m-d H:i:s') . ': ' . $message . PHP_EOL, FILE_APPEND); + } +} diff --git a/r_app/CI_report_dashboard_planet.Rmd b/r_app/CI_report_dashboard_planet.Rmd index 5958292..f466b04 100644 --- a/r_app/CI_report_dashboard_planet.Rmd +++ b/r_app/CI_report_dashboard_planet.Rmd @@ -52,30 +52,14 @@ library(caret) library(randomForest) library(CAST) -# source(here("r_app/report_utils.R")) source("report_utils.R") - ``` ```{r directories, message=FALSE, warning=FALSE, include=FALSE} project_dir <- params$data_dir -# laravel_storage_dir <- here("laravel_app/storage/app/",params$data_dir) -laravel_storage_dir <- here("laravel_app/storage/app",project_dir) - -data_dir <- here(laravel_storage_dir, "Data") -# message('DATA_DIR',data_dir) -extracted_CI_dir <- here(data_dir, "extracted_ci") -daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") -cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") -harvest_dir <- here(data_dir, "HarvestData") - -weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") - source(here("r_app", "parameters_project.R")) - - log_message("Starting the R Markdown script") log_message(paste("mail_day params:", params$mail_day)) log_message(paste("report_date params:", params$report_date)) diff --git a/r_app/Rplots.pdf b/r_app/Rplots.pdf index dcaa63d..12e588a 100644 Binary files a/r_app/Rplots.pdf and b/r_app/Rplots.pdf differ diff --git a/r_app/ci_extraction.R b/r_app/ci_extraction.R index d9b609a..9e63793 100644 --- a/r_app/ci_extraction.R +++ b/r_app/ci_extraction.R @@ -1,5 +1,4 @@ # nolint start: commented_code_linter, line_length_linter,object_usage_linter. -library(here) library(sf) library(terra) library(tidyverse) @@ -18,7 +17,7 @@ if (length(args) == 0) { # Converteer het eerste argument naar een numerieke waarde end_date <- as.Date(args[1]) if (is.na(end_date)) { -end_date <- lubridate::dmy("28-08-2024") + end_date <- lubridate::dmy("28-08-2024") } offset <- as.numeric(args[2]) @@ -36,42 +35,10 @@ project_dir <- as.character(args[3]) if (!is.character(project_dir)) { project_dir <- "chemba" } - -laravel_storage_dir <- here("laravel_app/storage/app", project_dir) -#preparing directories -planet_tif_folder <- here(laravel_storage_dir, "merged_tif") -merged_final <- here(laravel_storage_dir, "merged_final_tif") - new_project_question = FALSE -planet_tif_folder <- here(laravel_storage_dir, "merged_tif") -merged_final <- here(laravel_storage_dir, "merged_final_tif") - -data_dir <- here(laravel_storage_dir, "Data") -extracted_CI_dir <- here(data_dir, "extracted_ci") -daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") -cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") - -weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") -daily_vrt <- here(data_dir, "vrt") -harvest_dir <- here(data_dir, "HarvestData") - -# source(here("r_app/parameters_project.R")) -# source(here("r_app/ci_extraction_utils.R")) -# source(here("r_app/mosaic_creation_utils.R")) source("parameters_project.R") source("ci_extraction_utils.R") -# source("mosaic_creation_utils.R") - -dir.create(here(laravel_storage_dir)) -dir.create(here(data_dir)) -dir.create(here(extracted_CI_dir)) -dir.create(here(daily_CI_vals_dir)) -dir.create(here(cumulative_CI_vals_dir)) -dir.create(here(weekly_CI_mosaic)) -dir.create(here(daily_vrt)) -dir.create(merged_final) -dir.create(harvest_dir) dates <- date_list(end_date, offset) print(dates) diff --git a/r_app/interpolate_growth_model.R b/r_app/interpolate_growth_model.R index 3518929..24bfb56 100644 --- a/r_app/interpolate_growth_model.R +++ b/r_app/interpolate_growth_model.R @@ -1,17 +1,12 @@ - -library(here) library(sf) library(terra) library(tidyverse) library(lubridate) library(exactextractr) -# library(readxl) - # Vang alle command line argumenten op args <- commandArgs(trailingOnly = TRUE) - # Converteer het tweede argument naar een string waarde project_dir <- as.character(args[1]) @@ -20,27 +15,6 @@ if (!is.character(project_dir)) { project_dir <- "chemba" } - -laravel_storage_dir <- here("laravel_app/storage/app", project_dir) -#preparing directories -planet_tif_folder <- here(laravel_storage_dir, "merged_tif") -merged_final <- here(laravel_storage_dir, "merged_final_tif") - -new_project_question = FALSE - -data_dir <- here(laravel_storage_dir, "Data") -extracted_CI_dir <- here(data_dir, "extracted_ci") -daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") -cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") - -weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") -daily_vrt <- here(data_dir, "vrt") -harvest_dir <- here(data_dir, "HarvestData") - -# source(here("r_app/parameters_project.R")) -# source(here("r_app/ci_extraction_utils.R")) -# source(here("r_app/mosaic_creation_utils.R")) - source("parameters_project.R") source("ci_extraction_utils.R") diff --git a/r_app/mosaic_creation.R b/r_app/mosaic_creation.R index 6e8ac56..85a3d18 100644 --- a/r_app/mosaic_creation.R +++ b/r_app/mosaic_creation.R @@ -1,19 +1,7 @@ -# activeer de renv omgeving; - -# renv::activate('~/smartCane/r_app') -# renv::restore() - - - -library(here) library(sf) library(terra) library(tidyverse) library(lubridate) -# library(exactextractr) -# library(readxl) -#funcion CI_prep package - # Vang alle command line argumenten op args <- commandArgs(trailingOnly = TRUE) @@ -42,51 +30,11 @@ if (!is.character(project_dir)) { project_dir <- "chemba" } - -laravel_storage_dir <- here("laravel_app/storage/app", project_dir) -#preparing directories -reports_dir <- here(laravel_storage_dir, "reports") -planet_tif_folder <- here(laravel_storage_dir, "merged_tif") -merged_final <- here(laravel_storage_dir, "merged_final_tif") - -new_project_question = FALSE -planet_tif_folder <- here(laravel_storage_dir, "merged_tif") -merged_final <- here(laravel_storage_dir, "merged_final_tif") - -data_dir <- here(laravel_storage_dir, "Data") -extracted_CI_dir <- here(data_dir, "extracted_ci") -daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") -cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") - -weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") -daily_vrt <- here(data_dir, "vrt") -harvest_dir <- here(data_dir, "HarvestData") - -# source(here("r_app/parameters_project.R")) -# source(here("r_app/mosaic_creation_utils.R")) - source("parameters_project.R") source("mosaic_creation_utils.R") -# dir.create(here(laravel_storage_dir)) -# dir.create(here(reports_dir)) -# dir.create(here(data_dir)) -# dir.create(here(extracted_CI_dir)) -# dir.create(here(daily_CI_vals_dir)) -# dir.create(here(cumulative_CI_vals_dir)) -# dir.create(here(weekly_CI_mosaic)) -# dir.create(here(daily_vrt)) -# dir.create(merged_final) -# dir.create(harvest_dir) - -# end_date <- lubridate::dmy("28-8-2024") week <- week(end_date) - -#weeks_ago = 0 - -# Creating weekly mosaic -#dates <- date_list(weeks_ago) dates <- date_list(end_date, offset) file_name_tif <- as.character(args[4]) diff --git a/r_app/parameters_project.R b/r_app/parameters_project.R index dd8ad28..5e6511a 100644 --- a/r_app/parameters_project.R +++ b/r_app/parameters_project.R @@ -1,7 +1,33 @@ - +library(here) library('readxl') #chemba +laravel_storage_dir <- here("laravel_app/storage/app", project_dir) +reports_dir <- here(laravel_storage_dir, "reports") +log_dir <- here(laravel_storage_dir, "logs") +planet_tif_folder <- here(laravel_storage_dir, "merged_tif") +merged_final <- here(laravel_storage_dir, "merged_final_tif") +planet_tif_folder <- here(laravel_storage_dir, "merged_tif") +merged_final <- here(laravel_storage_dir, "merged_final_tif") +data_dir <- here(laravel_storage_dir, "Data") +extracted_CI_dir <- here(data_dir, "extracted_ci") +daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals") +cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals") +weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic") +daily_vrt <- here(data_dir, "vrt") +harvest_dir <- here(data_dir, "HarvestData") + +dir.create(here(laravel_storage_dir), showWarnings = FALSE) +dir.create(here(data_dir), showWarnings = FALSE) +dir.create(here(log_dir), showWarnings = FALSE) +dir.create(here(extracted_CI_dir), showWarnings = FALSE) +dir.create(here(daily_CI_vals_dir), showWarnings = FALSE) +dir.create(here(cumulative_CI_vals_dir), showWarnings = FALSE) +dir.create(here(weekly_CI_mosaic),showWarnings = FALSE) +dir.create(here(daily_vrt), showWarnings = FALSE) +dir.create(merged_final,showWarnings = FALSE) +dir.create(harvest_dir,showWarnings = FALSE) + field_boundaries_sf <- st_read(here(data_dir, "pivot.geojson")) names(field_boundaries_sf) <- c("field", "sub_field", "geometry") @@ -36,8 +62,7 @@ harvesting_data <- read_excel(here(data_dir, "harvest.xlsx")) %>% age = round(as.numeric(season_end - season_start) / 7, 0) ) -# Define the log file path -log_file <- here("laravel_app/storage/app/rmd_log.txt") +log_file <- here(log_dir, paste0("log_", format(Sys.Date(), "%Y%m%d"), ".log")) # Create a logging function log_message <- function(message) {