Timon/SmartCane
1
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

added log as project file;

Browse source
This commit is contained in:
Martin Folkerts 2024-08-30 16:16:26 +02:00
parent 499744db6d
commit 7bf4f0930d
14 changed files with 74 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
laravel_app/app/Jobs/ProjectDownloadRDSJob.php
View file

@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\Project;
use App\ProjectLogger;
use Carbon\Carbon;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
@ -44,10 +45,10 @@ public function handle(): void
$process->run();
if (!$process->isSuccessful()) {
logger('error', [$process->getErrorOutput()]);
ProjectLogger::log($this->project, $process->getErrorOutput());
return;
}
logger($process->getOutput());
ProjectLogger::log($this->project, $process->getOutput());
}
@ -58,7 +59,7 @@ public static function fromDate(Project $project,Carbon $date,int $offset = 1):N
{
$filename = $date->format('Y-m-d') . '.tif';
if (!$project->downloads()->statusSuccess()->where(['name' => $filename])->exists()) {
Log::warning("The download $filename can't be processed. Please check it's status.");
ProjectLogger::log($project, "The download $filename can't be processed. Please check it's status.");
return new NullJob();
}
return new self($project,$date,$offset);

7
laravel_app/app/Jobs/ProjectDownloadTiffJob.php
View file

@ -4,6 +4,7 @@
use App\Models\Project;
use App\Models\ProjectDownload;
use App\ProjectLogger;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -49,13 +50,15 @@ public function handle(): void
$process->setTimeout(600);
$process->run();
$project = $this->download->project;
if (!$process->isSuccessful()) {
logger('error', [$process->getErrorOutput()]);
ProjectLogger::log($project, $process->getErrorOutput());
$this->download->setStatusFailed();
return;
}
// Success
logger($process->getOutput());
ProjectLogger::log($project, $process->getOutput());
$this->download->setStatusSuccess();
dispatch(ProjectDownloadRDSJob::fromDate($this->download->project, $this->date, $this->days));
}

9
laravel_app/app/Jobs/ProjectInterpolateGrowthModelJob.php
View file

@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\Project;
use App\ProjectLogger;
use Carbon\Carbon;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
@ -23,7 +24,7 @@ class ProjectInterpolateGrowthModelJob implements ShouldQueue
public function __construct(Project $project)
{
logger(__CLASS__ . __METHOD__);
ProjectLogger::log($project, __CLASS__ . __METHOD__);
$this->project = $project;
}
@ -32,7 +33,7 @@ public function __construct(Project $project)
*/
public function handle(): void
{
logger('start interpolate growth model', [$this->project->name]);
ProjectLogger::log($this->project,'start interpolate growth model', [$this->project->name]);
$command = [
sprintf('%s/interpolate_growth_model.sh', base_path('../')),
sprintf('--project_dir=%s', $this->project->download_path),
@ -43,10 +44,10 @@ public function handle(): void
$process->run();
if (!$process->isSuccessful()) {
logger('error', [$process->getErrorOutput()]);
ProjectLogger::log($this->project, $process->getErrorOutput());
return;
}
logger($process->getOutput());
ProjectLogger::log($this->project, $process->getOutput());
}
}

17
laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php
View file

@ -4,6 +4,7 @@
use App\Models\Project;
use App\Models\ProjectMosaic;
use App\ProjectLogger;
use Carbon\Carbon;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
@ -45,6 +46,7 @@ public function __construct(ProjectMosaic $mosaic)
public function handle(): void
{
$projectFolder = base_path('../');
$project = $this->mosaic->project;
$command = [
sprintf('%sbuild_mosaic.sh', $projectFolder),
@ -53,20 +55,23 @@ public function handle(): void
sprintf('--data_dir=%s', $this->mosaic->project->download_path),
sprintf('--file_name_tif=%s', basename($this->mosaic->path)),
];
ProjectLogger::log($project, 'command:'. print_r($command, true));
$currentPath = '/usr/bin:/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin/Users/mfolkerts/anaconda3/bin:/Library/Apple/usr/bin';
try {
$process = ProcessNew::timeout(220)
->env(['PATH' => $currentPath.':/usr/local/Cellar/pandoc/3.1.8/bin/pandoc'])
->start($command, function (string $type, string $output) {
logger($output);
->start($command, function (string $type, string $output) use ($project) {
ProjectLogger::log($project, $output);
});
$results = $process->wait();
if ($results->successful()) {
$this->mosaic->setStatusSuccess();
}
} catch (\RuntimeException|ProcessTimedOutException|ProcessFailedException $e) {
echo $e->getMessage();
ProjectLogger::log($project, $e->getMessage());
$this->mosaic->setStatusFailed();
}
}
@ -77,13 +82,13 @@ public function handle(): void
public static function handleFor(Project $project, Carbon $endDate, int $offset): NullJob|ProjectMosiacGeneratorJob
{
$endDate = $endDate->clone();
logger("ProjectMosiacGeneratorJob::handleFor($endDate, $offset)");
ProjectLogger::log($project, "ProjectMosiacGeneratorJob::handleFor($endDate, $offset)");
if ($project->hasInvalidMosaicFor($endDate, $offset)) {
logger("ProjecMosaicGeneratorJob::handleFor(end_date: $endDate, offset: $offset): InvalidMosaic.");
ProjectLogger::log($project,"ProjecMosaicGeneratorJob::handleFor(end_date: $endDate, offset: $offset): InvalidMosaic.");
return new NullJob();
}
logger(__CLASS__."::".__METHOD__."::Project->mail_day::".$project->mail_day);
ProjectLogger::log($project, __CLASS__."::".__METHOD__."::Project->mail_day::".$project->mail_day);
if (Carbon::parse($project->mail_day)->dayOfWeek < $endDate->dayOfWeek) {
$endDate->next($project->mail_day);

5
laravel_app/app/Jobs/ProjectReportGeneratorJob.php
View file

@ -4,6 +4,7 @@
use App\Livewire\Forms\MailingForm;
use App\Models\ProjectReport;
use App\ProjectLogger;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
@ -50,7 +51,7 @@ public function handle()
sprintf('--data_dir=%s', $this->projectReport->project->download_path),
sprintf('--borders=%s', $this->projectReport->project->borders ? 'True' : 'False'),
];
logger('command:'. print_r($command, true));
ProjectLogger::log($this->projectReport->project, 'command:'. print_r($command, true));
// Convert commands array to a single string
@ -65,7 +66,7 @@ public function handle()
$process->wait(function ($type, $buffer) use (&$myOutput) {
// $this->stream(to: 'processOutput', content: $buffer);
$myOutput[] = $buffer;
logger($buffer);
ProjectLogger::log($this->projectReport->project, $buffer);
});
$this->processOutput = collect($myOutput)->join('\n');
$this->projectReport->setStatusSuccess();

1
laravel_app/app/Jobs/newDownloadsUploaded.php
View file

@ -2,6 +2,7 @@
namespace App\Jobs;
use App\Models\Project;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;

1
laravel_app/app/Models/Project.php
View file

@ -477,7 +477,6 @@ public function newDownloadsUploaded()
public function getRdsAsDownload()
{
$path = $this->download_path.'/Data/extracted_ci/cumulative_vals/combined_CI_data.rds';
return Storage::download(

16
laravel_app/app/ProjectLogger.php Normal file
View file

@ -0,0 +1,16 @@
<?php
namespace App;
use App\Models\Project;
use Illuminate\Support\Facades\Storage;
class ProjectLogger
{
public static function log(Project $project, $message)
{
$filePath = $project->download_path . '/logs/' . now()->format('Ymd') . '.log';
Storage::makeDirectory(dirname($filePath));
file_put_contents(Storage::path($filePath), now()->format('Y-m-d H:i:s') . ': ' . $message . PHP_EOL, FILE_APPEND);
}
}

16
r_app/CI_report_dashboard_planet.Rmd
View file

@ -52,30 +52,14 @@ library(caret)
library(randomForest)
library(CAST)
# source(here("r_app/report_utils.R"))
source("report_utils.R")
```
```{r directories, message=FALSE, warning=FALSE, include=FALSE}
project_dir <- params$data_dir
# laravel_storage_dir <- here("laravel_app/storage/app/",params$data_dir)
laravel_storage_dir <- here("laravel_app/storage/app",project_dir)
data_dir <- here(laravel_storage_dir, "Data")
# message('DATA_DIR',data_dir)
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
harvest_dir <- here(data_dir, "HarvestData")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
source(here("r_app", "parameters_project.R"))
log_message("Starting the R Markdown script")
log_message(paste("mail_day params:", params$mail_day))
log_message(paste("report_date params:", params$report_date))

BIN
r_app/Rplots.pdf
View file

Binary file not shown.

35
r_app/ci_extraction.R
View file

@ -1,5 +1,4 @@
# nolint start: commented_code_linter, line_length_linter,object_usage_linter.
library(here)
library(sf)
library(terra)
library(tidyverse)
@ -18,7 +17,7 @@ if (length(args) == 0) {
# Converteer het eerste argument naar een numerieke waarde
end_date <- as.Date(args[1])
if (is.na(end_date)) {
end_date <- lubridate::dmy("28-08-2024")
end_date <- lubridate::dmy("28-08-2024")
}
offset <- as.numeric(args[2])
@ -36,42 +35,10 @@ project_dir <- as.character(args[3])
if (!is.character(project_dir)) {
project_dir <- "chemba"
}
laravel_storage_dir <- here("laravel_app/storage/app", project_dir)
#preparing directories
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
new_project_question = FALSE
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
data_dir <- here(laravel_storage_dir, "Data")
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
daily_vrt <- here(data_dir, "vrt")
harvest_dir <- here(data_dir, "HarvestData")
# source(here("r_app/parameters_project.R"))
# source(here("r_app/ci_extraction_utils.R"))
# source(here("r_app/mosaic_creation_utils.R"))
source("parameters_project.R")
source("ci_extraction_utils.R")
# source("mosaic_creation_utils.R")
dir.create(here(laravel_storage_dir))
dir.create(here(data_dir))
dir.create(here(extracted_CI_dir))
dir.create(here(daily_CI_vals_dir))
dir.create(here(cumulative_CI_vals_dir))
dir.create(here(weekly_CI_mosaic))
dir.create(here(daily_vrt))
dir.create(merged_final)
dir.create(harvest_dir)
dates <- date_list(end_date, offset)
print(dates)

26
r_app/interpolate_growth_model.R
View file

@ -1,17 +1,12 @@
library(here)
library(sf)
library(terra)
library(tidyverse)
library(lubridate)
library(exactextractr)
# library(readxl)
# Vang alle command line argumenten op
args <- commandArgs(trailingOnly = TRUE)
# Converteer het tweede argument naar een string waarde
project_dir <- as.character(args[1])
@ -20,27 +15,6 @@ if (!is.character(project_dir)) {
project_dir <- "chemba"
}
laravel_storage_dir <- here("laravel_app/storage/app", project_dir)
#preparing directories
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
new_project_question = FALSE
data_dir <- here(laravel_storage_dir, "Data")
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
daily_vrt <- here(data_dir, "vrt")
harvest_dir <- here(data_dir, "HarvestData")
# source(here("r_app/parameters_project.R"))
# source(here("r_app/ci_extraction_utils.R"))
# source(here("r_app/mosaic_creation_utils.R"))
source("parameters_project.R")
source("ci_extraction_utils.R")

52
r_app/mosaic_creation.R
View file

@ -1,19 +1,7 @@
# activeer de renv omgeving;
# renv::activate('~/smartCane/r_app')
# renv::restore()
library(here)
library(sf)
library(terra)
library(tidyverse)
library(lubridate)
# library(exactextractr)
# library(readxl)
#funcion CI_prep package
# Vang alle command line argumenten op
args <- commandArgs(trailingOnly = TRUE)
@ -42,51 +30,11 @@ if (!is.character(project_dir)) {
project_dir <- "chemba"
}
laravel_storage_dir <- here("laravel_app/storage/app", project_dir)
#preparing directories
reports_dir <- here(laravel_storage_dir, "reports")
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
new_project_question = FALSE
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
data_dir <- here(laravel_storage_dir, "Data")
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
daily_vrt <- here(data_dir, "vrt")
harvest_dir <- here(data_dir, "HarvestData")
# source(here("r_app/parameters_project.R"))
# source(here("r_app/mosaic_creation_utils.R"))
source("parameters_project.R")
source("mosaic_creation_utils.R")
# dir.create(here(laravel_storage_dir))
# dir.create(here(reports_dir))
# dir.create(here(data_dir))
# dir.create(here(extracted_CI_dir))
# dir.create(here(daily_CI_vals_dir))
# dir.create(here(cumulative_CI_vals_dir))
# dir.create(here(weekly_CI_mosaic))
# dir.create(here(daily_vrt))
# dir.create(merged_final)
# dir.create(harvest_dir)
# end_date <- lubridate::dmy("28-8-2024")
week <- week(end_date)
#weeks_ago = 0
# Creating weekly mosaic
#dates <- date_list(weeks_ago)
dates <- date_list(end_date, offset)
file_name_tif <- as.character(args[4])

31
r_app/parameters_project.R
View file

@ -1,7 +1,33 @@
library(here)
library('readxl')
#chemba
laravel_storage_dir <- here("laravel_app/storage/app", project_dir)
reports_dir <- here(laravel_storage_dir, "reports")
log_dir <- here(laravel_storage_dir, "logs")
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
planet_tif_folder <- here(laravel_storage_dir, "merged_tif")
merged_final <- here(laravel_storage_dir, "merged_final_tif")
data_dir <- here(laravel_storage_dir, "Data")
extracted_CI_dir <- here(data_dir, "extracted_ci")
daily_CI_vals_dir <- here(extracted_CI_dir, "daily_vals")
cumulative_CI_vals_dir <- here(extracted_CI_dir, "cumulative_vals")
weekly_CI_mosaic <- here(laravel_storage_dir, "weekly_mosaic")
daily_vrt <- here(data_dir, "vrt")
harvest_dir <- here(data_dir, "HarvestData")
dir.create(here(laravel_storage_dir), showWarnings = FALSE)
dir.create(here(data_dir), showWarnings = FALSE)
dir.create(here(log_dir), showWarnings = FALSE)
dir.create(here(extracted_CI_dir), showWarnings = FALSE)
dir.create(here(daily_CI_vals_dir), showWarnings = FALSE)
dir.create(here(cumulative_CI_vals_dir), showWarnings = FALSE)
dir.create(here(weekly_CI_mosaic),showWarnings = FALSE)
dir.create(here(daily_vrt), showWarnings = FALSE)
dir.create(merged_final,showWarnings = FALSE)
dir.create(harvest_dir,showWarnings = FALSE)
field_boundaries_sf <- st_read(here(data_dir, "pivot.geojson"))
names(field_boundaries_sf) <- c("field", "sub_field", "geometry")
@ -36,8 +62,7 @@ harvesting_data <- read_excel(here(data_dir, "harvest.xlsx")) %>%
age = round(as.numeric(season_end - season_start) / 7, 0)
)
# Define the log file path
log_file <- here("laravel_app/storage/app/rmd_log.txt")
log_file <- here(log_dir, paste0("log_", format(Sys.Date(), "%Y%m%d"), ".log"))
# Create a logging function
log_message <- function(message) {