Timon/SmartCane
1
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

added rule for missing mergedTifs;

Browse source
This commit is contained in:
Martin Folkerts 2023-12-24 18:02:17 +01:00
parent eb1def36a1
commit 3fb55ddcce
20 changed files with 1720 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
build_mosaic.sh Executable file
View file

@ -0,0 +1,30 @@
#!/bin/bash
filename="report_dummie.docx"
weeks_ago="0"
report_date="2022-12-12"
# Parse de command line argumenten
while [ "$#" -gt 0 ]; do
case "$1" in
--weeks_ago=*)
weeks_ago="${1#*=}"
;;
*)
echo "Onbekende optie: $1"
exit 1
;;
esac
shift
done
# Controleer of de vereiste argumenten zijn ingesteld
if [ -z "$filename" ] || [ -z "$weeks_ago" ] || [ -z $report_date ]; then
echo "Missende argumenten. Gebruik: build_reports.sh --filename=hello.txt --weeks_ago=3 --report_date=2020-01-01"
exit 1
fi
echo "Weeks ago: $weeks_ago"
cd /Users/mfolkerts/smartCane/r_app
Rscript 2_CI_data_prep.R $weeks_ago

48
build_report.sh
View file

@ -1,15 +1,51 @@
#!/bin/bash
## Runnen van Rmd bestand
# -e betekent "evalueren" en -i specificeert de input file
Rscript --version
echo $PATH
filename="report_dummie.docx"
weeks_ago="0"
report_date="2022-12-12"
# Parse de command line argumenten
while [ "$#" -gt 0 ]; do
case "$1" in
--filename=*)
filename="${1#*=}"
;;
--weeks_ago=*)
weeks_ago="${1#*=}"
;;
--report_date=*)
report_date="${1#*=}"
;;
*)
echo "Onbekende optie: $1"
exit 1
;;
esac
shift
done
# Controleer of de vereiste argumenten zijn ingesteld
if [ -z "$filename" ] || [ -z "$weeks_ago" ] || [ -z $report_date ]; then
echo "Missende argumenten. Gebruik: build_reports.sh --filename=hello.txt --weeks_ago=3 --report_date=2020-01-01"
exit 1
fi
# Je script logica hier
echo "Filename: $filename"
echo "Weeks ago: $weeks_ago"
echo "Report date: $report_date"
#script_dir="$(dirname "$0")"
#echo $script_dir;
#/bin/ps aux | grep fontd
#output_file="/Users/mfolkerts/smartCane/laravel_app/storage/app/chemba/$filename"
#echo "Output file: $output_file"
echo '\n'
echo 'zx28tb' | /usr/bin/sudo -S launchctl load -w /System/Library/LaunchAgents/com.apple.fontd.useragent.plist
cd /Users/mfolkerts/smartCane/r_app
Rscript -e "rmarkdown::render('/Users/mfolkerts/smartCane/r_app/CI_Report_dashboard_planet.Rmd', 'all')"
#Rscript -e "rmarkdown::render('/Users/mfolkerts/smartCane/r_app/CI_Report_dashboard_planet.Rmd', 'all')"
#Rscript 1_harvest_data_EcoFarm_v2.R
Rscript 2_CI_data_prep.R $weeks_ago
Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$filename', params=list(ref='$ref', report_date='$report_date'))"

69
laravel_app/app/Jobs/ProjectMosiacGeneratorJob.php Normal file
View file

@ -0,0 +1,69 @@
<?php
namespace App\Jobs;
use App\Models\Project;
use App\Models\ProjectReport;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Symfony\Component\Process\Process;
class ProjectMosiacGeneratorJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $year;
public $week;
public Project $project;
/**
* Create a new job instance.
*/
public function __construct(Project $project, $year, $week)
{
$this->year = $year;
$this->week = $week;
//
}
/**
* Execute the job.
*/
public function handle(): void
{
$weeksAgo = ProjectReport::weeksAgoForYearAndWeek($this->year, $this->week);
$projectFolder = base_path('../');
$command = [
sprintf('%sbuild_mosaic.sh', $projectFolder),
sprintf('--weeks_ago=%s', $weeksAgo),
];
$process = new Process($command);
$process->setTimeout(120);
$currentPath = '/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin/Users/mfolkerts/anaconda3/bin:/Library/Apple/usr/bin';
$process->setEnv(['PATH' => $currentPath.':/usr/local/Cellar/pandoc/3.1.8/bin/pandoc']);
// $process->setTimeout(36000); // stel een geschikte timeout in
$process->start();
try {
$myOutput = [];
$process->wait(function ($type, $buffer) use (&$myOutput) {
// $this->stream(to: 'processOutput', content: $buffer);
$myOutput[] = $buffer;
logger($buffer);
});
$this->processOutput = collect($myOutput)->join('\n');
} catch (ProcessFailedException $exception) {
echo $exception->getMessage();
}
}
}

58
laravel_app/app/Livewire/Projects/MosaicManager.php Normal file
View file

@ -0,0 +1,58 @@
<?php
namespace App\Livewire\Projects;
use App\Jobs\ProjectMosiacGeneratorJob;
use App\Models\Project;
use Livewire\Component;
class MosaicManager extends Component
{
public $project;
public $formData = [
'week' => '',
'year' => '',
];
public $showCreateModal = false;
public function mount(Project $project) {
$this->path = $project->download_path;
}
public function render()
{
return view('livewire.projects.mosaic-manager', [
'downloads' => $this->project
]);
}
public function saveMosiac(){
$this->validate([
'formData.year' => 'required',
'formData.week' => 'required',
]);
ProjectMosiacGeneratorJob::dispatch($this->project, $this->formData['year'], $this->formData['week']);
$this->showCreateModal = false;
}
public function openCreateMosiacsModal(){
$this->showCreateModal = true;
}
public function getDateRangeProperty()
{
if (empty($this->formData['week']) || strlen($this->formData['year']) !== 4) {
return '<span class="text-red-500">Invalid week or year</span>';
}
$begin = now()
->setISODate($this->formData['year'], $this->formData['week'])
->startOfWeek();
$end = now()
->setISODate($this->formData['year'], $this->formData['week'])
->endOfWeek();
return $begin->format('Y-m-d').' - '.$end->format('Y-m-d');
}
}

50
laravel_app/app/Models/Project.php
View file

@ -5,6 +5,7 @@
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
@ -143,7 +144,7 @@ public function allMosaicsPresent(Carbon $endDate)
$mosaicsNotPresentInFilesystem = $this->getMosiacFilenameListByEndDate($endDate)
->filter(function ($filename) {
return !$this->getMosaicList()->contains(function ($mosaicFilename) use ($filename) {
return Str::endsWith( $mosaicFilename, substr($filename, -16));
return Str::endsWith($mosaicFilename, substr($filename, -16));
});
});
if ($mosaicsNotPresentInFilesystem->count() === 0) {
@ -173,5 +174,52 @@ public function getMosiacFilenameListByEndDate(Carbon $endDate): \Illuminate\Sup
return $result;
}
public function getMosiacList()
{
return collect(Storage::files($this->getMosaicPath()))
->filter(fn($file) => Str::endsWith($file, '.tif'))
->sortByDesc(function ($file) {
$parts = explode('_', str_replace('.tif', '', $file));
$week = $parts[1];
$year = $parts[2];
return $year.sprintf('%02d', $week);
})
->values();
}
public static function getAllDatesOfWeeksInYear($year, $weekNumber): Collection
{
$startOfWeek = Carbon::now()->setISODate($year, $weekNumber)->startOfWeek();
$dates = collect([]);
for ($day = 0; $day < 7; $day++) {
$dates->push((clone $startOfWeek)->addDays($day)->toDateString());
}
return $dates;
}
public function allMergedTiffsPresent(Collection $haystack, Collection $needles)
{
$needlesNotInHaystack = $needles->filter(function ($needle) use ($haystack) {
return !$haystack->contains(function ($item) use ($needle) {
return str_contains($item, $needle);
});
});
if ($needlesNotInHaystack->count() > 0) {
$message = sprintf(
'Missing merged tiffs: %s',
$needlesNotInHaystack->implode(', ')
);
throw new \Exception($message);
}
return true;
}
public function getMergedTiffList()
{
return collect([]);;
}
}

7
laravel_app/app/Models/ProjectReport.php
View file

@ -29,7 +29,12 @@ public function project()
public function weeksAgo()
{
return now()->diffInWeeks(now()->setISODate($this->year, $this->week));
return $this->weeksAgoForYearAndWeek($this->year, $this->week);
}
public static function weeksAgoForYearAndWeek($year, $week)
{
return now()->diffInWeeks(now()->setISODate($year, $week));
}
public function getFileName()

36
laravel_app/app/Rules/AllMergedTiffsPresentRule.php Normal file
View file

@ -0,0 +1,36 @@
<?php
namespace App\Rules;
use Illuminate\Contracts\Validation\Rule;
use App\Models\Project;
use App\Models\ProjectReport;
class AllMergedTiffsPresentRule implements Rule
{
protected $project;
protected $errorMessage = '';
public function __construct(Project $project)
{
$this->project = $project;
}
public function passes($attribute, $value)
{
try {
return $this->project->allMergedTiffsPresent(
$this->project->getMergedTiffList(),
Project::getAllDatesOfWeeksInYear($value['year'], $value['week'])
);
} catch (\Exception $e) {
$this->errorMessage = $e->getMessage();
return false;
}
}
public function message()
{
return $this->errorMessage;
}
}

51
laravel_app/resources/views/components/mosaic-create-modal.blade.php Normal file
View file

@ -0,0 +1,51 @@
@props([
'formData',
/** @var \App\Livewire\Projects\MosaicManager */
'manager'
])
<x-modal wire:model.live="showCreateModal" {{ $attributes }} >
<x-form-modal submit="saveMosiac" wire:loading.class="opacity-50">
<x-slot name="title">
{{ __('Project') }}
</x-slot>
<x-slot name="description">
{{ __('Report generator for generating reports') }}
</x-slot>
<x-slot name="form">
<div class="col-span-6 sm:col-span-4">
<x-label for="year" value="{{ __('Year') }}"/>
<x-input id="year" type="text" class="mt-1 block w-full" wire:model.live="formData.year" autofocus/>
<x-input-error for="formData.year" class="mt-2"/>
</div><div class="col-span-6 sm:col-span-4">
<x-label for="week" value="{{ __('Week') }}"/>
<x-input id="week" type="text" class="mt-1 block w-full" wire:model.live="formData.week" autofocus/>
<x-input-error for="formData.week" class="mt-2"/>
<x-input-error for="formData" class="mt-2"/>
</div>
<div>
<span class="whitespace-nowrap">{!! $manager->dateRange !!}</span>
</div>
</x-slot>
<x-slot name="actions">
<x-action-message class="mr-3" on="saved">
{{ __('Saved.') }}
</x-action-message>
<x-secondary-button class="mr-3"
type="button"
x-on:click="$wire.showCreateModal = false"
>
{{ __('Cancel') }}
</x-secondary-button>
<x-button wire:loading.disabled>
{{ __('Save') }}
</x-button>
</x-slot>
</x-form-modal>
</x-modal>

52
laravel_app/resources/views/livewire/projects/mosaic-manager.blade.php Normal file
View file

@ -0,0 +1,52 @@
<div>
<div class="px-4 sm:px-6 lg:px-8">
<div class="sm:flex sm:items-center">
<div class="sm:flex-auto">
<h1 class="text-base font-semibold leading-6 text-gray-900">Reports</h1>
<p class="mt-2 text-sm text-gray-700"></p>
</div>
<div class="mt-4 sm:ml-16 sm:mt-0 sm:flex-none">
<x-button wire:click="openCreateMosiacsModal"
class="block rounded-md bg-indigo-600 px-3 py-2 text-center text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600">
Create Mosaic
</x-button>
</div>
</div>
<div class="mt-8 flow-root">
<div class="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div class="inline-block min-w-full py-2 align-middle">
<table class="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th scope="col"
class="py-3.5 pl-4 pr-3 text-left text-sm font-semibold text-gray-900 sm:pl-6 lg:pl-8">
Name
</th>
<th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">
Status
</th>
<th scope="col" class="relative py-3.5 pl-3 pr-4 sm:pr-6 lg:pr-8">
<span class="sr-only">Edit</span>
</th>
</tr>
</thead>
<tbody class="divide-y divide-gray-200 bg-white">
@foreach($project->getMosaicList() as $mosaic)
<tr>
<td class="whitespace-nowrap py-4 pl-4 pr-3 text-sm font-medium text-gray-900 sm:pl-6 lg:pl-8">{{ $project->name }}</td>
<td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{{ $mosaic }}
</td>
<td class="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-6 lg:pr-8">
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
<x-mosaic-create-modal :manager="$this"/>
</div>

2
laravel_app/resources/views/livewire/projects/report-manager.blade.php
View file

@ -1,6 +1,4 @@
<div>
<div class="px-4 sm:px-6 lg:px-8">
<div class="sm:flex sm:items-center">
<div class="sm:flex-auto">

9
laravel_app/resources/views/projects/show.blade.php
View file

@ -12,6 +12,11 @@
:class="$tab.isSelected ? 'border-gray-200 bg-white' : 'border-transparent'"
class="inline-flex rounded-l-md border-t border-l border-b px-5 py-2.5"
>{{ __('Downloads') }}</button>
<button x-tabs:tab type="button"
data-tab-name="mosaics"
:class="$tab.isSelected ? 'border-gray-200 bg-white' : 'border-transparent'"
class="inline-flex rounded-l-md border-t border-l border-b px-5 py-2.5"
>{{ __('Mosaic') }}</button>
<button x-tabs:tab type="button"
data-tab-name="mailing"
:class="$tab.isSelected ? 'border-gray-200 bg-white' : 'border-transparent'"
@ -28,7 +33,9 @@ class="inline-flex rounded-l-md border-t border-l border-b px-5 py-2.5"
<section x-tabs:panel class="p-8">
<livewire:projects.download-manager :project="$project" />
</section>
<section x-tabs:panel class="p-8">
<livewire:projects.mosaic-manager :project="$project" />
</section>
<section x-tabs:panel class="p-8">
<livewire:projects.mailing-manager :project="$project" />
</section>

94
laravel_app/tests/Unit/Rules/AllMergedTiffsPresentRuleTest.php Normal file
View file

@ -0,0 +1,94 @@
<?php
namespace Tests\Unit\Rules;
use App\Models\Project;
use App\Rules\AllMergedTiffsPresentRule;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Mockery;
use Tests\TestCase;
class AllMergedTiffsPresentRuleTest extends TestCase
{
use RefreshDatabase;
protected function setUp(): void
{
parent::setUp(); // TODO: Change the autogenerated stub
}
/**
* @test
*/
public function validatesCorrectValue()
{
$project = Project::create([
'name' => 'Test project',
'mail_template' => 'Test mail template',
'mail_subject' => 'Test mail subject',
'mail_frequency' => 'Test mail frequency',
'mail_day' => 'Friday',
'download_path' => 'test_project',
]);
$projectMock = Mockery::mock($project)->makePartial();
$projectMock->shouldReceive('getMergedTiffList')->andReturn(
collect([
"chemba/merged_tiff/2021-01-04.tif",
"chemba/merged_tiff/2021-01-05.tif",
"chemba/merged_tiff/2021-01-06.tif",
"chemba/merged_tiff/2021-01-07.tif",
"chemba/merged_tiff/2021-01-08.tif",
"chemba/merged_tiff/2021-01-09.tif",
"chemba/merged_tiff/2021-01-10.tif",
]));
$rule = new AllMergedTiffsPresentRule($projectMock);
$this->assertTrue($rule->passes(
'attribute',
['year' => '2021', 'week' => '1']
));
}
public function testInvalidatesIncorrectValue()
{
$project = Project::create([
'name' => 'Test project',
'mail_template' => 'Test mail template',
'mail_subject' => 'Test mail subject',
'mail_frequency' => 'Test mail frequency',
'mail_day' => 'Friday',
'download_path' => 'test_project',
]);
$projectMock = Mockery::mock($project)->makePartial();
$projectMock->shouldReceive('getMergedTiffList')->andReturn(
collect([
// "chemba/merged_tiff/2021-01-04.tif",
"chemba/merged_tiff/2021-01-05.tif",
"chemba/merged_tiff/2021-01-06.tif",
"chemba/merged_tiff/2021-01-07.tif",
"chemba/merged_tiff/2021-01-08.tif",
"chemba/merged_tiff/2021-01-09.tif",
"chemba/merged_tiff/2021-01-10.tif",
]));
$rule = new AllMergedTiffsPresentRule($projectMock);
$this->assertFalse($rule->passes(
'attribute',
['year' => '2021', 'week' => '1']
));
$this->assertEquals(
'Missing merged tiffs: 2021-01-04',
$rule->message()
);
}
}

2
python_app/Chemba_download.ipynb
View file

@ -335,6 +335,8 @@
" # Adjust the number of days needed\n",
"\n",
"end = datetime.date.today()\n",
"#end = datetime.datetime(2023, 11, 10)\n",
"#start = datetime.datetime(2023, 10, 19)\n",
"start = end - datetime.timedelta(days=days_needed - 1)\n",
"\n",
"slots = [(start + datetime.timedelta(days=i)).strftime('%Y-%m-%d') for i in range(days_needed)]\n",

20
r_app/2_CI_data_prep.R
View file

@ -11,7 +11,21 @@ library(exactextractr)
library(CIprep)
# Vang alle command line argumenten op
args <- commandArgs(trailingOnly = TRUE)
# Controleer of er ten minste één argument is doorgegeven
if (length(args) == 0) {
stop("Geen argumenten doorgegeven aan het script")
}
# Converteer het eerste argument naar een numerieke waarde
weeks_ago <- as.numeric(args[1])
# Controleer of weeks_ago een geldig getal is
if (is.na(weeks_ago)) {
stop("Het argument is geen geldig getal")
}
laravel_storage_dir <- here("../laravel_app/storage/app")
#preparing directories
@ -38,13 +52,13 @@ dir.create(merged_final)
# Creating weekly mosaic
dates <- date_list(0)
dates <- date_list(weeks_ago)
head(dates)
print(planet_tif_folder)
#load pivot geojson
pivot_sf_q <- st_read(here( "pivot_20210625.geojson")) %>% dplyr::select(pivot, pivot_quadrant) %>% vect()
#load and filter raster files
raster_files <- list.files(planet_tif_folder,full.names = T, pattern = ".tif") #use pattern = '.tif$' or something else if you have multiple files in this folder
filtered_files <- map(dates$days_filter, ~ raster_files[grepl(pattern = .x, x = raster_files)]) %>%
compact() %>%
flatten_chr()

18
r_app/CI_report_dashboard_planet.Rmd
View file

@ -2,6 +2,8 @@
# title: paste0("CI report week ", week, " - all pivots from ", last_tuesday, " to ", today)
params:
ref: word-styles-reference-03.docx
output_file: "CI_report.docx"
report_date: "2023-12-12"
output:
word_document:
reference_docx: !expr file.path("word-styles-reference-03.docx")
@ -11,6 +13,12 @@ editor_options:
---
```{r setup, include=FALSE}
#set de filename van de output
knitr::opts_chunk$set(echo = TRUE)
output_file <- params$output_file
report_date <- params$report_date
# Activeer de renv omgeving
renv::activate()
@ -50,12 +58,13 @@ s2_dir <- "C:/Users/timon/Resilience BV/4002 CMD App - General/4002 CMD Team/400
```{r week, message=FALSE, warning=FALSE, include=FALSE}
# week <- 5
# today = "2023-10-08"
# week <- week(today)
#today = "2023-12-12"
today <- as.character(report_date)
week <- week(today)
#today = "2022-08-18"
today = as.character(Sys.Date())
week = lubridate::week(Sys.time())
#today = as.character(Sys.Date())
#week = lubridate::week(Sys.time())
## week = 26
title_var <- paste0("CI dashboard week ", week, " - all pivots dashboard using 3x3 meter resolution")
```
@ -97,7 +106,6 @@ last_week_dif_raster <- ((CI - CI_m1) / CI_m1) * 100
last_week_dif_raster_abs <- (CI - CI_m1)
two_week_dif_raster_abs <- (CI - CI_m2)
AllPivots0 <-st_read(here(data_dir, "pivot_20210625.geojson"))
joined_spans <-st_read(here(data_dir, "spans2.geojson")) %>% st_transform(crs(AllPivots0))

BIN
r_app/Rplots.pdf
View file

Binary file not shown.

7
renv/.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
library/
local/
cellar/
lock/
python/
sandbox/
staging/

1180
renv/activate.R Normal file
View file

File diff suppressed because it is too large Load diff

6
runcane.sh
View file

@ -1,4 +1,7 @@
#!/bin/bash
output_file=${1:-"standaard_naam.html"}
ref=${2:-"word-styles-reference-03.docx"}
# Directory waar de virtuele omgeving zal worden aangemaakt
#VENV_DIR="./python_app/myenv"
@ -18,5 +21,6 @@ Rscript 2_CI_data_prep.R
#
## Runnen van Rmd bestand
# -e betekent "evalueren" en -i specificeert de input file
Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', 'all')"
#Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', 'all')"
Rscript -e "rmarkdown::render('CI_report_dashboard_planet.Rmd', output_file='$output_file', params=list(ref='$ref'))"
cd ..

2
runpython.sh
View file

@ -1,7 +1,7 @@
#!/bin/bash
# Standaardwaarde voor days
days=28
days=7
# Loop door alle argumenten
for arg in "$@"