570 lines
20 KiB
PHP
570 lines
20 KiB
PHP
<?php
|
|
|
|
namespace App\Models;
|
|
|
|
use App\Jobs\ProjectDownloadRDSJob;
|
|
use App\Jobs\ProjectDownloadTiffJob;
|
|
use App\Jobs\ProjectInterpolateGrowthModelJob;
|
|
use App\Jobs\ProjectMosiacGeneratorJob;
|
|
use App\Jobs\ProjectReportGeneratorJob;
|
|
use Carbon\CarbonPeriod;
|
|
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Carbon\Carbon;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\Bus;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Illuminate\Support\Str;
|
|
use Maatwebsite\Excel\Facades\Excel;
|
|
use PhpOffice\PhpSpreadsheet\Shared\Date as SharedDate;
|
|
|
|
class Project extends Model
|
|
{
|
|
use HasFactory;
|
|
|
|
protected $casts = [
|
|
'min_harvest_date' => 'date'
|
|
];
|
|
|
|
protected $fillable = [
|
|
'name',
|
|
'mail_template',
|
|
'mail_subject',
|
|
'mail_frequency',
|
|
'mail_day',
|
|
'mail_scheduled',
|
|
'download_path',
|
|
'pivot_json_path',
|
|
'span_json_path',
|
|
'harvest_json_path',
|
|
'min_harvest_date',
|
|
'borders',
|
|
];
|
|
|
|
public static function saveWithFormData(mixed $formData)
|
|
{
|
|
$uniqueIdentifier = ['id' => $formData['id'] ?? null];
|
|
/**
|
|
* @var Project $project
|
|
*/
|
|
logger($formData);
|
|
$project = Project::updateOrCreate($uniqueIdentifier, $formData);
|
|
$baseFrom = 'livewire-tmp/';
|
|
$baseTo = $project->download_path.'/Data/';
|
|
if ($formData['pivot_file']) {
|
|
Storage::copy($baseFrom.$formData['pivot_file']['tmpFilename'], $baseTo.'pivot.geojson');
|
|
$project->update(['pivot_json_path' => $baseTo.'pivot.geojson']);
|
|
}
|
|
if ($formData['span_file']) {
|
|
Storage::copy($baseFrom.$formData['span_file']['tmpFilename'], $baseTo.'span.geojson');
|
|
$project->update(['span_json_path' => $baseTo.'span.geojson']);
|
|
}
|
|
if ($formData['harvest_file']) {
|
|
Storage::copy($baseFrom.$formData['harvest_file']['tmpFilename'],
|
|
$baseTo.'harvest.'.$formData['harvest_file']['extension']);
|
|
if ($project->update(['harvest_json_path' => $baseTo.'harvest.'.$formData['harvest_file']['extension']])) {
|
|
$project->setMinHarvestDate();
|
|
}
|
|
}
|
|
self::upsertMailRecipients($formData, $project);
|
|
}
|
|
|
|
private static function upsertMailRecipients($formData, Project $project)
|
|
{
|
|
$mailRecipientsData = array_map(function ($mailRecipient) use ($project) {
|
|
$mailRecipient['project_id'] = $project->id;
|
|
unset($mailRecipient['created_at']);
|
|
unset($mailRecipient['updated_at']);
|
|
$mailRecipient['id'] ??= null;
|
|
return $mailRecipient;
|
|
}, $formData['mail_recipients'] ?? []);
|
|
ProjectEmailRecipient::upsert(
|
|
$mailRecipientsData,
|
|
['id', 'project_id'],
|
|
['name', 'email',]
|
|
);
|
|
}
|
|
|
|
public function getMosaicPath()
|
|
{
|
|
return sprintf('%s/%s', $this->download_path, 'weekly_mosaic');
|
|
}
|
|
|
|
public function getMosaicList(): Collection
|
|
{
|
|
return collect(Storage::files($this->getMosaicPath()))
|
|
->filter(fn($filename) => Str::endsWith($filename, '.tif'))
|
|
->values();
|
|
}
|
|
|
|
protected static function boot()
|
|
{
|
|
parent::boot(); // TODO: Change the autogenerated stub
|
|
|
|
static::deleting(function ($project) {
|
|
$project->emailRecipients()->delete();
|
|
$project->mailings()->each(function ($mailing) {
|
|
$mailing->attachments()->delete();
|
|
$mailing->recipients()->delete();
|
|
});
|
|
$project->mailings()->delete();
|
|
});
|
|
}
|
|
|
|
public function reports()
|
|
{
|
|
return $this->hasMany(ProjectReport::class);
|
|
}
|
|
|
|
public function getAttachmentPathAttribute()
|
|
{
|
|
return storage_path(sprintf('%s/attachments', $this->download_path));
|
|
|
|
return '/storage/'.$this->download_path.'/attachments';
|
|
}
|
|
|
|
public function emailRecipients()
|
|
{
|
|
return $this->hasMany(ProjectEmailRecipient::class);
|
|
}
|
|
|
|
public function mailings()
|
|
{
|
|
return $this->hasMany(ProjectMailing::class);
|
|
}
|
|
|
|
public function downloads(): \Illuminate\Database\Eloquent\Relations\HasMany
|
|
{
|
|
return $this->hasMany(ProjectDownload::class);
|
|
}
|
|
|
|
public function nonFailedDownloads(): \Illuminate\Database\Eloquent\Relations\HasMany
|
|
{
|
|
return $this->hasMany(ProjectDownload::class)->where('status', '<>', 'failed');
|
|
}
|
|
|
|
public function mosaics(): \Illuminate\Database\Eloquent\Relations\HasMany
|
|
{
|
|
return $this->hasMany(ProjectMosaic::class);
|
|
}
|
|
|
|
|
|
public function allMosaicsPresent(Carbon $endDate): bool
|
|
{
|
|
// end date is in the future
|
|
if ($endDate->isFuture()) {
|
|
throw new \Exception('Mosaic can\'t be generated for the future. Change the end date.');
|
|
}
|
|
$mosaicsNotPresentInFilesystem = $this->getMissingMosaicsInFileSystem($endDate);
|
|
if ($mosaicsNotPresentInFilesystem->count() === 0) {
|
|
return true;
|
|
}
|
|
$message = sprintf(
|
|
'Missing mosaics: %s',
|
|
$mosaicsNotPresentInFilesystem->implode(', ')
|
|
);
|
|
throw new \Exception($message);
|
|
}
|
|
|
|
public function getMissingMosaicsInFileSystem(Carbon $endDate)
|
|
{
|
|
return collect($this->getMosaicFilenameListByEndDate($endDate))
|
|
->filter(function ($filename) {
|
|
return !$this->getMosaicList()->contains(function ($mosaicFilename) use ($filename) {
|
|
return Str::endsWith($mosaicFilename, substr($filename, -16));
|
|
});
|
|
});
|
|
}
|
|
|
|
public static function getMosaicFilenameListByEndDate(Carbon $endDate, int $offset = 7): \Generator
|
|
{
|
|
for ($i = 0; $i < 4; $i++) {
|
|
yield (ProjectMosaic::getFilenameByPeriod($endDate->copy()->subDays($offset * $i), $offset));
|
|
}
|
|
}
|
|
|
|
public function getMosiacList()
|
|
{
|
|
return collect(Storage::files($this->getMosaicPath()))
|
|
->filter(fn($file) => Str::endsWith($file, '.tif'))
|
|
->sortByDesc(function ($file) {
|
|
$parts = explode('_', str_replace('.tif', '', $file));
|
|
$week = $parts[1];
|
|
$year = $parts[2];
|
|
return $year.sprintf('%02d', $week);
|
|
})
|
|
->values();
|
|
}
|
|
|
|
public static function getAllDatesOfWeeksInYear($year, $weekNumber): Collection
|
|
{
|
|
$startOfWeek = Carbon::now()->setISODate($year, $weekNumber)->startOfWeek();
|
|
$dates = collect([]);
|
|
|
|
for ($day = 0; $day < 7; $day++) {
|
|
$dates->push((clone $startOfWeek)->addDays($day)->toDateString());
|
|
}
|
|
|
|
return $dates;
|
|
}
|
|
|
|
public function allMergedTiffsPresent(Collection $haystack, Collection $needles)
|
|
{
|
|
$needlesNotInHaystack = $needles->filter(function ($needle) use ($haystack) {
|
|
return !$haystack->contains(function ($item) use ($needle) {
|
|
return str_contains($item, $needle);
|
|
});
|
|
});
|
|
|
|
if ($needlesNotInHaystack->count() > 0) {
|
|
$message = sprintf(
|
|
'Missing merged tiffs: %s',
|
|
$needlesNotInHaystack->implode(', ')
|
|
);
|
|
|
|
throw new \Exception($message);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public function getMergedTiffList()
|
|
{
|
|
return collect(Storage::files($this->download_path.'/merged_final_tif'))
|
|
->filter(fn($file) => Str::endsWith($file, '.tif'))
|
|
->sortByDesc(function ($file) {
|
|
$parts = explode('_', str_replace('.tif', '', $file));
|
|
$date = $parts[1];
|
|
return $date;
|
|
})
|
|
->values();
|
|
}
|
|
|
|
public function hasPendingDownload(): bool
|
|
{
|
|
return $this->downloads()->statusPending()->count() > 0;
|
|
}
|
|
|
|
public function hasPendingReport(): bool
|
|
{
|
|
return $this->reports()->statusPending()->count() > 0;
|
|
}
|
|
|
|
public function hasPendingMosaic(): bool
|
|
{
|
|
return $this->mosaics()->statusPending()->count() > 0;
|
|
}
|
|
|
|
public function startDownload(Carbon $date)
|
|
{
|
|
$downloadRequest = $this->downloads()->updateOrCreate(
|
|
[
|
|
'project_id' => $this->id, // of een andere manier om project_id te bepalen
|
|
'name' => sprintf('%s.tif', $date->format('Y-m-d')),
|
|
],
|
|
[
|
|
'path' => sprintf('%s/%s/%s.tif', $this->download_path, 'merged_final_tif', $date->format('Y-m-d')),
|
|
]
|
|
);
|
|
ProjectDownloadTiffJob::dispatch($downloadRequest, $date);
|
|
}
|
|
|
|
public function schedule()
|
|
{
|
|
//TODO check the ranges.
|
|
$this->scheduleReport();
|
|
}
|
|
|
|
public function shouldSchedule(): bool
|
|
{
|
|
if (strtolower($this->mail_day) === strtolower(now()->englishDayOfWeek)) {
|
|
return strtolower($this->mail_frequency) === 'weekly' || now()->day <= 7;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
|
|
public function hasInvalidMosaicFor(Carbon $endDate, int $offset): bool
|
|
{
|
|
// parameters : $
|
|
// check if the mail day happens the day before mosaic -> good
|
|
$dayOfWeekIso = Carbon::parse($this->mail_day)->dayOfWeekIso;
|
|
$min_updated_at_date = $endDate->copy()
|
|
->startOfWeek()
|
|
->addDays($dayOfWeekIso - 1)
|
|
->format('Y-m-d');
|
|
|
|
return $this->mosaics()
|
|
->where('updated_at', '>=', $min_updated_at_date)
|
|
->statusSuccess()
|
|
->where(['end_date' => $endDate, 'offset' => $offset])
|
|
->exists();
|
|
|
|
}
|
|
|
|
public function scheduleReport(?Carbon $endDate = null, ?int $offset = null)
|
|
{
|
|
if ($endDate?->isFuture() || $endDate?->isToday() || $offset <= 0) {
|
|
logger('EndDate is today or in the future.');
|
|
$endDate = null;
|
|
$offset = null;
|
|
}
|
|
|
|
$endDate ??= Carbon::yesterday();
|
|
$offset ??= 7;
|
|
|
|
for ($step = 0; $step<=3; $step++) {
|
|
$latestMosaicToDelete = ProjectMosaic::projectMosaicNameFormat($endDate->clone(), $offset*$step);
|
|
logger('Deleting mosaic: '.$latestMosaicToDelete);
|
|
$this->mosaics()->where('name', $latestMosaicToDelete)->first()?->delete();
|
|
}
|
|
|
|
logger('Scheduling report for '.$endDate->format('d-m-Y').' with offset '.$offset.' days');
|
|
Bus::chain([
|
|
Bus::batch($this->getFileDownloadsFor($endDate->clone(), $offset)),
|
|
Bus::batch($this->getMosaicsFor($endDate->clone(), $offset)),
|
|
Bus::batch(
|
|
[
|
|
new ProjectInterpolateGrowthModelJob($this),
|
|
$this->getReportFor($endDate->clone(), $offset, true)
|
|
]),
|
|
])
|
|
->dispatch();
|
|
return "done";
|
|
}
|
|
|
|
public function scheduleTestReport()
|
|
{
|
|
$endDate = Carbon::yesterday();
|
|
$offset = 7;
|
|
$this->mail_day = $endDate->dayName;
|
|
|
|
for ($step = 0; $step<=3; $step++) {
|
|
$latestMosaicToDelete = ProjectMosaic::projectMosaicNameFormat($endDate->clone(), $offset*$step);
|
|
logger('Deleting mosaic: '.$latestMosaicToDelete);
|
|
$this->mosaics()->where('name', $latestMosaicToDelete)->first()?->delete();
|
|
}
|
|
|
|
logger('Scheduling test report for '.$endDate->format('d-m-Y').' with offset '.$offset.' days');
|
|
Bus::chain([
|
|
Bus::batch($this->getFileDownloadsFor($endDate->clone(), $offset)),
|
|
Bus::batch($this->getTestMosaicsFor($endDate->clone())),
|
|
Bus::batch(
|
|
[
|
|
new ProjectInterpolateGrowthModelJob($this),
|
|
$this->getTestReportFor($endDate->clone(), $offset, true)
|
|
]),
|
|
])
|
|
->dispatch();
|
|
return "done";
|
|
}
|
|
|
|
public function getReportFor(Carbon $endDate, int $offset = 7, $sendMail = false): ProjectReportGeneratorJob
|
|
{
|
|
$report = $this->reports()->create([
|
|
'name' => 'Report of the '.$endDate->format('d-m-Y').' from the past '.$offset.' days',
|
|
'end_date' => $endDate,
|
|
'offset' => $offset,
|
|
'path' => 'reports/'.ProjectReport::getFileName($endDate, $offset).'.docx',
|
|
]);
|
|
|
|
return (new ProjectReportGeneratorJob($report, $sendMail));
|
|
}
|
|
|
|
public function getTestReportFor(Carbon $endDate, int $offset = 7, $sendMail = false): ProjectReportGeneratorJob
|
|
{
|
|
$report = $this->reports()->create([
|
|
'name' => 'Test Report of the '.$endDate->format('d-m-Y').' from the past '.$offset.' days',
|
|
'end_date' => $endDate,
|
|
'offset' => $offset,
|
|
'path' => 'reports/'.ProjectReport::getFileName($endDate, $offset).'.docx',
|
|
]);
|
|
|
|
return (new ProjectReportGeneratorJob($report, $sendMail, true));
|
|
}
|
|
|
|
public function getFileDownloadsFor(Carbon $endDate, int $offset = 7): array
|
|
{
|
|
$startOfRange = (clone $endDate)->subdays(4 * $offset - 1);
|
|
$dateRange = CarbonPeriod::create($startOfRange, $endDate);
|
|
|
|
return collect($dateRange)
|
|
->map(fn($date) => ProjectDownloadTiffJob::handleForDate($this, $date))
|
|
->filter()
|
|
->toArray();
|
|
}
|
|
|
|
public function getMosaicsFor(Carbon $endDate, int $offset = 7): array
|
|
{
|
|
return collect(range(0, 3))
|
|
->map(function () use ($endDate, $offset) {
|
|
$currentEndDate = $endDate->clone();
|
|
if (!$currentEndDate->isDayOfWeek($this->mail_day)) {
|
|
$endDate->previous($this->mail_day);
|
|
}
|
|
$endDate->subDay();
|
|
$calculatedOffSet = (int) $endDate->clone()->diffInDays($currentEndDate);
|
|
|
|
return ProjectMosiacGeneratorJob::handleFor($this, $currentEndDate, $calculatedOffSet);
|
|
})
|
|
->filter()
|
|
->toArray();
|
|
}
|
|
|
|
public function getTestMosaicsFor(Carbon $endDate)
|
|
{
|
|
return collect(range(0, 3))
|
|
->map(function () use ($endDate) {
|
|
$currentEndDate = $endDate->clone();
|
|
$endDate->subWeek();
|
|
|
|
return ProjectMosiacGeneratorJob::handleFor($this, $currentEndDate, 7);
|
|
})
|
|
->filter()
|
|
->toArray();
|
|
}
|
|
|
|
public function handleMissingDownloads()
|
|
{
|
|
$this->getMissingDownloads()
|
|
->each(function (Carbon $date) {
|
|
dispatch(ProjectDownloadTiffJob::handleForDate($this, $date));
|
|
});
|
|
}
|
|
|
|
public function hasMissingDownloadsDateInHarvestFile(): bool
|
|
{
|
|
return $this->getMissingDownloads()->count() > 0;
|
|
}
|
|
|
|
public function getMissingDownloads(): Collection
|
|
{
|
|
if (!$this->min_harvest_date) {
|
|
return collect([]);
|
|
}
|
|
$harvest_dates = $this->nonFailedDownloads()->get()->map(fn($d) => $d->date);
|
|
$all_dates = collect(CarbonPeriod::create($this->min_harvest_date, '1 day', now())->toArray());
|
|
return $all_dates->diff($harvest_dates);
|
|
}
|
|
|
|
public function setMinHarvestDate(): bool
|
|
{
|
|
if (!$this->harvest_json_path) {
|
|
return false;
|
|
}
|
|
return $this->update(['min_harvest_date' => $this->getMinimumDateFromHarvestExcelFile()->format('Y-m-d')]);
|
|
}
|
|
|
|
private function getMinimumDateFromHarvestExcelFile(): Carbon
|
|
{
|
|
$value = Storage::disk('local')->path($this->harvest_json_path);
|
|
$data = Excel::toCollection(new \App\Imports\ExcelFileImport(), $value);
|
|
$season_start_index = $data->first()->first()->search('season_start');
|
|
return collect($data->first()->slice(1))->reduce(function ($carry, $value, $key) use ($season_start_index) {
|
|
return min($carry, Carbon::instance(SharedDate::excelToDateTimeObject($value[$season_start_index])));
|
|
}, now());
|
|
}
|
|
|
|
public function newDownloadsUploaded()
|
|
{
|
|
// $this->createDownloadRecordsInDatabaseAndUpdateStatusForAllDownloadedImages();
|
|
$date = Carbon::parse('2023-02-09');
|
|
$now = Carbon::now();
|
|
|
|
$offset = (int) $date->diffInDays($now);
|
|
dispatch_sync(new ProjectDownloadRDSJob($this, Carbon::yesterday(), $offset));
|
|
dispatch_sync(new ProjectInterpolateGrowthModelJob($this));
|
|
}
|
|
|
|
public function getRdsAsDownload()
|
|
{
|
|
|
|
$path = $this->download_path.'/Data/extracted_ci/cumulative_vals/combined_CI_data.rds';
|
|
|
|
return Storage::download(
|
|
$path, 'combined_CI_data.rds'
|
|
);
|
|
}
|
|
|
|
public function getTifsAsZip(Carbon $startDate, Carbon $endDate)
|
|
{
|
|
$path = $this->download_path.'/merged_final_tif';
|
|
$files = collect(Storage::files($path))
|
|
->filter(fn($file) => Str::endsWith($file, '.tif'))
|
|
->filter(function ($file) use ($startDate, $endDate) {
|
|
$dateString = str_replace('.tif', '', basename($file));
|
|
$date = Carbon::parse($dateString);
|
|
return $date->between($startDate, $endDate);
|
|
});
|
|
logger(__CLASS__.'::'.__METHOD__.'::'.__LINE__);
|
|
logger($files);
|
|
|
|
return $this->createZipArchiveAndReturn($files);
|
|
}
|
|
|
|
public function getMosaicsAsZip(Carbon $startDate, Carbon $endDate)
|
|
{
|
|
$path = $this->download_path.'/weekly_mosaic';
|
|
|
|
// create a collection of all week numbers and years for given date range
|
|
$allowedFiles = collect(CarbonPeriod::create($startDate, $endDate)->toArray())
|
|
->map(fn($date) => sprintf('week_%s_%s.tif', $date->weekOfYear, $date->year))
|
|
->unique();
|
|
|
|
|
|
$files = collect(Storage::files($path))
|
|
->filter(fn($file) => Str::endsWith($file, '.tif'))
|
|
->filter(function ($file) use ($allowedFiles) {
|
|
return $allowedFiles->contains(basename($file));
|
|
});
|
|
putenv('TMPDIR='.storage_path('app'));
|
|
|
|
return $this->createZipArchiveAndReturn($files);
|
|
}
|
|
|
|
private function createZipArchiveAndReturn(Collection $files)
|
|
{
|
|
$zipPath = storage_path('app/'.$this->download_path.'/download.zip');
|
|
$zip = new \ZipArchive();
|
|
$zip->open($zipPath, \ZipArchive::CREATE | \ZipArchive::OVERWRITE);
|
|
$files->each(function ($file) use ($zip) {
|
|
$zip->addFile(storage_path('app/'.$file), basename($file));
|
|
});
|
|
$zip->close();
|
|
|
|
return response()->download(
|
|
$zipPath, 'download.zip'
|
|
);
|
|
}
|
|
|
|
public function createDownloadRecordsInDatabaseAndUpdateStatusForAllDownloadedImages(): void
|
|
{
|
|
$merged_tiffs = $this->getMergedTiffList();
|
|
$this->downloads->each(function ($download) use ($merged_tiffs) {
|
|
if ($merged_tiffs->contains($download->path) && $download->status !== 'success') {
|
|
$download->setStatusSuccess();
|
|
}
|
|
});
|
|
|
|
$merged_tiffs->each(function ($path) {
|
|
if ($this->downloads()->where('path', $path)->count() === 0) {
|
|
$this->downloads()->create([
|
|
'path' => $path,
|
|
'status' => 'success',
|
|
'name' => explode('/', $path)[count(explode('/', $path)) - 1]
|
|
]);
|
|
}
|
|
});
|
|
}
|
|
|
|
public function createAllMosaicsInDatabaseAndUpdateStatusForAllDownloadedImages()
|
|
{
|
|
$list_of_desired_mosaics = $this->getMergedTiffList()
|
|
->map(fn($file) => str_replace('.tif', '', basename($file)))
|
|
->map(function ($date) {
|
|
$carbon = Carbon::parse($date);
|
|
return sprintf('week_%s_%s', $carbon->format('W'), $carbon->year);
|
|
})
|
|
->unique();
|
|
}
|
|
}
|