changing paths CS-109
This commit is contained in:
parent
6efa6b6b05
commit
c4ef10f44f
|
|
@ -42,60 +42,12 @@
|
|||
library(terra)
|
||||
library(sf)
|
||||
|
||||
# ============================================================================
|
||||
# HELPER FUNCTIONS (DEFINE FIRST)
|
||||
# ============================================================================
|
||||
# ==============================================================================
|
||||
# LOAD CENTRALIZED PARAMETERS & PATHS
|
||||
# ==============================================================================
|
||||
source(here::here("r_app", "parameters_project.R"))
|
||||
|
||||
smartcane_log <- function(msg) {
|
||||
cat(paste0("[", Sys.time(), "] ", msg, "\n"))
|
||||
}
|
||||
|
||||
# Load field boundaries from GeoJSON
|
||||
load_field_boundaries <- function(geojson_path) {
|
||||
smartcane_log(paste("Loading field boundaries from:", geojson_path))
|
||||
|
||||
if (!file.exists(geojson_path)) {
|
||||
stop("GeoJSON file not found:", geojson_path)
|
||||
}
|
||||
|
||||
fields <- st_read(geojson_path, quiet = TRUE)
|
||||
|
||||
# Standardize field name property
|
||||
if (!"field_name" %in% names(fields)) {
|
||||
if ("field" %in% names(fields)) {
|
||||
fields$field_name <- fields$field
|
||||
} else if ("FIELD_ID" %in% names(fields)) {
|
||||
fields$field_name <- fields$FIELD_ID
|
||||
} else if ("Name" %in% names(fields)) {
|
||||
fields$field_name <- fields$Name
|
||||
} else {
|
||||
# Default: use first non-geometry column
|
||||
field_col <- names(fields)[!names(fields) %in% c("geometry", "geom")]
|
||||
if (length(field_col) > 0) {
|
||||
fields$field_name <- fields[[field_col[1]]]
|
||||
} else {
|
||||
stop("No suitable field name column found in GeoJSON")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# FIX: Validate and repair geometries (handles duplicate vertices, degenerate edges, etc)
|
||||
invalid_count <- sum(!st_is_valid(fields))
|
||||
if (invalid_count > 0) {
|
||||
smartcane_log(paste("WARNING: Found", invalid_count, "invalid geometry/geometries - attempting repair"))
|
||||
fields <- st_make_valid(fields)
|
||||
smartcane_log(paste("Repaired invalid geometries using st_make_valid()"))
|
||||
}
|
||||
|
||||
smartcane_log(paste("Loaded", nrow(fields), "field(s)"))
|
||||
return(fields)
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PROJECT SETUP
|
||||
# ============================================================================
|
||||
|
||||
# Get project parameter
|
||||
# Get project parameter from command line
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
if (length(args) == 0) {
|
||||
PROJECT <- "angata"
|
||||
|
|
@ -103,13 +55,12 @@ if (length(args) == 0) {
|
|||
PROJECT <- args[1]
|
||||
}
|
||||
|
||||
# Construct paths directly (avoid complex parameter initialization)
|
||||
base_path <- file.path(getwd(), "laravel_app", "storage", "app", PROJECT)
|
||||
data_dir <- file.path(base_path, "Data")
|
||||
# Load centralized path structure (creates all directories automatically)
|
||||
paths <- setup_project_directories(PROJECT)
|
||||
|
||||
smartcane_log(paste("Project:", PROJECT))
|
||||
smartcane_log(paste("Base path:", base_path))
|
||||
smartcane_log(paste("Data dir:", data_dir))
|
||||
smartcane_log(paste("Base path:", paths$laravel_storage_dir))
|
||||
smartcane_log(paste("Data dir:", paths$data_dir))
|
||||
|
||||
# Unified function to crop TIFF to field boundaries
|
||||
# Called by both migration and processing phases
|
||||
|
|
@ -267,24 +218,21 @@ process_new_merged_tif <- function(merged_tif_dir, field_tiles_dir, fields, fiel
|
|||
}
|
||||
|
||||
# ============================================================================
|
||||
# ==============================================================================
|
||||
# MAIN EXECUTION
|
||||
# ============================================================================
|
||||
# ==============================================================================
|
||||
|
||||
smartcane_log("========================================")
|
||||
smartcane_log(paste("Script 10: Per-Field TIFF Creation for", PROJECT))
|
||||
smartcane_log("========================================")
|
||||
|
||||
# Create necessary directories
|
||||
dir.create(data_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
# Load field boundaries using centralized path (no dir.create needed - already created by setup_project_directories)
|
||||
fields <- load_field_boundaries(paths$field_boundaries_path)
|
||||
|
||||
# Load field boundaries
|
||||
geojson_path <- file.path(data_dir, "pivot.geojson")
|
||||
fields <- load_field_boundaries(geojson_path)
|
||||
|
||||
# Define input and output directories
|
||||
merged_tif_dir <- file.path(base_path, "merged_tif")
|
||||
field_tiles_dir <- file.path(base_path, "field_tiles")
|
||||
field_tiles_ci_dir <- file.path(base_path, "field_tiles_CI")
|
||||
# Define input and output directories (from centralized paths)
|
||||
merged_tif_dir <- paths$merged_tif_folder
|
||||
field_tiles_dir <- paths$field_tiles_dir
|
||||
field_tiles_ci_dir <- paths$field_tiles_ci_dir
|
||||
|
||||
# PHASE 1: Process new downloads (always runs)
|
||||
# Pass field_tiles_ci_dir so it can skip dates already migrated
|
||||
|
|
|
|||
|
|
@ -111,6 +111,9 @@ main <- function() {
|
|||
stop(e)
|
||||
})
|
||||
|
||||
# Load centralized path structure (creates all directories automatically)
|
||||
paths <- setup_project_directories(project_dir)
|
||||
|
||||
cat("[DEBUG] Attempting to source r_app/20_ci_extraction_utils.R\n")
|
||||
tryCatch({
|
||||
source("r_app/20_ci_extraction_utils.R")
|
||||
|
|
@ -193,8 +196,8 @@ main <- function() {
|
|||
# -----------------------------------
|
||||
log_message("Searching for raster files")
|
||||
|
||||
# Check if tiles exist (Script 01 output) - detect grid size dynamically
|
||||
tiles_split_base <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split")
|
||||
# Check if tiles exist (Script 10 output) - detect grid size dynamically using centralized paths
|
||||
tiles_split_base <- paths$daily_tiles_split_dir
|
||||
|
||||
# Detect grid size from daily_tiles_split folder structure
|
||||
# Expected structure: daily_tiles_split/5x5/ or daily_tiles_split/10x10/ etc.
|
||||
|
|
@ -293,7 +296,7 @@ main <- function() {
|
|||
log_message(paste("Combining all", length(all_daily_files), "daily CI files into combined_CI_data.rds"))
|
||||
|
||||
# Load and combine ALL daily files (creates complete dataset)
|
||||
combined_ci_path <- file.path(cumulative_CI_vals_dir, "combined_CI_data.rds")
|
||||
combined_ci_path <- file.path(paths$cumulative_ci_vals_dir, "combined_CI_data.rds")
|
||||
|
||||
combined_data <- all_daily_files %>%
|
||||
purrr::map(readRDS) %>%
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ main <- function() {
|
|||
|
||||
cat(sprintf("Converting CI RDS to CSV: project=%s\n", project_dir))
|
||||
|
||||
# Initialize project configuration
|
||||
# Initialize project configuration and centralized paths
|
||||
tryCatch({
|
||||
source("parameters_project.R")
|
||||
}, error = function(e) {
|
||||
|
|
@ -152,15 +152,12 @@ main <- function() {
|
|||
})
|
||||
})
|
||||
|
||||
# Define paths
|
||||
ci_data_source_dir <- here::here("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "cumulative_vals")
|
||||
ci_data_output_dir <- here::here("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "ci_data_for_python")
|
||||
# Load centralized path structure (creates all directories automatically)
|
||||
paths <- setup_project_directories(project_dir)
|
||||
|
||||
# Create output directory if it doesn't exist (for new projects)
|
||||
if (!dir.exists(ci_data_output_dir)) {
|
||||
dir.create(ci_data_output_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
cat(sprintf("✓ Created output directory: %s\n", ci_data_output_dir))
|
||||
}
|
||||
# Use centralized paths (no need for dir.create - already handled)
|
||||
ci_data_source_dir <- paths$cumulative_ci_vals_dir
|
||||
ci_data_output_dir <- paths$ci_for_python_dir
|
||||
|
||||
input_file <- file.path(ci_data_source_dir, "combined_CI_data.rds")
|
||||
output_file <- file.path(ci_data_output_dir, "ci_data_for_python.csv")
|
||||
|
|
|
|||
|
|
@ -92,32 +92,21 @@ main <- function() {
|
|||
# IMPORTANT: Only consider a folder as valid if it contains actual files
|
||||
laravel_storage <- here::here("laravel_app/storage/app", project_dir)
|
||||
|
||||
# If data_source was explicitly provided from pipeline, validate it; otherwise auto-detect
|
||||
if (!is.null(data_source_from_args)) {
|
||||
# Use the provided data_source, but verify it has data
|
||||
proposed_path <- file.path(laravel_storage, data_source_from_args)
|
||||
has_data <- dir.exists(proposed_path) && length(list.files(proposed_path, pattern = "\\.tif$")) > 0
|
||||
|
||||
if (has_data) {
|
||||
data_source <- data_source_from_args
|
||||
message("✓ Using provided data source '", data_source, "' - contains files")
|
||||
} else {
|
||||
message("WARNING: Provided data source '", data_source_from_args, "' is empty or doesn't exist. Auto-detecting...")
|
||||
data_source_from_args <- NULL # Fall through to auto-detection
|
||||
}
|
||||
}
|
||||
|
||||
# Auto-detect if no valid data_source was provided
|
||||
if (is.null(data_source_from_args)) {
|
||||
# Check merged_tif_8b - only if it exists AND contains files
|
||||
merged_tif_8b_path <- file.path(laravel_storage, "merged_tif_8b")
|
||||
has_8b_data <- dir.exists(merged_tif_8b_path) && length(list.files(merged_tif_8b_path, pattern = "\\.tif$")) > 0
|
||||
|
||||
# Check merged_tif - only if it exists AND contains files
|
||||
merged_tif_path <- file.path(laravel_storage, "merged_tif")
|
||||
has_legacy_data <- dir.exists(merged_tif_path) && length(list.files(merged_tif_path, pattern = "\\.tif$")) > 0
|
||||
|
||||
# Select data source based on what has actual data
|
||||
# Load centralized path structure
|
||||
tryCatch({
|
||||
source("r_app/parameters_project.R")
|
||||
paths <- setup_project_directories(project_dir)
|
||||
}, error = function(e) {
|
||||
message("Note: Could not open files from r_app directory")
|
||||
message("Attempting to source from default directory instead...")
|
||||
tryCatch({
|
||||
source("parameters_project.R")
|
||||
paths <- setup_project_directories(project_dir)
|
||||
message("✓ Successfully sourced files from default directory")
|
||||
}, error = function(e) {
|
||||
stop("Failed to source required files from both 'r_app' and default directories.")
|
||||
})
|
||||
})
|
||||
data_source <- if (has_8b_data) {
|
||||
message("Auto-detected data source: merged_tif_8b (8-band optimized) - contains files")
|
||||
"merged_tif_8b"
|
||||
|
|
@ -142,27 +131,18 @@ main <- function() {
|
|||
message("Attempting to source from default directory instead...")
|
||||
tryCatch({
|
||||
source("parameters_project.R")
|
||||
source("40_mosaic_creation_utils.R")
|
||||
paths <- setup_project_directories(project_dir)
|
||||
message("✓ Successfully sourced files from default directory")
|
||||
}, error = function(e) {
|
||||
stop("Failed to source required files from both 'r_app' and default directories.")
|
||||
})
|
||||
})
|
||||
|
||||
# Extract path variables from global environment (set by parameters_project.R)
|
||||
merged_final <- if (exists("merged_final", envir = .GlobalEnv)) {
|
||||
get("merged_final", envir = .GlobalEnv)
|
||||
} else {
|
||||
file.path(laravel_storage, "merged_final_tif")
|
||||
}
|
||||
# Use centralized paths (no need to manually construct or create dirs)
|
||||
merged_final <- paths$growth_model_interpolated_dir # or merged_final_tif if needed
|
||||
daily_vrt <- paths$vrt_dir
|
||||
|
||||
daily_vrt <- if (exists("daily_vrt", envir = .GlobalEnv)) {
|
||||
get("daily_vrt", envir = .GlobalEnv)
|
||||
} else {
|
||||
file.path(laravel_storage, "Data", "vrt")
|
||||
}
|
||||
|
||||
safe_log(paste("Using merged_final_tif directory:", merged_final))
|
||||
safe_log(paste("Using growth model/mosaic directory:", merged_final))
|
||||
safe_log(paste("Using daily VRT directory:", daily_vrt))
|
||||
|
||||
# 4. Generate date range for processing
|
||||
|
|
@ -216,10 +196,11 @@ main <- function() {
|
|||
# Point to the grid-specific merged_final_tif directory
|
||||
merged_final_with_grid <- file.path(merged_final_base, grid_size)
|
||||
|
||||
# Set output directory for per-tile mosaics, organized by grid size
|
||||
# Set output directory for per-tile mosaics, organized by grid size (from centralized paths)
|
||||
# Output: weekly_tile_max/{grid_size}/week_WW_YYYY_TT.tif
|
||||
tile_output_base <- file.path(laravel_storage, "weekly_tile_max", grid_size)
|
||||
dir.create(tile_output_base, recursive = TRUE, showWarnings = FALSE)
|
||||
tile_output_base <- file.path(paths$weekly_tile_max_dir, grid_size)
|
||||
# Note: no dir.create needed - paths$weekly_tile_max_dir already created by setup_project_directories()
|
||||
dir.create(tile_output_base, recursive = TRUE, showWarnings = FALSE) # Create grid-size subfolder
|
||||
|
||||
created_tile_files <- create_weekly_mosaic_from_tiles(
|
||||
dates = dates,
|
||||
|
|
@ -242,8 +223,8 @@ main <- function() {
|
|||
tryCatch({
|
||||
safe_log("Starting single-file mosaic creation (backward-compatible approach)...")
|
||||
|
||||
# Set output directory for single-file mosaics
|
||||
single_file_output_dir <- file.path(laravel_storage, "weekly_mosaic")
|
||||
# Set output directory for single-file mosaics (from centralized paths)
|
||||
single_file_output_dir <- paths$weekly_mosaic_dir
|
||||
|
||||
created_file <- create_weekly_mosaic(
|
||||
dates = dates,
|
||||
|
|
|
|||
|
|
@ -251,14 +251,10 @@ main <- function() {
|
|||
message("KPI Calculations:", paste(client_config$kpi_calculations, collapse = ", "))
|
||||
message("Output Formats:", paste(client_config$outputs, collapse = ", "))
|
||||
|
||||
# Define paths for mosaic detection (used in PHASE 1)
|
||||
# NEW: Support both per-field and legacy single-file mosaics
|
||||
base_project_path <- file.path("laravel_app", "storage", "app", project_dir)
|
||||
weekly_tile_max <- file.path(base_project_path, "weekly_tile_max")
|
||||
weekly_mosaic <- file.path(base_project_path, "weekly_mosaic") # NEW: Per-field structure
|
||||
|
||||
# Also set up per-field daily RDS path for Script 80 historical data loading
|
||||
daily_vals_dir <- file.path(base_project_path, "Data", "extracted_ci", "daily_vals")
|
||||
# Use centralized paths from setup object (no need for file.path calls)
|
||||
weekly_tile_max <- setup$weekly_tile_max_dir
|
||||
weekly_mosaic <- setup$weekly_mosaic_dir
|
||||
daily_vals_dir <- setup$daily_ci_vals_dir
|
||||
|
||||
tryCatch({
|
||||
source(here("r_app", "30_growth_model_utils.R"))
|
||||
|
|
@ -283,11 +279,8 @@ main <- function() {
|
|||
stop("Error loading 80_kpi_utils.R: ", e$message)
|
||||
})
|
||||
|
||||
# Prepare inputs for KPI calculation
|
||||
reports_dir_kpi <- file.path(base_project_path, "reports", "kpis")
|
||||
if (!dir.exists(reports_dir_kpi)) {
|
||||
dir.create(reports_dir_kpi, recursive = TRUE)
|
||||
}
|
||||
# Prepare inputs for KPI calculation (already created by setup_project_directories)
|
||||
reports_dir_kpi <- setup$kpi_reports_dir
|
||||
|
||||
cumulative_CI_vals_dir <- setup$cumulative_CI_vals_dir
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,9 @@ tryCatch({
|
|||
stop("Error loading parameters_project.R: ", e$message)
|
||||
})
|
||||
|
||||
# Load centralized paths
|
||||
paths <- setup_project_directories(project_dir)
|
||||
|
||||
# Log initial configuration
|
||||
safe_log("Starting the R Markdown script with KPIs")
|
||||
safe_log(paste("mail_day params:", params$mail_day))
|
||||
|
|
@ -115,8 +118,8 @@ safe_log(paste("mail_day variable:", mail_day))
|
|||
|
||||
```{r load_kpi_data, message=FALSE, warning=FALSE, include=FALSE}
|
||||
## SIMPLE KPI LOADING - robust lookup with fallbacks
|
||||
# Primary expected directory inside the laravel storage
|
||||
kpi_data_dir <- file.path("..", "laravel_app", "storage", "app", project_dir, "reports", "kpis")
|
||||
# Primary expected directory from centralized paths
|
||||
kpi_data_dir <- paths$kpi_reports_dir
|
||||
date_suffix <- format(as.Date(report_date), "%Y%m%d")
|
||||
|
||||
# Calculate current week from report_date using ISO 8601 week numbering
|
||||
|
|
|
|||
|
|
@ -105,6 +105,9 @@ tryCatch({
|
|||
stop("Error loading parameters_project.R: ", e$message)
|
||||
})
|
||||
|
||||
# Load centralized paths
|
||||
paths <- setup_project_directories(project_dir)
|
||||
|
||||
# Log initial configuration
|
||||
safe_log("Starting the R Markdown script with KPIs")
|
||||
safe_log(paste("mail_day params:", params$mail_day))
|
||||
|
|
@ -120,8 +123,8 @@ cat("\n=== DEBUG: R Markdown Working Directory ===\n")
|
|||
cat(paste("getwd():", getwd(), "\n"))
|
||||
cat(paste("Expected knit_dir from R Markdown:", knitr::opts_knit$get("root.dir"), "\n\n"))
|
||||
|
||||
# Primary expected directory inside the laravel storage
|
||||
kpi_data_dir <- file.path("..", "laravel_app", "storage", "app", project_dir, "reports", "kpis")
|
||||
# Primary expected directory from centralized paths
|
||||
kpi_data_dir <- paths$kpi_reports_dir
|
||||
date_suffix <- format(as.Date(report_date), "%Y%m%d")
|
||||
|
||||
# Calculate current week from report_date using ISO 8601 week numbering
|
||||
|
|
|
|||
|
|
@ -210,71 +210,228 @@ detect_tile_structure_from_merged_final <- function(merged_final_tif_dir, daily_
|
|||
|
||||
# 4. Define project directory structure
|
||||
# -----------------------------------
|
||||
# ==============================================================================
|
||||
# CENTRALIZED PATH MANAGEMENT - setup_project_directories()
|
||||
# ==============================================================================
|
||||
# This function is the single source of truth for ALL file paths used across the pipeline.
|
||||
# All scripts should call this function once at startup and use returned paths.
|
||||
# This eliminates ~88 hardcoded file.path() calls scattered across 8 scripts.
|
||||
#
|
||||
# USAGE:
|
||||
# paths <- setup_project_directories(project_dir)
|
||||
# merged_tif_dir <- paths$merged_tif_folder
|
||||
# daily_ci_dir <- paths$daily_ci_vals_dir
|
||||
# kpi_output_dir <- paths$kpi_reports_dir
|
||||
#
|
||||
# TIERS (8-layer directory structure):
|
||||
# Tier 1: Raw data (merged_tif)
|
||||
# Tier 2: Per-field TIFFs (field_tiles, field_tiles_CI)
|
||||
# Tier 3: CI Extraction (daily_ci_vals, cumulative_ci_vals)
|
||||
# Tier 4: Growth Model (growth_model_interpolated)
|
||||
# Tier 5: Mosaics (weekly_mosaic, weekly_tile_max)
|
||||
# Tier 6: KPI & Reporting (kpi_reports_dir, kpi_field_stats_dir)
|
||||
# Tier 7: Support (data, vrt, harvest, logs)
|
||||
# Tier 8: Config & Metadata (field_boundaries_path, tiling_config_path)
|
||||
#
|
||||
# BENEFITS:
|
||||
# ✓ Single source of truth (eliminates ~88 hardcoded file.path() calls)
|
||||
# ✓ Auto-creates all directories (no scattered dir.create() calls)
|
||||
# ✓ Easy to update storage structure globally
|
||||
# ✓ Consistent naming across all 8 scripts
|
||||
# ==============================================================================
|
||||
setup_project_directories <- function(project_dir, data_source = "merged_tif") {
|
||||
# Base directories
|
||||
# ===========================================================================
|
||||
# BASE DIRECTORIES (Foundation for all paths)
|
||||
# ===========================================================================
|
||||
laravel_storage_dir <- here("laravel_app", "storage", "app", project_dir)
|
||||
|
||||
# Use standard merged_tif directory for all projects
|
||||
merged_tif_folder <- here(laravel_storage_dir, "merged_tif")
|
||||
# ===========================================================================
|
||||
# TIER 1: RAW DATA & INPUT PATHS (Script 00 - Python download output)
|
||||
# ===========================================================================
|
||||
merged_tif_folder <- here(laravel_storage_dir, "merged_tif") # 4-band raw GeoTIFFs from Planet
|
||||
|
||||
# Detect tile mode based on file patterns
|
||||
# ===========================================================================
|
||||
# TIER 2: TILING PATHS (Script 10 - Per-field tiff creation)
|
||||
# ===========================================================================
|
||||
# Per-field TIFF structure: field_tiles/{FIELD_NAME}/{YYYY-MM-DD}.tif
|
||||
field_tiles_dir <- here(laravel_storage_dir, "field_tiles")
|
||||
|
||||
# Per-field CI TIFFs (pre-computed, used by Script 40): field_tiles_CI/{FIELD_NAME}/{YYYY-MM-DD}.tif
|
||||
field_tiles_ci_dir <- here(laravel_storage_dir, "field_tiles_CI")
|
||||
|
||||
# Legacy tiling (for backward compatibility): daily_tiles_split/{grid_size}/{YYYY-MM-DD}/{YYYY-MM-DD}_XX.tif
|
||||
daily_tiles_split_dir <- here(laravel_storage_dir, "daily_tiles_split")
|
||||
|
||||
# Simplified: only check daily_tiles_split for per-field structure
|
||||
use_tile_mosaic <- dir.exists(daily_tiles_split_dir) && length(list.dirs(daily_tiles_split_dir, full.names = FALSE, recursive = FALSE)) > 0
|
||||
# ===========================================================================
|
||||
# TIER 3: CI EXTRACTION PATHS (Script 20 - Canopy Index calculation)
|
||||
# ===========================================================================
|
||||
extracted_ci_base_dir <- here(laravel_storage_dir, "Data", "extracted_ci")
|
||||
|
||||
# Main subdirectories
|
||||
dirs <- list(
|
||||
reports = here(laravel_storage_dir, "reports"),
|
||||
logs = here(laravel_storage_dir, "logs"),
|
||||
data = here(laravel_storage_dir, "Data"),
|
||||
tif = list(
|
||||
merged = merged_tif_folder
|
||||
),
|
||||
# New per-field directory structure (Script 10 output)
|
||||
field_tiles = here(laravel_storage_dir, "field_tiles"),
|
||||
field_tiles_ci = here(laravel_storage_dir, "field_tiles_CI"),
|
||||
weekly_mosaic = here(laravel_storage_dir, "weekly_mosaic"),
|
||||
extracted_ci = list(
|
||||
base = here(laravel_storage_dir, "Data", "extracted_ci"),
|
||||
daily = here(laravel_storage_dir, "Data", "extracted_ci", "daily_vals"),
|
||||
cumulative = here(laravel_storage_dir, "Data", "extracted_ci", "cumulative_vals"),
|
||||
# New per-field daily RDS structure (Script 20 output)
|
||||
daily_per_field = here(laravel_storage_dir, "Data", "extracted_ci", "daily_vals")
|
||||
),
|
||||
vrt = here(laravel_storage_dir, "Data", "vrt"),
|
||||
harvest = here(laravel_storage_dir, "Data", "HarvestData")
|
||||
# Daily CI values (cumulative RDS): combined_CI_data.rds
|
||||
daily_ci_vals_dir <- here(extracted_ci_base_dir, "daily_vals")
|
||||
|
||||
# Cumulative CI across time: All_pivots_Cumulative_CI_quadrant_year_v2.rds
|
||||
cumulative_ci_vals_dir <- here(extracted_ci_base_dir, "cumulative_vals")
|
||||
|
||||
# Per-field CI data for Python harvest prediction (Script 21): ci_data_for_python.csv
|
||||
ci_for_python_dir <- here(extracted_ci_base_dir, "ci_data_for_python")
|
||||
|
||||
# ===========================================================================
|
||||
# TIER 4: GROWTH MODEL PATHS (Script 30 - Interpolation & smoothing)
|
||||
# ===========================================================================
|
||||
growth_model_interpolated_dir <- here(laravel_storage_dir, "growth_model_interpolated")
|
||||
|
||||
# ===========================================================================
|
||||
# TIER 5: MOSAIC PATHS (Script 40 - Weekly mosaics)
|
||||
# ===========================================================================
|
||||
# Per-field weekly mosaics (per-field architecture): weekly_mosaic/{FIELD}/{week_XX_YYYY}.tif
|
||||
weekly_mosaic_dir <- here(laravel_storage_dir, "weekly_mosaic")
|
||||
|
||||
# Tile-based weekly max (legacy): weekly_tile_max/{grid_size}/week_XX_YYYY.tif
|
||||
weekly_tile_max_dir <- here(laravel_storage_dir, "weekly_tile_max")
|
||||
|
||||
# ===========================================================================
|
||||
# TIER 6: KPI & REPORTING PATHS (Scripts 80, 90, 91)
|
||||
# ===========================================================================
|
||||
reports_dir <- here(laravel_storage_dir, "reports")
|
||||
kpi_reports_dir <- here(reports_dir, "kpis") # Where Script 80 outputs KPI CSV/RDS files
|
||||
kpi_field_stats_dir <- here(kpi_reports_dir, "field_stats") # Per-field KPI details
|
||||
kpi_field_analysis_dir <- here(kpi_reports_dir, "field_analysis") # Field-level analysis for Script 91
|
||||
|
||||
# ===========================================================================
|
||||
# TIER 7: SUPPORT PATHS (Data, VRT, Harvest)
|
||||
# ===========================================================================
|
||||
data_dir <- here(laravel_storage_dir, "Data")
|
||||
vrt_dir <- here(data_dir, "vrt") # Virtual Raster files created during CI extraction
|
||||
harvest_dir <- here(data_dir, "HarvestData") # Harvest schedule data
|
||||
log_dir <- here(laravel_storage_dir, "logs") # Log files
|
||||
|
||||
# ===========================================================================
|
||||
# TIER 8: CONFIG & METADATA PATHS
|
||||
# ===========================================================================
|
||||
# Field boundaries GeoJSON (same across all scripts)
|
||||
field_boundaries_path <- here(data_dir, "pivot.geojson")
|
||||
|
||||
# Tiling configuration metadata from Script 10
|
||||
tiling_config_path <- here(daily_tiles_split_dir, "tiling_config.json")
|
||||
|
||||
# ===========================================================================
|
||||
# CREATE ALL DIRECTORIES (once per pipeline run)
|
||||
# ===========================================================================
|
||||
all_dirs <- c(
|
||||
# Tier 1
|
||||
merged_tif_folder,
|
||||
# Tier 2
|
||||
field_tiles_dir, field_tiles_ci_dir, daily_tiles_split_dir,
|
||||
# Tier 3
|
||||
extracted_ci_base_dir, daily_ci_vals_dir, cumulative_ci_vals_dir, ci_for_python_dir,
|
||||
# Tier 4
|
||||
growth_model_interpolated_dir,
|
||||
# Tier 5
|
||||
weekly_mosaic_dir, weekly_tile_max_dir,
|
||||
# Tier 6
|
||||
reports_dir, kpi_reports_dir, kpi_field_stats_dir, kpi_field_analysis_dir,
|
||||
# Tier 7
|
||||
data_dir, vrt_dir, harvest_dir, log_dir
|
||||
)
|
||||
|
||||
# Create all directories
|
||||
for (dir_path in unlist(dirs)) {
|
||||
for (dir_path in all_dirs) {
|
||||
dir.create(dir_path, showWarnings = FALSE, recursive = TRUE)
|
||||
}
|
||||
|
||||
# Return directory structure for use in other functions
|
||||
# ===========================================================================
|
||||
# RETURN COMPREHENSIVE PATH LIST
|
||||
# Scripts should source parameters_project.R and receive paths object like:
|
||||
# paths <- setup_project_directories(project_dir)
|
||||
# Then use: paths$merged_tif_folder, paths$daily_ci_vals_dir, etc.
|
||||
# ===========================================================================
|
||||
return(list(
|
||||
# PROJECT ROOT
|
||||
laravel_storage_dir = laravel_storage_dir,
|
||||
reports_dir = dirs$reports,
|
||||
log_dir = dirs$logs,
|
||||
data_dir = dirs$data,
|
||||
planet_tif_folder = dirs$tif$merged,
|
||||
merged_final = dirs$tif$final,
|
||||
daily_CI_vals_dir = dirs$extracted_ci$daily,
|
||||
cumulative_CI_vals_dir = dirs$extracted_ci$cumulative,
|
||||
# New per-field directory paths (Script 10 & 20 outputs)
|
||||
field_tiles_dir = dirs$field_tiles,
|
||||
field_tiles_ci_dir = dirs$field_tiles_ci,
|
||||
daily_vals_per_field_dir = dirs$extracted_ci$daily_per_field,
|
||||
# Field boundaries path for all scripts
|
||||
field_boundaries_path = here(laravel_storage_dir, "Data", "pivot.geojson"),
|
||||
weekly_CI_mosaic = dirs$weekly_mosaic, # Per-field weekly mosaics (per-field architecture)
|
||||
daily_vrt = dirs$vrt, # Point to Data/vrt folder where R creates VRT files from CI extraction
|
||||
use_tile_mosaic = use_tile_mosaic, # Flag indicating if tiles are used for this project
|
||||
harvest_dir = dirs$harvest,
|
||||
extracted_CI_dir = dirs$extracted_ci$base
|
||||
|
||||
# TIER 1: Raw data
|
||||
merged_tif_folder = merged_tif_folder,
|
||||
|
||||
# TIER 2: Per-field TIFFs
|
||||
field_tiles_dir = field_tiles_dir,
|
||||
field_tiles_ci_dir = field_tiles_ci_dir,
|
||||
daily_tiles_split_dir = daily_tiles_split_dir,
|
||||
|
||||
# TIER 3: CI Extraction
|
||||
extracted_ci_base_dir = extracted_ci_base_dir,
|
||||
daily_ci_vals_dir = daily_ci_vals_dir,
|
||||
cumulative_ci_vals_dir = cumulative_ci_vals_dir,
|
||||
ci_for_python_dir = ci_for_python_dir,
|
||||
|
||||
# TIER 4: Growth Model
|
||||
growth_model_interpolated_dir = growth_model_interpolated_dir,
|
||||
|
||||
# TIER 5: Mosaics
|
||||
weekly_mosaic_dir = weekly_mosaic_dir,
|
||||
weekly_tile_max_dir = weekly_tile_max_dir,
|
||||
|
||||
# TIER 6: KPI & Reporting
|
||||
reports_dir = reports_dir,
|
||||
kpi_reports_dir = kpi_reports_dir,
|
||||
kpi_field_stats_dir = kpi_field_stats_dir,
|
||||
kpi_field_analysis_dir = kpi_field_analysis_dir,
|
||||
|
||||
# TIER 7: Support
|
||||
data_dir = data_dir,
|
||||
vrt_dir = vrt_dir,
|
||||
harvest_dir = harvest_dir,
|
||||
log_dir = log_dir,
|
||||
|
||||
# TIER 8: Config & Metadata
|
||||
field_boundaries_path = field_boundaries_path,
|
||||
tiling_config_path = tiling_config_path
|
||||
))
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# TIER-BY-TIER PATH REFERENCE (for setup_project_directories output)
|
||||
# ==============================================================================
|
||||
#
|
||||
# TIER 1: RAW DATA (Script 00 - Python download)
|
||||
# paths$merged_tif_folder
|
||||
# └─ {YYYY-MM-DD}.tif (4-band uint16 GeoTIFFs from Planet API)
|
||||
#
|
||||
# TIER 2: PER-FIELD TIFFS (Script 10)
|
||||
# paths$field_tiles_dir/{FIELD_NAME}/{YYYY-MM-DD}.tif
|
||||
# paths$field_tiles_ci_dir/{FIELD_NAME}/{YYYY-MM-DD}.tif
|
||||
# paths$daily_tiles_split_dir/{grid_size}/{YYYY-MM-DD}/{YYYY-MM-DD}_XX.tif (legacy)
|
||||
#
|
||||
# TIER 3: CI EXTRACTION (Script 20)
|
||||
# paths$daily_ci_vals_dir/combined_CI_data.rds
|
||||
# paths$cumulative_ci_vals_dir/All_pivots_Cumulative_CI_quadrant_year_v2.rds
|
||||
# paths$ci_for_python_dir/ci_data_for_python.csv (Script 21 output)
|
||||
#
|
||||
# TIER 4: GROWTH MODEL (Script 30)
|
||||
# paths$growth_model_interpolated_dir/ (RDS files with interpolated CI)
|
||||
#
|
||||
# TIER 5: MOSAICS (Script 40)
|
||||
# paths$weekly_mosaic_dir/{FIELD_NAME}/week_XX_YYYY.tif
|
||||
# paths$weekly_tile_max_dir/{grid_size}/week_XX_YYYY_00.tif (legacy)
|
||||
#
|
||||
# TIER 6: KPI & REPORTING (Scripts 80, 90, 91)
|
||||
# paths$kpi_reports_dir/ (KPI outputs from Script 80)
|
||||
# paths$kpi_field_stats_dir/ (Per-field KPI RDS)
|
||||
# paths$kpi_field_analysis_dir/ (Analysis RDS for Script 91)
|
||||
# paths$reports_dir/ (Word/HTML reports)
|
||||
#
|
||||
# TIER 7: SUPPORT (Various scripts)
|
||||
# paths$data_dir/pivot.geojson (Field boundaries)
|
||||
# paths$data_dir/harvest.xlsx (Harvest schedule)
|
||||
# paths$vrt_dir/ (Virtual raster files)
|
||||
# paths$harvest_dir/ (Harvest predictions from Python)
|
||||
# paths$log_dir/ (Pipeline logs)
|
||||
#
|
||||
# TIER 8: CONFIG & METADATA
|
||||
# paths$field_boundaries_path (Full path to pivot.geojson)
|
||||
# paths$tiling_config_path (Metadata from Script 10)
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
#set working dir.
|
||||
# 5. Load field boundaries
|
||||
# ----------------------
|
||||
|
|
|
|||
|
|
@ -39,8 +39,9 @@ force_rerun <- FALSE # Set to TRUE to force all scripts to run even if outputs e
|
|||
# Define Rscript path for running external R scripts via system()
|
||||
RSCRIPT_PATH <- file.path("C:", "Program Files", "R", "R-4.4.3", "bin", "x64", "Rscript.exe")
|
||||
|
||||
# Load client type mapping from parameters_project.R
|
||||
# Load client type mapping and centralized paths from parameters_project.R
|
||||
source("r_app/parameters_project.R")
|
||||
paths <- setup_project_directories(project_dir)
|
||||
client_type <- get_client_type(project_dir)
|
||||
cat(sprintf("\nProject: %s → Client Type: %s\n", project_dir, client_type))
|
||||
|
||||
|
|
@ -105,7 +106,7 @@ for (i in 1:nrow(weeks_needed)) {
|
|||
files_this_week <- list.files(mosaic_dir_check, pattern = week_pattern_check, recursive = TRUE, full.names = FALSE)
|
||||
}
|
||||
} else if (mosaic_mode == "single-file") {
|
||||
mosaic_dir_check <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic")
|
||||
mosaic_dir_check <- paths$weekly_mosaic_dir
|
||||
if (dir.exists(mosaic_dir_check)) {
|
||||
# NEW: Support per-field architecture - search recursively for mosaics in field subdirectories
|
||||
# Check both top-level (legacy) and field subdirectories (per-field architecture)
|
||||
|
|
@ -222,7 +223,7 @@ cat("\n========== CHECKING EXISTING OUTPUTS ==========\n")
|
|||
cat(sprintf("Auto-detected mosaic mode: %s\n", mosaic_mode))
|
||||
|
||||
# Check Script 10 outputs - FLEXIBLE: look for tiles either directly OR in grid subdirs
|
||||
tiles_split_base <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split")
|
||||
tiles_split_base <- paths$daily_tiles_split_dir
|
||||
tiles_dates <- c()
|
||||
if (dir.exists(tiles_split_base)) {
|
||||
# Try grid-size subdirectories first (5x5, 10x10, etc.) - preferred new structure
|
||||
|
|
@ -241,7 +242,7 @@ if (dir.exists(tiles_split_base)) {
|
|||
cat(sprintf("Script 10: %d dates already tiled\n", length(tiles_dates)))
|
||||
|
||||
# Check Script 20 outputs (CI extraction) - daily RDS files
|
||||
ci_daily_dir <- file.path("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "daily_vals")
|
||||
ci_daily_dir <- paths$daily_ci_vals_dir
|
||||
ci_files <- if (dir.exists(ci_daily_dir)) {
|
||||
list.files(ci_daily_dir, pattern = "\\.rds$")
|
||||
} else {
|
||||
|
|
@ -301,8 +302,7 @@ tryCatch(
|
|||
# Setup paths
|
||||
# NOTE: All downloads go to merged_tif/ regardless of project
|
||||
# (data_source variable is used later by Script 20 for reading, but downloads always go to merged_tif)
|
||||
base_path <- file.path("laravel_app", "storage", "app", project_dir)
|
||||
merged_tifs_dir <- file.path(base_path, "merged_tif") # Always check merged_tif for downloads
|
||||
merged_tifs_dir <- paths$merged_tif_folder # Always check merged_tif for downloads
|
||||
|
||||
cat(sprintf("[DEBUG] Checking for existing files in: %s\n", merged_tifs_dir))
|
||||
cat(sprintf("[DEBUG] Directory exists: %s\n", dir.exists(merged_tifs_dir)))
|
||||
|
|
@ -404,7 +404,7 @@ if (pipeline_success && !skip_10) {
|
|||
}
|
||||
|
||||
# Verify output - check per-field structure
|
||||
field_tiles_dir <- file.path("laravel_app", "storage", "app", project_dir, "field_tiles")
|
||||
field_tiles_dir <- paths$field_tiles_dir
|
||||
if (dir.exists(field_tiles_dir)) {
|
||||
fields <- list.dirs(field_tiles_dir, full.names = FALSE, recursive = FALSE)
|
||||
fields <- fields[fields != ""]
|
||||
|
|
@ -445,7 +445,7 @@ if (pipeline_success && !skip_20) {
|
|||
}
|
||||
|
||||
# Verify CI output was created
|
||||
ci_daily_dir <- file.path("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "daily_vals")
|
||||
ci_daily_dir <- paths$daily_ci_vals_dir
|
||||
if (dir.exists(ci_daily_dir)) {
|
||||
files <- list.files(ci_daily_dir, pattern = "\\.rds$")
|
||||
cat(sprintf("✓ Script 20 completed - generated %d CI files\n", length(files)))
|
||||
|
|
@ -478,7 +478,7 @@ if (pipeline_success && !skip_21) {
|
|||
main() # Call main() to execute the script with the environment variables
|
||||
|
||||
# Verify CSV output was created
|
||||
ci_csv_path <- file.path("laravel_app", "storage", "app", project_dir, "ci_extracted")
|
||||
ci_csv_path <- paths$ci_for_python_dir
|
||||
if (dir.exists(ci_csv_path)) {
|
||||
csv_files <- list.files(ci_csv_path, pattern = "\\.csv$")
|
||||
cat(sprintf("✓ Script 21 completed - converted to %d CSV files\n", length(csv_files)))
|
||||
|
|
@ -517,7 +517,7 @@ if (pipeline_success && !skip_30) {
|
|||
}
|
||||
|
||||
# Verify interpolated output
|
||||
growth_dir <- file.path("laravel_app", "storage", "app", project_dir, "growth_model_interpolated")
|
||||
growth_dir <- paths$growth_model_interpolated_dir
|
||||
if (dir.exists(growth_dir)) {
|
||||
files <- list.files(growth_dir, pattern = "\\.rds$|\\.csv$")
|
||||
cat(sprintf("✓ Script 30 completed - generated %d growth model files\n", length(files)))
|
||||
|
|
@ -619,7 +619,7 @@ if (pipeline_success && !skip_40) {
|
|||
mosaic_created <- length(mosaic_files) > 0
|
||||
}
|
||||
} else {
|
||||
mosaic_dir <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic")
|
||||
mosaic_dir <- paths$weekly_mosaic_dir
|
||||
if (dir.exists(mosaic_dir)) {
|
||||
week_pattern <- sprintf("week_%02d_%d\\.tif", week_num, year_num)
|
||||
# NEW: Support per-field architecture - search recursively for mosaics in field subdirectories
|
||||
|
|
@ -768,12 +768,9 @@ if (pipeline_success && run_legacy_report) {
|
|||
tryCatch(
|
||||
{
|
||||
# Script 90 is an RMarkdown file - compile it with rmarkdown::render()
|
||||
output_dir <- file.path("laravel_app", "storage", "app", project_dir, "reports")
|
||||
output_dir <- paths$reports_dir
|
||||
|
||||
# Ensure output directory exists
|
||||
if (!dir.exists(output_dir)) {
|
||||
dir.create(output_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
}
|
||||
# Reports directory already created by setup_project_directories
|
||||
|
||||
output_filename <- sprintf(
|
||||
"CI_report_week%02d_%d.docx",
|
||||
|
|
@ -817,12 +814,9 @@ if (pipeline_success && run_modern_report) {
|
|||
tryCatch(
|
||||
{
|
||||
# Script 91 is an RMarkdown file - compile it with rmarkdown::render()
|
||||
output_dir <- file.path("laravel_app", "storage", "app", project_dir, "reports")
|
||||
output_dir <- paths$reports_dir
|
||||
|
||||
# Ensure output directory exists
|
||||
if (!dir.exists(output_dir)) {
|
||||
dir.create(output_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
}
|
||||
# Reports directory already created by setup_project_directories
|
||||
|
||||
output_filename <- sprintf(
|
||||
"CI_report_week%02d_%d.docx",
|
||||
|
|
|
|||
Loading…
Reference in a new issue