diff --git a/r_app/10_create_per_field_tiffs.R b/r_app/10_create_per_field_tiffs.R index 5d88573..c82a15f 100644 --- a/r_app/10_create_per_field_tiffs.R +++ b/r_app/10_create_per_field_tiffs.R @@ -42,60 +42,12 @@ library(terra) library(sf) -# ============================================================================ -# HELPER FUNCTIONS (DEFINE FIRST) -# ============================================================================ +# ============================================================================== +# LOAD CENTRALIZED PARAMETERS & PATHS +# ============================================================================== +source(here::here("r_app", "parameters_project.R")) -smartcane_log <- function(msg) { - cat(paste0("[", Sys.time(), "] ", msg, "\n")) -} - -# Load field boundaries from GeoJSON -load_field_boundaries <- function(geojson_path) { - smartcane_log(paste("Loading field boundaries from:", geojson_path)) - - if (!file.exists(geojson_path)) { - stop("GeoJSON file not found:", geojson_path) - } - - fields <- st_read(geojson_path, quiet = TRUE) - - # Standardize field name property - if (!"field_name" %in% names(fields)) { - if ("field" %in% names(fields)) { - fields$field_name <- fields$field - } else if ("FIELD_ID" %in% names(fields)) { - fields$field_name <- fields$FIELD_ID - } else if ("Name" %in% names(fields)) { - fields$field_name <- fields$Name - } else { - # Default: use first non-geometry column - field_col <- names(fields)[!names(fields) %in% c("geometry", "geom")] - if (length(field_col) > 0) { - fields$field_name <- fields[[field_col[1]]] - } else { - stop("No suitable field name column found in GeoJSON") - } - } - } - - # FIX: Validate and repair geometries (handles duplicate vertices, degenerate edges, etc) - invalid_count <- sum(!st_is_valid(fields)) - if (invalid_count > 0) { - smartcane_log(paste("WARNING: Found", invalid_count, "invalid geometry/geometries - attempting repair")) - fields <- st_make_valid(fields) - smartcane_log(paste("Repaired invalid geometries using st_make_valid()")) - } - - smartcane_log(paste("Loaded", nrow(fields), "field(s)")) - return(fields) -} - -# ============================================================================ -# PROJECT SETUP -# ============================================================================ - -# Get project parameter +# Get project parameter from command line args <- commandArgs(trailingOnly = TRUE) if (length(args) == 0) { PROJECT <- "angata" @@ -103,13 +55,12 @@ if (length(args) == 0) { PROJECT <- args[1] } -# Construct paths directly (avoid complex parameter initialization) -base_path <- file.path(getwd(), "laravel_app", "storage", "app", PROJECT) -data_dir <- file.path(base_path, "Data") +# Load centralized path structure (creates all directories automatically) +paths <- setup_project_directories(PROJECT) smartcane_log(paste("Project:", PROJECT)) -smartcane_log(paste("Base path:", base_path)) -smartcane_log(paste("Data dir:", data_dir)) +smartcane_log(paste("Base path:", paths$laravel_storage_dir)) +smartcane_log(paste("Data dir:", paths$data_dir)) # Unified function to crop TIFF to field boundaries # Called by both migration and processing phases @@ -267,24 +218,21 @@ process_new_merged_tif <- function(merged_tif_dir, field_tiles_dir, fields, fiel } # ============================================================================ +# ============================================================================== # MAIN EXECUTION -# ============================================================================ +# ============================================================================== smartcane_log("========================================") smartcane_log(paste("Script 10: Per-Field TIFF Creation for", PROJECT)) smartcane_log("========================================") -# Create necessary directories -dir.create(data_dir, recursive = TRUE, showWarnings = FALSE) +# Load field boundaries using centralized path (no dir.create needed - already created by setup_project_directories) +fields <- load_field_boundaries(paths$field_boundaries_path) -# Load field boundaries -geojson_path <- file.path(data_dir, "pivot.geojson") -fields <- load_field_boundaries(geojson_path) - -# Define input and output directories -merged_tif_dir <- file.path(base_path, "merged_tif") -field_tiles_dir <- file.path(base_path, "field_tiles") -field_tiles_ci_dir <- file.path(base_path, "field_tiles_CI") +# Define input and output directories (from centralized paths) +merged_tif_dir <- paths$merged_tif_folder +field_tiles_dir <- paths$field_tiles_dir +field_tiles_ci_dir <- paths$field_tiles_ci_dir # PHASE 1: Process new downloads (always runs) # Pass field_tiles_ci_dir so it can skip dates already migrated diff --git a/r_app/20_ci_extraction.R b/r_app/20_ci_extraction.R index 36bf5b7..79f7f86 100644 --- a/r_app/20_ci_extraction.R +++ b/r_app/20_ci_extraction.R @@ -111,6 +111,9 @@ main <- function() { stop(e) }) + # Load centralized path structure (creates all directories automatically) + paths <- setup_project_directories(project_dir) + cat("[DEBUG] Attempting to source r_app/20_ci_extraction_utils.R\n") tryCatch({ source("r_app/20_ci_extraction_utils.R") @@ -193,8 +196,8 @@ main <- function() { # ----------------------------------- log_message("Searching for raster files") - # Check if tiles exist (Script 01 output) - detect grid size dynamically - tiles_split_base <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split") + # Check if tiles exist (Script 10 output) - detect grid size dynamically using centralized paths + tiles_split_base <- paths$daily_tiles_split_dir # Detect grid size from daily_tiles_split folder structure # Expected structure: daily_tiles_split/5x5/ or daily_tiles_split/10x10/ etc. @@ -293,7 +296,7 @@ main <- function() { log_message(paste("Combining all", length(all_daily_files), "daily CI files into combined_CI_data.rds")) # Load and combine ALL daily files (creates complete dataset) - combined_ci_path <- file.path(cumulative_CI_vals_dir, "combined_CI_data.rds") + combined_ci_path <- file.path(paths$cumulative_ci_vals_dir, "combined_CI_data.rds") combined_data <- all_daily_files %>% purrr::map(readRDS) %>% diff --git a/r_app/21_convert_ci_rds_to_csv.R b/r_app/21_convert_ci_rds_to_csv.R index f75f6af..be458b6 100644 --- a/r_app/21_convert_ci_rds_to_csv.R +++ b/r_app/21_convert_ci_rds_to_csv.R @@ -140,7 +140,7 @@ main <- function() { cat(sprintf("Converting CI RDS to CSV: project=%s\n", project_dir)) - # Initialize project configuration + # Initialize project configuration and centralized paths tryCatch({ source("parameters_project.R") }, error = function(e) { @@ -152,15 +152,12 @@ main <- function() { }) }) - # Define paths - ci_data_source_dir <- here::here("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "cumulative_vals") - ci_data_output_dir <- here::here("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "ci_data_for_python") + # Load centralized path structure (creates all directories automatically) + paths <- setup_project_directories(project_dir) - # Create output directory if it doesn't exist (for new projects) - if (!dir.exists(ci_data_output_dir)) { - dir.create(ci_data_output_dir, recursive = TRUE, showWarnings = FALSE) - cat(sprintf("✓ Created output directory: %s\n", ci_data_output_dir)) - } + # Use centralized paths (no need for dir.create - already handled) + ci_data_source_dir <- paths$cumulative_ci_vals_dir + ci_data_output_dir <- paths$ci_for_python_dir input_file <- file.path(ci_data_source_dir, "combined_CI_data.rds") output_file <- file.path(ci_data_output_dir, "ci_data_for_python.csv") diff --git a/r_app/40_mosaic_creation.R b/r_app/40_mosaic_creation.R index a89fab8..14f5b05 100644 --- a/r_app/40_mosaic_creation.R +++ b/r_app/40_mosaic_creation.R @@ -92,32 +92,21 @@ main <- function() { # IMPORTANT: Only consider a folder as valid if it contains actual files laravel_storage <- here::here("laravel_app/storage/app", project_dir) - # If data_source was explicitly provided from pipeline, validate it; otherwise auto-detect - if (!is.null(data_source_from_args)) { - # Use the provided data_source, but verify it has data - proposed_path <- file.path(laravel_storage, data_source_from_args) - has_data <- dir.exists(proposed_path) && length(list.files(proposed_path, pattern = "\\.tif$")) > 0 - - if (has_data) { - data_source <- data_source_from_args - message("✓ Using provided data source '", data_source, "' - contains files") - } else { - message("WARNING: Provided data source '", data_source_from_args, "' is empty or doesn't exist. Auto-detecting...") - data_source_from_args <- NULL # Fall through to auto-detection - } - } - - # Auto-detect if no valid data_source was provided - if (is.null(data_source_from_args)) { - # Check merged_tif_8b - only if it exists AND contains files - merged_tif_8b_path <- file.path(laravel_storage, "merged_tif_8b") - has_8b_data <- dir.exists(merged_tif_8b_path) && length(list.files(merged_tif_8b_path, pattern = "\\.tif$")) > 0 - - # Check merged_tif - only if it exists AND contains files - merged_tif_path <- file.path(laravel_storage, "merged_tif") - has_legacy_data <- dir.exists(merged_tif_path) && length(list.files(merged_tif_path, pattern = "\\.tif$")) > 0 - - # Select data source based on what has actual data + # Load centralized path structure + tryCatch({ + source("r_app/parameters_project.R") + paths <- setup_project_directories(project_dir) + }, error = function(e) { + message("Note: Could not open files from r_app directory") + message("Attempting to source from default directory instead...") + tryCatch({ + source("parameters_project.R") + paths <- setup_project_directories(project_dir) + message("✓ Successfully sourced files from default directory") + }, error = function(e) { + stop("Failed to source required files from both 'r_app' and default directories.") + }) + }) data_source <- if (has_8b_data) { message("Auto-detected data source: merged_tif_8b (8-band optimized) - contains files") "merged_tif_8b" @@ -142,27 +131,18 @@ main <- function() { message("Attempting to source from default directory instead...") tryCatch({ source("parameters_project.R") - source("40_mosaic_creation_utils.R") + paths <- setup_project_directories(project_dir) message("✓ Successfully sourced files from default directory") }, error = function(e) { stop("Failed to source required files from both 'r_app' and default directories.") }) }) - # Extract path variables from global environment (set by parameters_project.R) - merged_final <- if (exists("merged_final", envir = .GlobalEnv)) { - get("merged_final", envir = .GlobalEnv) - } else { - file.path(laravel_storage, "merged_final_tif") - } + # Use centralized paths (no need to manually construct or create dirs) + merged_final <- paths$growth_model_interpolated_dir # or merged_final_tif if needed + daily_vrt <- paths$vrt_dir - daily_vrt <- if (exists("daily_vrt", envir = .GlobalEnv)) { - get("daily_vrt", envir = .GlobalEnv) - } else { - file.path(laravel_storage, "Data", "vrt") - } - - safe_log(paste("Using merged_final_tif directory:", merged_final)) + safe_log(paste("Using growth model/mosaic directory:", merged_final)) safe_log(paste("Using daily VRT directory:", daily_vrt)) # 4. Generate date range for processing @@ -216,10 +196,11 @@ main <- function() { # Point to the grid-specific merged_final_tif directory merged_final_with_grid <- file.path(merged_final_base, grid_size) - # Set output directory for per-tile mosaics, organized by grid size + # Set output directory for per-tile mosaics, organized by grid size (from centralized paths) # Output: weekly_tile_max/{grid_size}/week_WW_YYYY_TT.tif - tile_output_base <- file.path(laravel_storage, "weekly_tile_max", grid_size) - dir.create(tile_output_base, recursive = TRUE, showWarnings = FALSE) + tile_output_base <- file.path(paths$weekly_tile_max_dir, grid_size) + # Note: no dir.create needed - paths$weekly_tile_max_dir already created by setup_project_directories() + dir.create(tile_output_base, recursive = TRUE, showWarnings = FALSE) # Create grid-size subfolder created_tile_files <- create_weekly_mosaic_from_tiles( dates = dates, @@ -242,8 +223,8 @@ main <- function() { tryCatch({ safe_log("Starting single-file mosaic creation (backward-compatible approach)...") - # Set output directory for single-file mosaics - single_file_output_dir <- file.path(laravel_storage, "weekly_mosaic") + # Set output directory for single-file mosaics (from centralized paths) + single_file_output_dir <- paths$weekly_mosaic_dir created_file <- create_weekly_mosaic( dates = dates, diff --git a/r_app/80_calculate_kpis.R b/r_app/80_calculate_kpis.R index 5acba9a..5df29de 100644 --- a/r_app/80_calculate_kpis.R +++ b/r_app/80_calculate_kpis.R @@ -251,14 +251,10 @@ main <- function() { message("KPI Calculations:", paste(client_config$kpi_calculations, collapse = ", ")) message("Output Formats:", paste(client_config$outputs, collapse = ", ")) - # Define paths for mosaic detection (used in PHASE 1) - # NEW: Support both per-field and legacy single-file mosaics - base_project_path <- file.path("laravel_app", "storage", "app", project_dir) - weekly_tile_max <- file.path(base_project_path, "weekly_tile_max") - weekly_mosaic <- file.path(base_project_path, "weekly_mosaic") # NEW: Per-field structure - - # Also set up per-field daily RDS path for Script 80 historical data loading - daily_vals_dir <- file.path(base_project_path, "Data", "extracted_ci", "daily_vals") + # Use centralized paths from setup object (no need for file.path calls) + weekly_tile_max <- setup$weekly_tile_max_dir + weekly_mosaic <- setup$weekly_mosaic_dir + daily_vals_dir <- setup$daily_ci_vals_dir tryCatch({ source(here("r_app", "30_growth_model_utils.R")) @@ -283,11 +279,8 @@ main <- function() { stop("Error loading 80_kpi_utils.R: ", e$message) }) - # Prepare inputs for KPI calculation - reports_dir_kpi <- file.path(base_project_path, "reports", "kpis") - if (!dir.exists(reports_dir_kpi)) { - dir.create(reports_dir_kpi, recursive = TRUE) - } + # Prepare inputs for KPI calculation (already created by setup_project_directories) + reports_dir_kpi <- setup$kpi_reports_dir cumulative_CI_vals_dir <- setup$cumulative_CI_vals_dir diff --git a/r_app/90_CI_report_with_kpis_simple.Rmd b/r_app/90_CI_report_with_kpis_simple.Rmd index 8d2135f..363ef18 100644 --- a/r_app/90_CI_report_with_kpis_simple.Rmd +++ b/r_app/90_CI_report_with_kpis_simple.Rmd @@ -106,6 +106,9 @@ tryCatch({ stop("Error loading parameters_project.R: ", e$message) }) +# Load centralized paths +paths <- setup_project_directories(project_dir) + # Log initial configuration safe_log("Starting the R Markdown script with KPIs") safe_log(paste("mail_day params:", params$mail_day)) @@ -115,8 +118,8 @@ safe_log(paste("mail_day variable:", mail_day)) ```{r load_kpi_data, message=FALSE, warning=FALSE, include=FALSE} ## SIMPLE KPI LOADING - robust lookup with fallbacks -# Primary expected directory inside the laravel storage -kpi_data_dir <- file.path("..", "laravel_app", "storage", "app", project_dir, "reports", "kpis") +# Primary expected directory from centralized paths +kpi_data_dir <- paths$kpi_reports_dir date_suffix <- format(as.Date(report_date), "%Y%m%d") # Calculate current week from report_date using ISO 8601 week numbering diff --git a/r_app/91_CI_report_with_kpis_Angata.Rmd b/r_app/91_CI_report_with_kpis_Angata.Rmd index fd0875b..ace4167 100644 --- a/r_app/91_CI_report_with_kpis_Angata.Rmd +++ b/r_app/91_CI_report_with_kpis_Angata.Rmd @@ -105,6 +105,9 @@ tryCatch({ stop("Error loading parameters_project.R: ", e$message) }) +# Load centralized paths +paths <- setup_project_directories(project_dir) + # Log initial configuration safe_log("Starting the R Markdown script with KPIs") safe_log(paste("mail_day params:", params$mail_day)) @@ -120,8 +123,8 @@ cat("\n=== DEBUG: R Markdown Working Directory ===\n") cat(paste("getwd():", getwd(), "\n")) cat(paste("Expected knit_dir from R Markdown:", knitr::opts_knit$get("root.dir"), "\n\n")) -# Primary expected directory inside the laravel storage -kpi_data_dir <- file.path("..", "laravel_app", "storage", "app", project_dir, "reports", "kpis") +# Primary expected directory from centralized paths +kpi_data_dir <- paths$kpi_reports_dir date_suffix <- format(as.Date(report_date), "%Y%m%d") # Calculate current week from report_date using ISO 8601 week numbering diff --git a/r_app/parameters_project.R b/r_app/parameters_project.R index bd7c9fa..daf20c3 100644 --- a/r_app/parameters_project.R +++ b/r_app/parameters_project.R @@ -210,71 +210,228 @@ detect_tile_structure_from_merged_final <- function(merged_final_tif_dir, daily_ # 4. Define project directory structure # ----------------------------------- +# ============================================================================== +# CENTRALIZED PATH MANAGEMENT - setup_project_directories() +# ============================================================================== +# This function is the single source of truth for ALL file paths used across the pipeline. +# All scripts should call this function once at startup and use returned paths. +# This eliminates ~88 hardcoded file.path() calls scattered across 8 scripts. +# +# USAGE: +# paths <- setup_project_directories(project_dir) +# merged_tif_dir <- paths$merged_tif_folder +# daily_ci_dir <- paths$daily_ci_vals_dir +# kpi_output_dir <- paths$kpi_reports_dir +# +# TIERS (8-layer directory structure): +# Tier 1: Raw data (merged_tif) +# Tier 2: Per-field TIFFs (field_tiles, field_tiles_CI) +# Tier 3: CI Extraction (daily_ci_vals, cumulative_ci_vals) +# Tier 4: Growth Model (growth_model_interpolated) +# Tier 5: Mosaics (weekly_mosaic, weekly_tile_max) +# Tier 6: KPI & Reporting (kpi_reports_dir, kpi_field_stats_dir) +# Tier 7: Support (data, vrt, harvest, logs) +# Tier 8: Config & Metadata (field_boundaries_path, tiling_config_path) +# +# BENEFITS: +# ✓ Single source of truth (eliminates ~88 hardcoded file.path() calls) +# ✓ Auto-creates all directories (no scattered dir.create() calls) +# ✓ Easy to update storage structure globally +# ✓ Consistent naming across all 8 scripts +# ============================================================================== setup_project_directories <- function(project_dir, data_source = "merged_tif") { - # Base directories + # =========================================================================== + # BASE DIRECTORIES (Foundation for all paths) + # =========================================================================== laravel_storage_dir <- here("laravel_app", "storage", "app", project_dir) - # Use standard merged_tif directory for all projects - merged_tif_folder <- here(laravel_storage_dir, "merged_tif") + # =========================================================================== + # TIER 1: RAW DATA & INPUT PATHS (Script 00 - Python download output) + # =========================================================================== + merged_tif_folder <- here(laravel_storage_dir, "merged_tif") # 4-band raw GeoTIFFs from Planet - # Detect tile mode based on file patterns + # =========================================================================== + # TIER 2: TILING PATHS (Script 10 - Per-field tiff creation) + # =========================================================================== + # Per-field TIFF structure: field_tiles/{FIELD_NAME}/{YYYY-MM-DD}.tif + field_tiles_dir <- here(laravel_storage_dir, "field_tiles") + + # Per-field CI TIFFs (pre-computed, used by Script 40): field_tiles_CI/{FIELD_NAME}/{YYYY-MM-DD}.tif + field_tiles_ci_dir <- here(laravel_storage_dir, "field_tiles_CI") + + # Legacy tiling (for backward compatibility): daily_tiles_split/{grid_size}/{YYYY-MM-DD}/{YYYY-MM-DD}_XX.tif daily_tiles_split_dir <- here(laravel_storage_dir, "daily_tiles_split") - # Simplified: only check daily_tiles_split for per-field structure - use_tile_mosaic <- dir.exists(daily_tiles_split_dir) && length(list.dirs(daily_tiles_split_dir, full.names = FALSE, recursive = FALSE)) > 0 + # =========================================================================== + # TIER 3: CI EXTRACTION PATHS (Script 20 - Canopy Index calculation) + # =========================================================================== + extracted_ci_base_dir <- here(laravel_storage_dir, "Data", "extracted_ci") - # Main subdirectories - dirs <- list( - reports = here(laravel_storage_dir, "reports"), - logs = here(laravel_storage_dir, "logs"), - data = here(laravel_storage_dir, "Data"), - tif = list( - merged = merged_tif_folder - ), - # New per-field directory structure (Script 10 output) - field_tiles = here(laravel_storage_dir, "field_tiles"), - field_tiles_ci = here(laravel_storage_dir, "field_tiles_CI"), - weekly_mosaic = here(laravel_storage_dir, "weekly_mosaic"), - extracted_ci = list( - base = here(laravel_storage_dir, "Data", "extracted_ci"), - daily = here(laravel_storage_dir, "Data", "extracted_ci", "daily_vals"), - cumulative = here(laravel_storage_dir, "Data", "extracted_ci", "cumulative_vals"), - # New per-field daily RDS structure (Script 20 output) - daily_per_field = here(laravel_storage_dir, "Data", "extracted_ci", "daily_vals") - ), - vrt = here(laravel_storage_dir, "Data", "vrt"), - harvest = here(laravel_storage_dir, "Data", "HarvestData") + # Daily CI values (cumulative RDS): combined_CI_data.rds + daily_ci_vals_dir <- here(extracted_ci_base_dir, "daily_vals") + + # Cumulative CI across time: All_pivots_Cumulative_CI_quadrant_year_v2.rds + cumulative_ci_vals_dir <- here(extracted_ci_base_dir, "cumulative_vals") + + # Per-field CI data for Python harvest prediction (Script 21): ci_data_for_python.csv + ci_for_python_dir <- here(extracted_ci_base_dir, "ci_data_for_python") + + # =========================================================================== + # TIER 4: GROWTH MODEL PATHS (Script 30 - Interpolation & smoothing) + # =========================================================================== + growth_model_interpolated_dir <- here(laravel_storage_dir, "growth_model_interpolated") + + # =========================================================================== + # TIER 5: MOSAIC PATHS (Script 40 - Weekly mosaics) + # =========================================================================== + # Per-field weekly mosaics (per-field architecture): weekly_mosaic/{FIELD}/{week_XX_YYYY}.tif + weekly_mosaic_dir <- here(laravel_storage_dir, "weekly_mosaic") + + # Tile-based weekly max (legacy): weekly_tile_max/{grid_size}/week_XX_YYYY.tif + weekly_tile_max_dir <- here(laravel_storage_dir, "weekly_tile_max") + + # =========================================================================== + # TIER 6: KPI & REPORTING PATHS (Scripts 80, 90, 91) + # =========================================================================== + reports_dir <- here(laravel_storage_dir, "reports") + kpi_reports_dir <- here(reports_dir, "kpis") # Where Script 80 outputs KPI CSV/RDS files + kpi_field_stats_dir <- here(kpi_reports_dir, "field_stats") # Per-field KPI details + kpi_field_analysis_dir <- here(kpi_reports_dir, "field_analysis") # Field-level analysis for Script 91 + + # =========================================================================== + # TIER 7: SUPPORT PATHS (Data, VRT, Harvest) + # =========================================================================== + data_dir <- here(laravel_storage_dir, "Data") + vrt_dir <- here(data_dir, "vrt") # Virtual Raster files created during CI extraction + harvest_dir <- here(data_dir, "HarvestData") # Harvest schedule data + log_dir <- here(laravel_storage_dir, "logs") # Log files + + # =========================================================================== + # TIER 8: CONFIG & METADATA PATHS + # =========================================================================== + # Field boundaries GeoJSON (same across all scripts) + field_boundaries_path <- here(data_dir, "pivot.geojson") + + # Tiling configuration metadata from Script 10 + tiling_config_path <- here(daily_tiles_split_dir, "tiling_config.json") + + # =========================================================================== + # CREATE ALL DIRECTORIES (once per pipeline run) + # =========================================================================== + all_dirs <- c( + # Tier 1 + merged_tif_folder, + # Tier 2 + field_tiles_dir, field_tiles_ci_dir, daily_tiles_split_dir, + # Tier 3 + extracted_ci_base_dir, daily_ci_vals_dir, cumulative_ci_vals_dir, ci_for_python_dir, + # Tier 4 + growth_model_interpolated_dir, + # Tier 5 + weekly_mosaic_dir, weekly_tile_max_dir, + # Tier 6 + reports_dir, kpi_reports_dir, kpi_field_stats_dir, kpi_field_analysis_dir, + # Tier 7 + data_dir, vrt_dir, harvest_dir, log_dir ) - # Create all directories - for (dir_path in unlist(dirs)) { + for (dir_path in all_dirs) { dir.create(dir_path, showWarnings = FALSE, recursive = TRUE) } - # Return directory structure for use in other functions + # =========================================================================== + # RETURN COMPREHENSIVE PATH LIST + # Scripts should source parameters_project.R and receive paths object like: + # paths <- setup_project_directories(project_dir) + # Then use: paths$merged_tif_folder, paths$daily_ci_vals_dir, etc. + # =========================================================================== return(list( + # PROJECT ROOT laravel_storage_dir = laravel_storage_dir, - reports_dir = dirs$reports, - log_dir = dirs$logs, - data_dir = dirs$data, - planet_tif_folder = dirs$tif$merged, - merged_final = dirs$tif$final, - daily_CI_vals_dir = dirs$extracted_ci$daily, - cumulative_CI_vals_dir = dirs$extracted_ci$cumulative, - # New per-field directory paths (Script 10 & 20 outputs) - field_tiles_dir = dirs$field_tiles, - field_tiles_ci_dir = dirs$field_tiles_ci, - daily_vals_per_field_dir = dirs$extracted_ci$daily_per_field, - # Field boundaries path for all scripts - field_boundaries_path = here(laravel_storage_dir, "Data", "pivot.geojson"), - weekly_CI_mosaic = dirs$weekly_mosaic, # Per-field weekly mosaics (per-field architecture) - daily_vrt = dirs$vrt, # Point to Data/vrt folder where R creates VRT files from CI extraction - use_tile_mosaic = use_tile_mosaic, # Flag indicating if tiles are used for this project - harvest_dir = dirs$harvest, - extracted_CI_dir = dirs$extracted_ci$base + + # TIER 1: Raw data + merged_tif_folder = merged_tif_folder, + + # TIER 2: Per-field TIFFs + field_tiles_dir = field_tiles_dir, + field_tiles_ci_dir = field_tiles_ci_dir, + daily_tiles_split_dir = daily_tiles_split_dir, + + # TIER 3: CI Extraction + extracted_ci_base_dir = extracted_ci_base_dir, + daily_ci_vals_dir = daily_ci_vals_dir, + cumulative_ci_vals_dir = cumulative_ci_vals_dir, + ci_for_python_dir = ci_for_python_dir, + + # TIER 4: Growth Model + growth_model_interpolated_dir = growth_model_interpolated_dir, + + # TIER 5: Mosaics + weekly_mosaic_dir = weekly_mosaic_dir, + weekly_tile_max_dir = weekly_tile_max_dir, + + # TIER 6: KPI & Reporting + reports_dir = reports_dir, + kpi_reports_dir = kpi_reports_dir, + kpi_field_stats_dir = kpi_field_stats_dir, + kpi_field_analysis_dir = kpi_field_analysis_dir, + + # TIER 7: Support + data_dir = data_dir, + vrt_dir = vrt_dir, + harvest_dir = harvest_dir, + log_dir = log_dir, + + # TIER 8: Config & Metadata + field_boundaries_path = field_boundaries_path, + tiling_config_path = tiling_config_path )) } +# ============================================================================== +# TIER-BY-TIER PATH REFERENCE (for setup_project_directories output) +# ============================================================================== +# +# TIER 1: RAW DATA (Script 00 - Python download) +# paths$merged_tif_folder +# └─ {YYYY-MM-DD}.tif (4-band uint16 GeoTIFFs from Planet API) +# +# TIER 2: PER-FIELD TIFFS (Script 10) +# paths$field_tiles_dir/{FIELD_NAME}/{YYYY-MM-DD}.tif +# paths$field_tiles_ci_dir/{FIELD_NAME}/{YYYY-MM-DD}.tif +# paths$daily_tiles_split_dir/{grid_size}/{YYYY-MM-DD}/{YYYY-MM-DD}_XX.tif (legacy) +# +# TIER 3: CI EXTRACTION (Script 20) +# paths$daily_ci_vals_dir/combined_CI_data.rds +# paths$cumulative_ci_vals_dir/All_pivots_Cumulative_CI_quadrant_year_v2.rds +# paths$ci_for_python_dir/ci_data_for_python.csv (Script 21 output) +# +# TIER 4: GROWTH MODEL (Script 30) +# paths$growth_model_interpolated_dir/ (RDS files with interpolated CI) +# +# TIER 5: MOSAICS (Script 40) +# paths$weekly_mosaic_dir/{FIELD_NAME}/week_XX_YYYY.tif +# paths$weekly_tile_max_dir/{grid_size}/week_XX_YYYY_00.tif (legacy) +# +# TIER 6: KPI & REPORTING (Scripts 80, 90, 91) +# paths$kpi_reports_dir/ (KPI outputs from Script 80) +# paths$kpi_field_stats_dir/ (Per-field KPI RDS) +# paths$kpi_field_analysis_dir/ (Analysis RDS for Script 91) +# paths$reports_dir/ (Word/HTML reports) +# +# TIER 7: SUPPORT (Various scripts) +# paths$data_dir/pivot.geojson (Field boundaries) +# paths$data_dir/harvest.xlsx (Harvest schedule) +# paths$vrt_dir/ (Virtual raster files) +# paths$harvest_dir/ (Harvest predictions from Python) +# paths$log_dir/ (Pipeline logs) +# +# TIER 8: CONFIG & METADATA +# paths$field_boundaries_path (Full path to pivot.geojson) +# paths$tiling_config_path (Metadata from Script 10) +# +# ============================================================================== + #set working dir. # 5. Load field boundaries # ---------------------- diff --git a/r_app/run_full_pipeline.R b/r_app/run_full_pipeline.R index d3d1e33..dac54d0 100644 --- a/r_app/run_full_pipeline.R +++ b/r_app/run_full_pipeline.R @@ -39,8 +39,9 @@ force_rerun <- FALSE # Set to TRUE to force all scripts to run even if outputs e # Define Rscript path for running external R scripts via system() RSCRIPT_PATH <- file.path("C:", "Program Files", "R", "R-4.4.3", "bin", "x64", "Rscript.exe") -# Load client type mapping from parameters_project.R +# Load client type mapping and centralized paths from parameters_project.R source("r_app/parameters_project.R") +paths <- setup_project_directories(project_dir) client_type <- get_client_type(project_dir) cat(sprintf("\nProject: %s → Client Type: %s\n", project_dir, client_type)) @@ -105,7 +106,7 @@ for (i in 1:nrow(weeks_needed)) { files_this_week <- list.files(mosaic_dir_check, pattern = week_pattern_check, recursive = TRUE, full.names = FALSE) } } else if (mosaic_mode == "single-file") { - mosaic_dir_check <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic") + mosaic_dir_check <- paths$weekly_mosaic_dir if (dir.exists(mosaic_dir_check)) { # NEW: Support per-field architecture - search recursively for mosaics in field subdirectories # Check both top-level (legacy) and field subdirectories (per-field architecture) @@ -222,7 +223,7 @@ cat("\n========== CHECKING EXISTING OUTPUTS ==========\n") cat(sprintf("Auto-detected mosaic mode: %s\n", mosaic_mode)) # Check Script 10 outputs - FLEXIBLE: look for tiles either directly OR in grid subdirs -tiles_split_base <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split") +tiles_split_base <- paths$daily_tiles_split_dir tiles_dates <- c() if (dir.exists(tiles_split_base)) { # Try grid-size subdirectories first (5x5, 10x10, etc.) - preferred new structure @@ -241,7 +242,7 @@ if (dir.exists(tiles_split_base)) { cat(sprintf("Script 10: %d dates already tiled\n", length(tiles_dates))) # Check Script 20 outputs (CI extraction) - daily RDS files -ci_daily_dir <- file.path("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "daily_vals") +ci_daily_dir <- paths$daily_ci_vals_dir ci_files <- if (dir.exists(ci_daily_dir)) { list.files(ci_daily_dir, pattern = "\\.rds$") } else { @@ -301,8 +302,7 @@ tryCatch( # Setup paths # NOTE: All downloads go to merged_tif/ regardless of project # (data_source variable is used later by Script 20 for reading, but downloads always go to merged_tif) - base_path <- file.path("laravel_app", "storage", "app", project_dir) - merged_tifs_dir <- file.path(base_path, "merged_tif") # Always check merged_tif for downloads + merged_tifs_dir <- paths$merged_tif_folder # Always check merged_tif for downloads cat(sprintf("[DEBUG] Checking for existing files in: %s\n", merged_tifs_dir)) cat(sprintf("[DEBUG] Directory exists: %s\n", dir.exists(merged_tifs_dir))) @@ -404,7 +404,7 @@ if (pipeline_success && !skip_10) { } # Verify output - check per-field structure - field_tiles_dir <- file.path("laravel_app", "storage", "app", project_dir, "field_tiles") + field_tiles_dir <- paths$field_tiles_dir if (dir.exists(field_tiles_dir)) { fields <- list.dirs(field_tiles_dir, full.names = FALSE, recursive = FALSE) fields <- fields[fields != ""] @@ -445,7 +445,7 @@ if (pipeline_success && !skip_20) { } # Verify CI output was created - ci_daily_dir <- file.path("laravel_app", "storage", "app", project_dir, "Data", "extracted_ci", "daily_vals") + ci_daily_dir <- paths$daily_ci_vals_dir if (dir.exists(ci_daily_dir)) { files <- list.files(ci_daily_dir, pattern = "\\.rds$") cat(sprintf("✓ Script 20 completed - generated %d CI files\n", length(files))) @@ -478,7 +478,7 @@ if (pipeline_success && !skip_21) { main() # Call main() to execute the script with the environment variables # Verify CSV output was created - ci_csv_path <- file.path("laravel_app", "storage", "app", project_dir, "ci_extracted") + ci_csv_path <- paths$ci_for_python_dir if (dir.exists(ci_csv_path)) { csv_files <- list.files(ci_csv_path, pattern = "\\.csv$") cat(sprintf("✓ Script 21 completed - converted to %d CSV files\n", length(csv_files))) @@ -517,7 +517,7 @@ if (pipeline_success && !skip_30) { } # Verify interpolated output - growth_dir <- file.path("laravel_app", "storage", "app", project_dir, "growth_model_interpolated") + growth_dir <- paths$growth_model_interpolated_dir if (dir.exists(growth_dir)) { files <- list.files(growth_dir, pattern = "\\.rds$|\\.csv$") cat(sprintf("✓ Script 30 completed - generated %d growth model files\n", length(files))) @@ -619,7 +619,7 @@ if (pipeline_success && !skip_40) { mosaic_created <- length(mosaic_files) > 0 } } else { - mosaic_dir <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic") + mosaic_dir <- paths$weekly_mosaic_dir if (dir.exists(mosaic_dir)) { week_pattern <- sprintf("week_%02d_%d\\.tif", week_num, year_num) # NEW: Support per-field architecture - search recursively for mosaics in field subdirectories @@ -768,12 +768,9 @@ if (pipeline_success && run_legacy_report) { tryCatch( { # Script 90 is an RMarkdown file - compile it with rmarkdown::render() - output_dir <- file.path("laravel_app", "storage", "app", project_dir, "reports") + output_dir <- paths$reports_dir - # Ensure output directory exists - if (!dir.exists(output_dir)) { - dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) - } + # Reports directory already created by setup_project_directories output_filename <- sprintf( "CI_report_week%02d_%d.docx", @@ -817,12 +814,9 @@ if (pipeline_success && run_modern_report) { tryCatch( { # Script 91 is an RMarkdown file - compile it with rmarkdown::render() - output_dir <- file.path("laravel_app", "storage", "app", project_dir, "reports") + output_dir <- paths$reports_dir - # Ensure output directory exists - if (!dir.exists(output_dir)) { - dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) - } + # Reports directory already created by setup_project_directories output_filename <- sprintf( "CI_report_week%02d_%d.docx",