Refactor field analysis utilities and reporting; update column names for alerts, enhance directory setup, and improve KPI calculation robustness.

python_app/model_config.json

@@ -44,7 +44,7 @@
     "7d_std",
     "14d_std",
     "21d_std",
-    "DOY_normalized"
+    "DAH_normalized"
   ],
   "model": {
     "type": "LSTM",

r_app/80_utils_agronomic_support.R

@@ -789,8 +789,14 @@ calculate_all_field_analysis_agronomic_support <- function(
   
   # DETECT STRUCTURE FIRST - before any use of is_per_field
   week_file <- sprintf("week_%02d_%d.tif", current_week, current_year)
-  field_dirs <- list.dirs(current_mosaic_dir, full.names = FALSE, recursive = FALSE)
+  
-  field_dirs <- field_dirs[field_dirs != ""]
+  # Safely identify immediate child directories (not including root)
+  # Use list.files + dir.exists filter instead of list.dirs for robustness
+  all_entries <- list.files(current_mosaic_dir, full.names = FALSE)
+  field_dirs <- all_entries[sapply(
+    file.path(current_mosaic_dir, all_entries),
+    dir.exists
+  )]
   
   is_per_field <- length(field_dirs) > 0 && 
                   file.exists(file.path(current_mosaic_dir, field_dirs[1], week_file))
@@ -919,7 +925,7 @@ calculate_all_field_analysis_agronomic_support <- function(
   
   message("Calculating KPI 2: Area Change...")
   if (!is.null(previous_stats)) {
-    area_change_kpi <- calculate_area_change_kpi(current_stats, previous_stats)
+    area_change_kpi <- calculate_area_change_kpi(current_stats, previous_stats, field_boundaries_sf)
   } else {
     area_change_kpi <- data.frame(
       field_idx = seq_len(nrow(field_boundaries_sf)),

r_app/80_utils_cane_supply.R

@@ -31,10 +31,16 @@ library(writexl)
 # ALERT THRESHOLDS & CONFIGURATION CONSTANTS
 # ============================================================================
 
-# CI change thresholds for alert categorization
+# CI change thresholds for alert categorization and status determination
 # These values are project-standard and should be consistent across all workflows
-CI_CHANGE_DECLINE_THRESHOLD <- -0.5    # Weekly CI change threshold for decline alerts
+CI_CHANGE_RAPID_GROWTH_THRESHOLD <- 0.5      # Weekly CI change for positive growth alert
-CI_CHANGE_INCREASE_THRESHOLD <- 0.5    # Weekly CI change threshold for increase alerts
+CI_CHANGE_POSITIVE_GROWTH_THRESHOLD <- 0.2   # Weekly CI change for acceptable growth
+CI_CHANGE_STABLE_THRESHOLD <- -0.2            # Weekly CI change for stable status (between -0.2 and +0.2)
+CI_CHANGE_STRESS_TREND_THRESHOLD <- -0.3    # 4-week trend threshold for stress detection
+CI_CHANGE_RAPID_DECLINE_THRESHOLD <- -0.5   # Weekly CI change for rapid decline alert
+# Deprecated aliases (for backward compatibility if needed):
+CI_CHANGE_DECLINE_THRESHOLD <- CI_CHANGE_RAPID_DECLINE_THRESHOLD    # Weekly CI change threshold for decline alerts
+CI_CHANGE_INCREASE_THRESHOLD <- CI_CHANGE_RAPID_GROWTH_THRESHOLD    # Weekly CI change threshold for increase alerts
 
 # ============================================================================
 # ANGATA-SPECIFIC HELPER FUNCTIONS (Placeholder Section)
@@ -73,6 +79,9 @@ calculate_field_acreages <- function(field_boundaries_sf) {
     # Convert hectares to acres
     lookup_df %>%
       mutate(acreage = area_ha / 0.404686) %>%
+      # Aggregate by field to handle multi-row fields (e.g., sub_fields)
+      group_by(field) %>%
+      summarise(acreage = sum(acreage, na.rm = TRUE), .groups = "drop") %>%
       select(field, acreage)
   }, error = function(e) {
     message(paste("Warning: Could not calculate acreages from geometries -", e$message))
@@ -94,15 +103,26 @@ calculate_age_week <- function(planting_date, reference_date) {
 
 #' Assign crop phase based on age in weeks
 #'
+#' Determines crop phase from age in weeks using canonical PHASE_DEFINITIONS
+#' from 80_utils_common.R for consistency across all workflows.
+#'
 #' @param age_week Numeric age in weeks
-#' @return Character phase name
+#' @return Character phase name (from PHASE_DEFINITIONS or "Unknown")
+#' 
+#' @details
+#' Uses the shared PHASE_DEFINITIONS to ensure identical phase boundaries
+#' across all scripts. This wrapper delegates to get_phase_by_age() which
+#' is the authoritative phase lookup function.
+#' 
+#' Phase boundaries (from PHASE_DEFINITIONS):
+#'   - Germination: 0-6 weeks
+#'   - Tillering: 4-16 weeks
+#'   - Grand Growth: 17-39 weeks
+#'   - Maturation: 39+ weeks
 calculate_phase <- function(age_week) {
-  if (is.na(age_week)) return(NA_character_)
+  # Delegate to canonical get_phase_by_age() from 80_utils_common.R
-  if (age_week >= 0 & age_week < 4) return("Germination")
+  # This ensures all phase boundaries are consistent across workflows
-  if (age_week >= 4 & age_week < 17) return("Tillering")
+  get_phase_by_age(age_week)
-  if (age_week >= 17 & age_week < 39) return("Grand Growth")
-  if (age_week >= 39) return("Maturation")
-  NA_character_
 }
 
 #' Bin percentage into 10% intervals with special handling for 90-100%
@@ -187,7 +207,8 @@ calculate_status_alert <- function(imminent_prob, age_week, mean_ci,
   # Priority 3: STRESS DETECTED - consistent health decline (weekly trend)
   # Uses Four_week_trend (smooth trend) NOT daily fluctuation to avoid noise
   # Crop declining but not yet bare → opportunity to investigate
-  if (!is.na(four_week_trend) && four_week_trend < -0.3 && 
+  # References: CI_CHANGE_STRESS_TREND_THRESHOLD for 4-week trend detection
+  if (!is.na(four_week_trend) && four_week_trend < CI_CHANGE_STRESS_TREND_THRESHOLD && 
       !is.na(mean_ci) && mean_ci > 1.5) {
     return("stress_detected")
   }
@@ -202,9 +223,10 @@ calculate_status_alert <- function(imminent_prob, age_week, mean_ci,
   # Priority 5: GROWTH ON TRACK - positive operational status
   # Field is healthy, uniform, and growing steadily (no action needed)
   # Conditions: good uniformity (CV < 0.15) AND stable/positive weekly trend
+  # References: CI_CHANGE_STABLE_THRESHOLD (±0.2 = stable, no change)
   if (!is.na(cv) && cv < 0.15 && 
-      !is.na(four_week_trend) && four_week_trend >= -0.2 &&
+      !is.na(four_week_trend) && four_week_trend >= CI_CHANGE_STABLE_THRESHOLD &&
-      !is.na(weekly_ci_change) && weekly_ci_change >= -0.2) {
+      !is.na(weekly_ci_change) && weekly_ci_change >= CI_CHANGE_STABLE_THRESHOLD) {
     return("growth_on_track")
   }
   
@@ -322,7 +344,7 @@ calculate_all_field_kpis <- function(current_stats,
       # Column 13: Imminent_prob (from script 31 or NA)
       Imminent_prob = {
         if (!is.null(imminent_prob_data)) {
-          imminent_prob_data$Imminent_prob_actual[match(Field_id, imminent_prob_data$Field_id)]
+          as.numeric(imminent_prob_data$Imminent_prob_actual[match(Field_id, imminent_prob_data$Field_id)])
         } else {
           rep(NA_real_, nrow(current_stats))
         }
@@ -558,7 +580,8 @@ calculate_field_analysis_cane_supply <- function(setup,
   
   # ========== PHASE 3: LOAD PLANTING DATES ==========
   message("\nLoading harvest data from harvest.xlsx for planting dates...")
-  harvesting_data <- load_harvesting_data(data_dir)
+  # Use load_harvest_data() from 80_utils_common.R for consistency with 80_calculate_kpis.R
+  harvesting_data <- load_harvest_data(data_dir)
   
   planting_dates <- extract_planting_dates(harvesting_data, field_boundaries_sf)
   
@@ -615,7 +638,7 @@ calculate_field_analysis_cane_supply <- function(setup,
   
   # ========== PHASE 6: LOAD HARVEST PROBABILITIES ==========
   message("\n4. Loading harvest probabilities from script 31...")
-  # Use get_harvest_output_path() to safely build path (avoids NULL setup$kpi_field_stats_dir)
+  # Use get_harvest_output_path() to safely build path (stored in kpi_reports_dir)
   harvest_prob_file <- get_harvest_output_path(project_dir, current_week, current_year)
   message(paste("  Looking for:", harvest_prob_file))
   

r_app/90_CI_report_with_kpis_agronomic_support.Rmd

@@ -773,7 +773,43 @@ generate_field_alerts <- function(field_details_table) {
 
 # Generate and display alerts table
 if (exists("field_details_table") && !is.null(field_details_table) && nrow(field_details_table) > 0) {
-  alerts_data <- generate_field_alerts(field_details_table)
+  # Adapter: Map normalized column names back to legacy names for generate_field_alerts()
+  # (generates from the normalized schema created by normalize_field_details_columns + column_mappings)
+  field_details_for_alerts <- field_details_table
+  
+  # Rename normalized columns back to legacy names (only if they exist)
+  if ("Field_id" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(Field = Field_id)
+  }
+  if ("Mean_CI" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Mean CI` = Mean_CI)
+  }
+  if ("CV" %in% names(field_details_for_alerts) && !("CV Value" %in% names(field_details_for_alerts))) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`CV Value` = CV)
+  }
+  if ("TCH_Forecasted" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Yield Forecast (t/ha)` = TCH_Forecasted)
+  }
+  if ("Gap_Score" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Gap Score` = Gap_Score)
+  }
+  if ("Growth_Uniformity" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Growth Uniformity` = Growth_Uniformity)
+  }
+  if ("Decline_Risk" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Decline Risk` = Decline_Risk)
+  }
+  if ("Decline_Severity" %in% names(field_details_for_alerts) && !("Decline Risk" %in% names(field_details_for_alerts))) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Decline Risk` = Decline_Severity)
+  }
+  if ("Patchiness_Risk" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Patchiness Risk` = Patchiness_Risk)
+  }
+  if ("Morans_I" %in% names(field_details_for_alerts)) {
+    field_details_for_alerts <- field_details_for_alerts %>% dplyr::rename(`Moran's I` = Morans_I)
+  }
+  
+  alerts_data <- generate_field_alerts(field_details_for_alerts)
   if (!is.null(alerts_data) && nrow(alerts_data) > 0) {
     ft <- flextable(alerts_data) %>%
 #      set_caption("Field Alerts Summary") %>%

r_app/parameters_project.R

@@ -174,9 +174,6 @@ setup_project_directories <- function(project_dir, data_source = "merged_tif") {
   # TIER 6: KPI & REPORTING (Scripts 80/90/91 output)
   reports_dir <- here(laravel_storage_dir, "reports")
   kpi_reports_dir <- here(reports_dir, "kpis")
-  #kpi_reports_dir <- here(reports_dir, "kpis", "field_level")
-  #kpi_field_stats_dir <- here(reports_dir, "kpis", "field_stats")
-  #kpi_field_analysis_dir <- here(reports_dir, "kpis", "field_analysis")
   
   # TIER 7: SUPPORT (various scripts)
   vrt_dir <- here(data_dir, "vrt")  # Virtual Raster files created during CI extraction
@@ -189,7 +186,7 @@ setup_project_directories <- function(project_dir, data_source = "merged_tif") {
     extracted_ci_base_dir, daily_ci_vals_dir, cumulative_ci_vals_dir, ci_for_python_dir,
     growth_model_interpolated_dir,
     weekly_mosaic_dir,
-    reports_dir, kpi_reports_dir, #kpi_field_stats_dir, kpi_field_analysis_dir,
+    reports_dir, kpi_reports_dir,
     data_dir, vrt_dir, harvest_dir, log_dir
   )
   
@@ -228,8 +225,6 @@ setup_project_directories <- function(project_dir, data_source = "merged_tif") {
     # TIER 6: KPI & reporting
     reports_dir = reports_dir,
     kpi_reports_dir = kpi_reports_dir,
-    #kpi_field_stats_dir = kpi_field_stats_dir,
-    #kpi_field_analysis_dir = kpi_field_analysis_dir,
     
     # TIER 7: Support
     data_dir = data_dir,