Fix backward compatibility: Support non-tiled (single-file) mosaic projects
- Script 80: Add fallback logic to detect and handle single-file mosaics
- Weekly stats utils: Support both tile-based and single-file mosaic detection
- Pipeline runner: Auto-detect mosaic mode (tiled vs single-file)
- Flexible grid size detection for tile-based projects (5x5, 10x10, etc)
Fixes:
- Script 80 now checks weekly_tile_max/{grid_size} first, falls back to weekly_mosaic
- calculate_field_statistics handles both tile patterns and single-file patterns
- run_full_pipeline detects project mode automatically
- All verification checks are now flexible and don't assume fixed paths
Projects like 'aura' (small ROI < 10km) will use single-file approach automatically
Projects like 'angata' (large ROI >= 10km) will use tile-based approach automatically
This commit is contained in:
parent
fde60cbbdf
commit
b2de819fc4
|
|
@ -173,9 +173,6 @@ STATUS_TRIGGERS <- data.frame(
|
||||||
# MAIN
|
# MAIN
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
# ============================================================================
|
|
||||||
# MAIN
|
|
||||||
# ============================================================================
|
|
||||||
|
|
||||||
main <- function() {
|
main <- function() {
|
||||||
# Parse command-line arguments
|
# Parse command-line arguments
|
||||||
|
|
@ -261,27 +258,50 @@ main <- function() {
|
||||||
|
|
||||||
message(paste("Week:", current_week, "/ Year:", year))
|
message(paste("Week:", current_week, "/ Year:", year))
|
||||||
|
|
||||||
# Find tile files - approach from Script 20
|
# Find mosaic files - support both tile-based AND single-file approaches
|
||||||
message("Finding tile files...")
|
message("Finding mosaic files...")
|
||||||
tile_pattern <- sprintf("week_%02d_%d_([0-9]{2})\\.tif", current_week, year)
|
tile_pattern <- sprintf("week_%02d_%d_([0-9]{2})\\.tif", current_week, year)
|
||||||
|
single_file_pattern <- sprintf("week_%02d_%d\\.tif", current_week, year)
|
||||||
|
|
||||||
# Detect grid size subdirectory
|
# PRIORITY 1: Check for tile-based mosaics (projects with large ROI)
|
||||||
detected_grid_size <- NA
|
detected_grid_size <- NA
|
||||||
|
mosaic_dir <- NA
|
||||||
|
mosaic_mode <- NA
|
||||||
|
|
||||||
if (dir.exists(weekly_tile_max)) {
|
if (dir.exists(weekly_tile_max)) {
|
||||||
subfolders <- list.dirs(weekly_tile_max, full.names = FALSE, recursive = FALSE)
|
subfolders <- list.dirs(weekly_tile_max, full.names = FALSE, recursive = FALSE)
|
||||||
grid_patterns <- grep("^\\d+x\\d+$", subfolders, value = TRUE)
|
grid_patterns <- grep("^\\d+x\\d+$", subfolders, value = TRUE)
|
||||||
if (length(grid_patterns) > 0) {
|
if (length(grid_patterns) > 0) {
|
||||||
detected_grid_size <- grid_patterns[1]
|
detected_grid_size <- grid_patterns[1]
|
||||||
mosaic_dir <- file.path(weekly_tile_max, detected_grid_size)
|
mosaic_dir <- file.path(weekly_tile_max, detected_grid_size)
|
||||||
message(paste(" Using grid-size subdirectory:", detected_grid_size))
|
tile_files <- list.files(mosaic_dir, pattern = tile_pattern, full.names = TRUE)
|
||||||
|
|
||||||
|
if (length(tile_files) > 0) {
|
||||||
|
message(paste(" ✓ Using tile-based approach (grid-size:", detected_grid_size, ")"))
|
||||||
|
message(paste(" Found", length(tile_files), "tiles"))
|
||||||
|
mosaic_mode <- "tiled"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tile_files <- list.files(mosaic_dir, pattern = tile_pattern, full.names = TRUE)
|
# PRIORITY 2: Fall back to single-file mosaic (projects with small ROI, legacy approach)
|
||||||
if (length(tile_files) == 0) {
|
if (is.na(mosaic_mode)) {
|
||||||
stop(paste("No tile files found for week", current_week, year, "in", mosaic_dir))
|
message(" No tiles found. Checking for single-file mosaic (legacy approach)...")
|
||||||
|
mosaic_dir <- weekly_mosaic
|
||||||
|
single_file <- list.files(mosaic_dir, pattern = single_file_pattern, full.names = TRUE)
|
||||||
|
|
||||||
|
if (length(single_file) > 0) {
|
||||||
|
message(paste(" ✓ Using single-file approach"))
|
||||||
|
message(paste(" Found 1 mosaic file:", basename(single_file[1])))
|
||||||
|
mosaic_mode <- "single-file"
|
||||||
|
} else {
|
||||||
|
stop(paste("ERROR: No mosaic files found for week", current_week, year,
|
||||||
|
"\n Checked (1) tile-based:", file.path(weekly_tile_max, "*", "week_*.tif"),
|
||||||
|
"\n Checked (2) single-file:", file.path(weekly_mosaic, "week_*.tif")))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
message(paste(" Found", length(tile_files), "tiles"))
|
|
||||||
|
message(paste(" Using mosaic mode:", mosaic_mode))
|
||||||
|
|
||||||
# Load field boundaries
|
# Load field boundaries
|
||||||
tryCatch({
|
tryCatch({
|
||||||
|
|
@ -354,8 +374,38 @@ main <- function() {
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
# Build tile grid (needed by calculate_field_statistics)
|
# Build tile grid (needed by calculate_field_statistics)
|
||||||
message("\nBuilding tile grid for current week...")
|
message("\nPreparing mosaic configuration for statistics calculation...")
|
||||||
tile_grid <- build_tile_grid(mosaic_dir, current_week, year)
|
|
||||||
|
# For tile-based mosaics: build the grid mapping
|
||||||
|
# For single-file: create a minimal grid structure (single "tile" = entire mosaic)
|
||||||
|
if (mosaic_mode == "tiled") {
|
||||||
|
tile_grid <- build_tile_grid(mosaic_dir, current_week, year)
|
||||||
|
message(paste(" ✓ Built tile grid with", nrow(tile_grid), "tiles"))
|
||||||
|
} else {
|
||||||
|
# Single-file mode: create a minimal grid with just the single mosaic
|
||||||
|
message(" ✓ Using single-file mosaic (no tile grid needed)")
|
||||||
|
single_file_pattern <- sprintf("week_%02d_%d\\.tif", current_week, year)
|
||||||
|
single_file <- list.files(mosaic_dir, pattern = single_file_pattern, full.names = TRUE)
|
||||||
|
|
||||||
|
if (length(single_file) == 0) {
|
||||||
|
stop("ERROR: Single-file mosaic not found in", mosaic_dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create a minimal tile_grid structure with one "tile" representing the entire mosaic
|
||||||
|
tile_grid <- list(
|
||||||
|
mosaic_dir = mosaic_dir,
|
||||||
|
data = data.frame(
|
||||||
|
id = 0, # Single tile ID = 0 (full extent)
|
||||||
|
xmin = NA_real_,
|
||||||
|
xmax = NA_real_,
|
||||||
|
ymin = NA_real_,
|
||||||
|
ymax = NA_real_,
|
||||||
|
stringsAsFactors = FALSE
|
||||||
|
),
|
||||||
|
mode = "single-file",
|
||||||
|
file = single_file[1]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
message("\nUsing modular RDS-based approach for weekly statistics...")
|
message("\nUsing modular RDS-based approach for weekly statistics...")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -381,14 +381,25 @@ calculate_field_statistics <- function(field_boundaries_sf, week_num, year,
|
||||||
|
|
||||||
message(paste("Calculating statistics for all fields - Week", week_num, year))
|
message(paste("Calculating statistics for all fields - Week", week_num, year))
|
||||||
|
|
||||||
|
# Support both tile-based and single-file mosaics
|
||||||
tile_pattern <- sprintf("week_%02d_%d_([0-9]{2})\\.tif", week_num, year)
|
tile_pattern <- sprintf("week_%02d_%d_([0-9]{2})\\.tif", week_num, year)
|
||||||
|
single_file_pattern <- sprintf("week_%02d_%d\\.tif", week_num, year)
|
||||||
|
|
||||||
|
# Try tile-based first
|
||||||
tile_files <- list.files(mosaic_dir, pattern = tile_pattern, full.names = TRUE)
|
tile_files <- list.files(mosaic_dir, pattern = tile_pattern, full.names = TRUE)
|
||||||
|
|
||||||
|
# If no tiles, try single-file
|
||||||
if (length(tile_files) == 0) {
|
if (length(tile_files) == 0) {
|
||||||
stop(paste("No tile files found for week", week_num, year, "in", mosaic_dir))
|
single_file <- list.files(mosaic_dir, pattern = single_file_pattern, full.names = TRUE)
|
||||||
|
if (length(single_file) > 0) {
|
||||||
|
message(paste(" Using single-file mosaic for week", week_num))
|
||||||
|
tile_files <- single_file[1] # Use first match as single "tile"
|
||||||
|
} else {
|
||||||
|
stop(paste("No mosaic files found for week", week_num, year, "in", mosaic_dir))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
message(paste(" Found", length(tile_files), "tiles for week", week_num))
|
message(paste(" Found", length(tile_files), "mosaic file(s) for week", week_num))
|
||||||
|
|
||||||
results_list <- list()
|
results_list <- list()
|
||||||
fields_processed <- 0
|
fields_processed <- 0
|
||||||
|
|
|
||||||
|
|
@ -48,12 +48,44 @@ pipeline_success <- TRUE
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
cat("\n========== CHECKING EXISTING OUTPUTS ==========\n")
|
cat("\n========== CHECKING EXISTING OUTPUTS ==========\n")
|
||||||
|
|
||||||
# Check Script 10 outputs (tiled splits)
|
# Detect mosaic mode (tile-based vs single-file) automatically
|
||||||
tiles_dir <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split", "5x5")
|
detect_mosaic_mode_simple <- function(project_dir) {
|
||||||
tiles_dates <- if (dir.exists(tiles_dir)) {
|
# Check for tile-based approach: weekly_tile_max/{grid_size}/week_*.tif
|
||||||
list.dirs(tiles_dir, full.names = FALSE, recursive = FALSE)
|
weekly_tile_max <- file.path("laravel_app", "storage", "app", project_dir, "weekly_tile_max")
|
||||||
} else {
|
if (dir.exists(weekly_tile_max)) {
|
||||||
c()
|
subfolders <- list.dirs(weekly_tile_max, full.names = FALSE, recursive = FALSE)
|
||||||
|
grid_patterns <- grep("^\\d+x\\d+$", subfolders, value = TRUE)
|
||||||
|
if (length(grid_patterns) > 0) {
|
||||||
|
return("tiled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check for single-file approach: weekly_mosaic/week_*.tif
|
||||||
|
weekly_mosaic <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic")
|
||||||
|
if (dir.exists(weekly_mosaic)) {
|
||||||
|
files <- list.files(weekly_mosaic, pattern = "^week_.*\\.tif$")
|
||||||
|
if (length(files) > 0) {
|
||||||
|
return("single-file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return("unknown")
|
||||||
|
}
|
||||||
|
|
||||||
|
mosaic_mode <- detect_mosaic_mode_simple(project_dir)
|
||||||
|
cat(sprintf("Auto-detected mosaic mode: %s\n", mosaic_mode))
|
||||||
|
|
||||||
|
# Check Script 10 outputs - look for daily_tiles_split/{GRID_SIZE} (flexible grid detection)
|
||||||
|
tiles_split_base <- file.path("laravel_app", "storage", "app", project_dir, "daily_tiles_split")
|
||||||
|
tiles_dates <- c()
|
||||||
|
if (dir.exists(tiles_split_base)) {
|
||||||
|
# Look for any grid-size subdirectories (5x5, 10x10, etc.)
|
||||||
|
subfolders <- list.dirs(tiles_split_base, full.names = FALSE, recursive = FALSE)
|
||||||
|
grid_patterns <- grep("^\\d+x\\d+$", subfolders, value = TRUE)
|
||||||
|
if (length(grid_patterns) > 0) {
|
||||||
|
grid_dir <- file.path(tiles_split_base, grid_patterns[1])
|
||||||
|
tiles_dates <- list.dirs(grid_dir, full.names = FALSE, recursive = FALSE)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
cat(sprintf("Script 10: %d dates already tiled\n", length(tiles_dates)))
|
cat(sprintf("Script 10: %d dates already tiled\n", length(tiles_dates)))
|
||||||
|
|
||||||
|
|
@ -71,12 +103,21 @@ cat(sprintf("Script 20: %d CI daily RDS files exist\n", length(ci_files)))
|
||||||
# For now, just note that CSV is time-dependent, not a good skip indicator
|
# For now, just note that CSV is time-dependent, not a good skip indicator
|
||||||
cat("Script 21: CSV file exists but gets overwritten - will run if Script 20 runs\n")
|
cat("Script 21: CSV file exists but gets overwritten - will run if Script 20 runs\n")
|
||||||
|
|
||||||
# Check Script 40 outputs (mosaics in weekly_tile_max/5x5)
|
# Check Script 40 outputs (mosaics) - flexible detection for both tile-based and single-file
|
||||||
mosaic_dir <- file.path("laravel_app", "storage", "app", project_dir, "weekly_tile_max", "5x5")
|
mosaic_files <- c()
|
||||||
mosaic_files <- if (dir.exists(mosaic_dir)) {
|
if (mosaic_mode == "tiled") {
|
||||||
list.files(mosaic_dir, pattern = "\\.tif$")
|
# For tile-based: look in weekly_tile_max/{grid_size}/
|
||||||
} else {
|
weekly_tile_max <- file.path("laravel_app", "storage", "app", project_dir, "weekly_tile_max")
|
||||||
c()
|
subfolders <- list.dirs(weekly_tile_max, full.names = FALSE, recursive = FALSE)
|
||||||
|
grid_patterns <- grep("^\\d+x\\d+$", subfolders, value = TRUE)
|
||||||
|
if (length(grid_patterns) > 0) {
|
||||||
|
mosaic_dir <- file.path(weekly_tile_max, grid_patterns[1])
|
||||||
|
mosaic_files <- list.files(mosaic_dir, pattern = "\\.tif$")
|
||||||
|
}
|
||||||
|
} else if (mosaic_mode == "single-file") {
|
||||||
|
# For single-file: look in weekly_mosaic/
|
||||||
|
mosaic_dir <- file.path("laravel_app", "storage", "app", project_dir, "weekly_mosaic")
|
||||||
|
mosaic_files <- list.files(mosaic_dir, pattern = "^week_.*\\.tif$")
|
||||||
}
|
}
|
||||||
cat(sprintf("Script 40: %d mosaic files exist\n", length(mosaic_files)))
|
cat(sprintf("Script 40: %d mosaic files exist\n", length(mosaic_files)))
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue