Remove: cleanup test/draft files not in Bitbucket
This commit is contained in:
parent
dfa0aa900d
commit
6d5509dd1b
|
|
@ -1,78 +0,0 @@
|
|||
s#!/usr/bin/env Rscript
|
||||
# Script to examine cross-validation fold results
|
||||
|
||||
library(dplyr)
|
||||
library(caret)
|
||||
|
||||
# Load the saved models
|
||||
models <- readRDS("laravel_app/storage/app/esa/reports/yield_prediction/esa_yield_models.rds")
|
||||
|
||||
# Model 1: CI Only
|
||||
cat("\n=== MODEL 1: CI ONLY ===\n")
|
||||
cat("Best mtry:", models$model1$bestTune$mtry, "\n\n")
|
||||
cat("Cross-validation results (5 folds):\n")
|
||||
print(models$model1$resample)
|
||||
cat("\nFold Performance Summary:\n")
|
||||
cat("RMSE - Mean:", round(mean(models$model1$resample$RMSE), 2),
|
||||
"± SD:", round(sd(models$model1$resample$RMSE), 2),
|
||||
"(CV:", round((sd(models$model1$resample$RMSE) / mean(models$model1$resample$RMSE)) * 100, 1), "%)\n")
|
||||
cat("MAE - Mean:", round(mean(models$model1$resample$MAE), 2),
|
||||
"± SD:", round(sd(models$model1$resample$MAE), 2), "\n")
|
||||
cat("R² - Mean:", round(mean(models$model1$resample$Rsquared), 3),
|
||||
"± SD:", round(sd(models$model1$resample$Rsquared), 3), "\n")
|
||||
cat("\nRange across folds:\n")
|
||||
cat("RMSE: [", round(min(models$model1$resample$RMSE), 2), "-",
|
||||
round(max(models$model1$resample$RMSE), 2), "]\n")
|
||||
cat("R²: [", round(min(models$model1$resample$Rsquared), 3), "-",
|
||||
round(max(models$model1$resample$Rsquared), 3), "]\n")
|
||||
|
||||
# Model 2: CI + Ratoon
|
||||
cat("\n\n=== MODEL 2: CI + RATOON ===\n")
|
||||
cat("Best mtry:", models$model2$bestTune$mtry, "\n\n")
|
||||
cat("Cross-validation results (5 folds):\n")
|
||||
print(models$model2$resample)
|
||||
cat("\nFold Performance Summary:\n")
|
||||
cat("RMSE - Mean:", round(mean(models$model2$resample$RMSE), 2),
|
||||
"± SD:", round(sd(models$model2$resample$RMSE), 2),
|
||||
"(CV:", round((sd(models$model2$resample$RMSE) / mean(models$model2$resample$RMSE)) * 100, 1), "%)\n")
|
||||
cat("MAE - Mean:", round(mean(models$model2$resample$MAE), 2),
|
||||
"± SD:", round(sd(models$model2$resample$MAE), 2), "\n")
|
||||
cat("R² - Mean:", round(mean(models$model2$resample$Rsquared), 3),
|
||||
"± SD:", round(sd(models$model2$resample$Rsquared), 3), "\n")
|
||||
cat("\nRange across folds:\n")
|
||||
cat("RMSE: [", round(min(models$model2$resample$RMSE), 2), "-",
|
||||
round(max(models$model2$resample$RMSE), 2), "]\n")
|
||||
cat("R²: [", round(min(models$model2$resample$Rsquared), 3), "-",
|
||||
round(max(models$model2$resample$Rsquared), 3), "]\n")
|
||||
|
||||
# Model 3: Full
|
||||
cat("\n\n=== MODEL 3: FULL MODEL ===\n")
|
||||
cat("Best mtry:", models$model3$bestTune$mtry, "\n\n")
|
||||
cat("Cross-validation results (5 folds):\n")
|
||||
print(models$model3$resample)
|
||||
cat("\nFold Performance Summary:\n")
|
||||
cat("RMSE - Mean:", round(mean(models$model3$resample$RMSE), 2),
|
||||
"± SD:", round(sd(models$model3$resample$RMSE), 2),
|
||||
"(CV:", round((sd(models$model3$resample$RMSE) / mean(models$model3$resample$RMSE)) * 100, 1), "%)\n")
|
||||
cat("MAE - Mean:", round(mean(models$model3$resample$MAE), 2),
|
||||
"± SD:", round(sd(models$model3$resample$MAE), 2), "\n")
|
||||
cat("R² - Mean:", round(mean(models$model3$resample$Rsquared), 3),
|
||||
"± SD:", round(sd(models$model3$resample$Rsquared), 3), "\n")
|
||||
cat("\nRange across folds:\n")
|
||||
cat("RMSE: [", round(min(models$model3$resample$RMSE), 2), "-",
|
||||
round(max(models$model3$resample$RMSE), 2), "]\n")
|
||||
cat("R²: [", round(min(models$model3$resample$Rsquared), 3), "-",
|
||||
round(max(models$model3$resample$Rsquared), 3), "]\n")
|
||||
|
||||
# Check seed info
|
||||
cat("\n\n=== SEED INFORMATION ===\n")
|
||||
cat("Note: The script uses set.seed(123) for reproducibility\n")
|
||||
cat("This ensures the same fold splits and randomForest initialization\n")
|
||||
cat("Different seeds WILL produce different results because:\n")
|
||||
cat(" 1. Different fold assignments in cross-validation\n")
|
||||
cat(" 2. Different bootstrap samples in randomForest\n")
|
||||
cat(" 3. Different random splits at each tree node\n")
|
||||
cat("\nExpected seed sensitivity:\n")
|
||||
cat(" - RMSE variation: ±1-3 t/ha (typical)\n")
|
||||
cat(" - R² variation: ±0.02-0.05 (typical)\n")
|
||||
cat(" - Fold-to-fold variation within single seed: see CV above\n")
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
# CREATE_ALL_WEEKLY_MOSAICS.R
|
||||
# ===========================
|
||||
# Generate weekly mosaics for all available weeks in the merged_final_tif dataset
|
||||
# This script identifies all unique weeks from the TIF files and creates mosaics
|
||||
# for weeks that don't already have mosaics.
|
||||
|
||||
suppressPackageStartupMessages({
|
||||
library(terra)
|
||||
library(sf)
|
||||
library(dplyr)
|
||||
library(lubridate)
|
||||
library(here)
|
||||
})
|
||||
|
||||
# Set project directory
|
||||
project_dir <- "esa"
|
||||
assign("project_dir", project_dir, envir = .GlobalEnv)
|
||||
|
||||
# Source required files
|
||||
cat("Loading project configuration...\n")
|
||||
source(here("r_app", "parameters_project.R"))
|
||||
source(here("r_app", "mosaic_creation_utils.R"))
|
||||
|
||||
# Get all TIF files from merged_final_tif
|
||||
merged_final_dir <- here("laravel_app/storage/app", project_dir, "merged_final_tif")
|
||||
tif_files <- list.files(merged_final_dir, pattern = "\\.tif$", full.names = FALSE)
|
||||
|
||||
cat("Found", length(tif_files), "TIF files\n")
|
||||
|
||||
# Extract dates from filenames
|
||||
dates <- as.Date(gsub("\\.tif$", "", tif_files))
|
||||
cat("Date range:", as.character(min(dates)), "to", as.character(max(dates)), "\n")
|
||||
|
||||
# Create a data frame with week and year for each date
|
||||
weeks_df <- data.frame(
|
||||
date = dates,
|
||||
week = isoweek(dates),
|
||||
year = isoyear(dates)
|
||||
) %>%
|
||||
# Get unique weeks
|
||||
distinct(week, year) %>%
|
||||
arrange(year, week) %>%
|
||||
# Create a representative date for each week (middle of the week)
|
||||
mutate(
|
||||
# Calculate the Monday of each ISO week
|
||||
week_start = as.Date(paste0(year, "-01-01")) + weeks((week - 1)),
|
||||
# Adjust to ensure it's actually the correct week
|
||||
week_start = week_start - wday(week_start, week_start = 1) + 1,
|
||||
# Use Wednesday as the representative date (middle of week)
|
||||
representative_date = week_start + 2
|
||||
)
|
||||
|
||||
cat("Total unique weeks:", nrow(weeks_df), "\n")
|
||||
|
||||
# Check which mosaics already exist
|
||||
weekly_mosaic_dir <- here("laravel_app/storage/app", project_dir, "weekly_mosaic")
|
||||
existing_mosaics <- list.files(weekly_mosaic_dir, pattern = "^week_.*\\.tif$", full.names = FALSE)
|
||||
existing_weeks <- gsub("^week_|.tif$", "", existing_mosaics) %>%
|
||||
strsplit("_") %>%
|
||||
lapply(function(x) data.frame(week = as.integer(x[1]), year = as.integer(x[2]))) %>%
|
||||
bind_rows()
|
||||
|
||||
cat("Existing mosaics:", nrow(existing_weeks), "\n")
|
||||
|
||||
# Find missing weeks
|
||||
weeks_df <- weeks_df %>%
|
||||
anti_join(existing_weeks, by = c("week", "year")) %>%
|
||||
arrange(year, week)
|
||||
|
||||
cat("Missing mosaics:", nrow(weeks_df), "\n")
|
||||
cat("\n")
|
||||
|
||||
if (nrow(weeks_df) == 0) {
|
||||
cat("All mosaics already exist!\n")
|
||||
quit(save = "no", status = 0)
|
||||
}
|
||||
|
||||
# Ask for confirmation
|
||||
cat("This will create", nrow(weeks_df), "weekly mosaics.\n")
|
||||
cat("Estimated time: ~", round(nrow(weeks_df) * 30 / 60, 1), "minutes (assuming 30 seconds per mosaic)\n")
|
||||
cat("\nProcessing will begin in 5 seconds... (Ctrl+C to cancel)\n")
|
||||
Sys.sleep(5)
|
||||
|
||||
# Create mosaics for each missing week
|
||||
cat("\n=== Starting mosaic creation ===\n\n")
|
||||
success_count <- 0
|
||||
error_count <- 0
|
||||
error_weeks <- list()
|
||||
|
||||
for (i in 1:nrow(weeks_df)) {
|
||||
week_info <- weeks_df[i, ]
|
||||
|
||||
cat(sprintf("[%d/%d] Creating mosaic for week %02d of %d...",
|
||||
i, nrow(weeks_df), week_info$week, week_info$year))
|
||||
|
||||
tryCatch({
|
||||
# Use the representative date (Wednesday of the week) with 7-day offset
|
||||
end_date <- week_info$representative_date
|
||||
offset <- 7 # Look back 7 days to cover the whole week
|
||||
|
||||
# Generate date range
|
||||
dates <- date_list(end_date, offset)
|
||||
|
||||
# Create output filename
|
||||
file_name_tif <- sprintf("week_%02d_%d.tif", week_info$week, week_info$year)
|
||||
|
||||
# Create the mosaic
|
||||
output_file <- create_weekly_mosaic(
|
||||
dates = dates,
|
||||
field_boundaries = field_boundaries,
|
||||
daily_vrt_dir = daily_vrt,
|
||||
merged_final_dir = merged_final_dir,
|
||||
output_dir = weekly_mosaic_dir,
|
||||
file_name_tif = file_name_tif,
|
||||
create_plots = FALSE # Disable plots for batch processing
|
||||
)
|
||||
|
||||
if (file.exists(output_file)) {
|
||||
cat(" ✓\n")
|
||||
success_count <- success_count + 1
|
||||
} else {
|
||||
cat(" ✗ (file not created)\n")
|
||||
error_count <- error_count + 1
|
||||
error_weeks[[length(error_weeks) + 1]] <- week_info
|
||||
}
|
||||
|
||||
}, error = function(e) {
|
||||
cat(" ✗\n")
|
||||
cat(" Error:", e$message, "\n")
|
||||
error_count <- error_count + 1
|
||||
error_weeks[[length(error_weeks) + 1]] <- week_info
|
||||
})
|
||||
|
||||
# Progress update every 10 mosaics
|
||||
if (i %% 10 == 0) {
|
||||
cat(sprintf("\nProgress: %d/%d completed (%.1f%%) | Success: %d | Errors: %d\n\n",
|
||||
i, nrow(weeks_df), (i/nrow(weeks_df))*100, success_count, error_count))
|
||||
}
|
||||
}
|
||||
|
||||
# Final summary
|
||||
cat("\n=== SUMMARY ===\n")
|
||||
cat("Total weeks processed:", nrow(weeks_df), "\n")
|
||||
cat("Successful:", success_count, "\n")
|
||||
cat("Errors:", error_count, "\n")
|
||||
|
||||
if (error_count > 0) {
|
||||
cat("\nWeeks with errors:\n")
|
||||
error_df <- bind_rows(error_weeks)
|
||||
print(error_df)
|
||||
}
|
||||
|
||||
cat("\nDone!\n")
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
# CREATE_MAX_MOSAIC.R
|
||||
# ===================
|
||||
# Create maximum value mosaics (Chlorophyll Index) for current and previous weeks
|
||||
# Uses pre-processed merged_final_tif files (already have 5 bands)
|
||||
# Usage: Rscript create_max_mosaic.R [end_date] [project_dir]
|
||||
# Example: Rscript r_app/create_max_mosaic.R 2025-12-24 angata
|
||||
|
||||
library(terra)
|
||||
library(here)
|
||||
library(lubridate)
|
||||
|
||||
main <- function() {
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
|
||||
# Parse arguments
|
||||
end_date <- if (length(args) >= 1 && !is.na(args[1])) {
|
||||
as.Date(args[1])
|
||||
} else {
|
||||
Sys.Date()
|
||||
}
|
||||
|
||||
project_dir <- if (length(args) >= 2) args[2] else "angata"
|
||||
|
||||
cat(sprintf("Creating MAX mosaics (Chlorophyll Index): end_date=%s, project=%s\n",
|
||||
format(end_date, "%Y-%m-%d"), project_dir))
|
||||
|
||||
# Calculate date ranges
|
||||
current_week_start <- end_date - 6 # Last 7 days
|
||||
prev_week_start <- end_date - 13 # 7-14 days back
|
||||
prev_week_end <- end_date - 7
|
||||
|
||||
cat(sprintf("Current week: %s to %s\n", format(current_week_start, "%Y-%m-%d"), format(end_date, "%Y-%m-%d")))
|
||||
cat(sprintf("Previous week: %s to %s\n", format(prev_week_start, "%Y-%m-%d"), format(prev_week_end, "%Y-%m-%d")))
|
||||
|
||||
# Set up paths
|
||||
tif_dir <- here("laravel_app/storage/app", project_dir, "merged_final_tif")
|
||||
output_dir <- here("laravel_app/storage/app", project_dir, "weekly_mosaic")
|
||||
dir.create(output_dir, showWarnings = FALSE, recursive = TRUE)
|
||||
|
||||
# Find all TIF files
|
||||
tif_files <- list.files(tif_dir, pattern = "\\.tif$", full.names = TRUE)
|
||||
|
||||
if (length(tif_files) == 0) {
|
||||
cat("✗ No TIF files found in:", tif_dir, "\n")
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
cat(sprintf("✓ Found %d total TIF files\n", length(tif_files)))
|
||||
|
||||
# Extract dates from filenames (assumes YYYY-MM-DD.tif format)
|
||||
extract_date_from_filename <- function(filepath) {
|
||||
basename <- basename(filepath)
|
||||
# Extract date part before .tif
|
||||
date_str <- sub("\\.tif$", "", basename)
|
||||
tryCatch(as.Date(date_str), error = function(e) NA)
|
||||
}
|
||||
|
||||
# Filter files by date range
|
||||
filter_files_by_date <- function(files, start_date, end_date) {
|
||||
dates <- sapply(files, extract_date_from_filename)
|
||||
valid_idx <- !is.na(dates) & dates >= start_date & dates <= end_date
|
||||
files[valid_idx]
|
||||
}
|
||||
|
||||
# Create mosaic for date range
|
||||
create_mosaic <- function(files, output_path, week_label) {
|
||||
if (length(files) == 0) {
|
||||
cat(sprintf("⚠ No files found for %s\n", week_label))
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
cat(sprintf("\n%s: Processing %d files\n", week_label, length(files)))
|
||||
|
||||
# Read all TIFs
|
||||
cat(" Reading TIFs...\n")
|
||||
rast_list <- lapply(files, function(f) {
|
||||
tryCatch(rast(f), error = function(e) {
|
||||
cat(sprintf(" ⚠ Failed to read: %s\n", basename(f)))
|
||||
NULL
|
||||
})
|
||||
})
|
||||
|
||||
rast_list <- rast_list[!sapply(rast_list, is.null)]
|
||||
|
||||
if (length(rast_list) == 0) {
|
||||
cat(sprintf(" ✗ Could not read any TIF files\n"))
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
cat(sprintf(" ✓ Successfully read %d TIFs\n", length(rast_list)))
|
||||
|
||||
# Stack them
|
||||
cat(" Stacking rasters...\n")
|
||||
stacked <- do.call(c, rast_list)
|
||||
|
||||
# Create max mosaic (pixel-wise maximum across all dates)
|
||||
cat(" Computing maximum mosaic...\n")
|
||||
max_mosaic <- max(stacked, na.rm = TRUE)
|
||||
|
||||
# Save
|
||||
cat(sprintf(" Saving to: %s\n", basename(output_path)))
|
||||
writeRaster(max_mosaic, output_path, overwrite = TRUE)
|
||||
cat(sprintf(" ✓ MAX mosaic created\n"))
|
||||
return(TRUE)
|
||||
}
|
||||
|
||||
# Create both mosaics
|
||||
current_files <- filter_files_by_date(tif_files, current_week_start, end_date)
|
||||
prev_files <- filter_files_by_date(tif_files, prev_week_start, prev_week_end)
|
||||
|
||||
success <- TRUE
|
||||
success <- create_mosaic(current_files,
|
||||
file.path(output_dir, "max_mosaic_current_week.tif"),
|
||||
"CURRENT WEEK") && success
|
||||
|
||||
success <- create_mosaic(prev_files,
|
||||
file.path(output_dir, "max_mosaic_previous_week.tif"),
|
||||
"PREVIOUS WEEK") && success
|
||||
|
||||
if (success) {
|
||||
cat("\n✓ Both mosaics created successfully\n")
|
||||
}
|
||||
return(success)
|
||||
}
|
||||
|
||||
if (sys.nframe() == 0) {
|
||||
main()
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env Rscript
|
||||
|
||||
rds_path <- '../laravel_app/storage/app/angata/reports/kpis/angata_kpi_summary_tables_week49.rds'
|
||||
data <- readRDS(rds_path)
|
||||
|
||||
cat("=== RDS Top-level names ===\n")
|
||||
print(names(data))
|
||||
|
||||
cat("\n=== field_analysis columns ===\n")
|
||||
print(names(data[['field_analysis']]))
|
||||
|
||||
cat("\n=== First row of field_analysis (all columns) ===\n")
|
||||
print(data[['field_analysis']][1, ])
|
||||
|
||||
cat("\n=== Data types ===\n")
|
||||
print(str(data[['field_analysis']]))
|
||||
Loading…
Reference in a new issue