241 lines
8.5 KiB
R
241 lines
8.5 KiB
R
# ============================================================================
|
||
# SCRIPT 21: Convert CI RDS to CSV (Python Compatibility Format)
|
||
# ============================================================================
|
||
# PURPOSE:
|
||
# Convert consolidated CI data from R's wide format (RDS) to Python-compatible
|
||
# long format (CSV). Prepares per-field CI time series for harvest detection
|
||
# models and Python ML workflows without requiring interpolated/modeled values.
|
||
#
|
||
# INPUT DATA:
|
||
# - Source: laravel_app/storage/app/{project}/combined_CI/combined_CI_data.rds
|
||
# - Format: RDS (wide format: fields × dates with CI values)
|
||
# - Requirement: Script 20 must have completed CI extraction
|
||
#
|
||
# OUTPUT DATA:
|
||
# - Destination: laravel_app/storage/app/{project}/Data/extracted_ci/cumulative_vals/
|
||
# - Format: CSV (long format)
|
||
# - Columns: field, sub_field, Date, FitData, DOY, value
|
||
#
|
||
# USAGE:
|
||
# Rscript 21_convert_ci_rds_to_csv.R [project]
|
||
#
|
||
# Example (Windows PowerShell):
|
||
# & "C:\Program Files\R\R-4.4.3\bin\x64\Rscript.exe" r_app/21_convert_ci_rds_to_csv.R angata
|
||
#
|
||
# PARAMETERS:
|
||
# - project: Project name (character) - angata, chemba, xinavane, esa, simba
|
||
#
|
||
# CLIENT TYPES:
|
||
# - cane_supply (ANGATA): Yes - data export
|
||
# - agronomic_support (AURA): Yes - Python ML integration
|
||
#
|
||
# DEPENDENCIES:
|
||
# - Packages: tidyverse, lubridate, zoo
|
||
# - Utils files: parameters_project.R, 00_common_utils.R
|
||
# - Input data: combined_CI_data.rds from Script 20
|
||
# - Data directories: extracted_ci/cumulative_vals/
|
||
#
|
||
# NOTES:
|
||
# - Transformation: Wide format (fields as rows, dates as columns) → Long format
|
||
# - Time series: Preserves all CI values without interpolation
|
||
# - DOY (Day of Year): Calculated from date for seasonal analysis
|
||
# - Python integration: CSV format compatible with pandas/scikit-learn workflows
|
||
# - Used by: Python harvest detection models (harvest_date_prediction.py)
|
||
# - Optional: Run only when exporting to Python for ML model training
|
||
#
|
||
# RELATED ISSUES:
|
||
# SC-112: Utilities restructuring
|
||
# SC-108: Core pipeline improvements
|
||
#
|
||
# ============================================================================
|
||
|
||
suppressPackageStartupMessages({
|
||
library(tidyverse)
|
||
library(lubridate)
|
||
library(zoo)
|
||
library(here)
|
||
})
|
||
|
||
# ============================================================================
|
||
# HELPER FUNCTIONS
|
||
# ============================================================================
|
||
|
||
#' Convert wide format RDS to long format
|
||
#'
|
||
#' @param ci_data_wide Tibble with columns: field, sub_field, and dates as columns
|
||
#' @return Long format tibble: field, sub_field, Date, FitData
|
||
wide_to_long_ci_data <- function(ci_data_wide) {
|
||
ci_data_wide %>%
|
||
pivot_longer(
|
||
cols = -c(field, sub_field),
|
||
names_to = "Date",
|
||
values_to = "FitData",
|
||
values_drop_na = TRUE
|
||
) %>%
|
||
mutate(
|
||
Date = as.Date(Date),
|
||
FitData = as.numeric(FitData)
|
||
) %>%
|
||
filter(!is.na(FitData))
|
||
}
|
||
|
||
#' Create daily interpolated sequences with DOY for each field
|
||
#'
|
||
#' For each field/sub_field combination, creates complete daily sequences from first to last date,
|
||
#' fills in measurements, and interpolates missing dates.
|
||
#'
|
||
#' @param ci_data_long Long format tibble: field, sub_field, Date, FitData
|
||
#' @return Tibble with: field, sub_field, Date, FitData, DOY, value
|
||
create_interpolated_daily_sequences <- function(ci_data_long) {
|
||
ci_data_long %>%
|
||
group_by(field, sub_field) %>%
|
||
nest() %>%
|
||
mutate(
|
||
data = map(data, function(df) {
|
||
# Sort measurements by date
|
||
df <- df %>% arrange(Date)
|
||
|
||
# Create complete daily sequence from first to last date
|
||
date_seq <- seq(min(df$Date), max(df$Date), by = "day")
|
||
|
||
# Build daily dataframe (field/sub_field stay in outer df, not here)
|
||
daily_df <- tibble(
|
||
Date = date_seq,
|
||
value = NA_real_,
|
||
FitData = NA_real_,
|
||
DOY = seq_along(date_seq) # Continuous day counter: 1, 2, 3, ...
|
||
)
|
||
|
||
# Fill in actual measurement values
|
||
for (i in seq_len(nrow(df))) {
|
||
idx <- which(daily_df$Date == df$Date[i])
|
||
if (length(idx) > 0) {
|
||
daily_df$value[idx] <- df$FitData[i]
|
||
}
|
||
}
|
||
|
||
# Interpolate missing dates linearly
|
||
daily_df$FitData <- zoo::na.approx(daily_df$value, na.rm = FALSE)
|
||
|
||
daily_df
|
||
})
|
||
) %>%
|
||
unnest(data) %>%
|
||
select(field, sub_field, Date, FitData, DOY, value) %>%
|
||
arrange(field, Date)
|
||
}
|
||
|
||
#' Validate conversion output
|
||
#'
|
||
#' @param ci_data_python Tibble with converted CI data
|
||
#' @return Invisibly returns the tibble (for piping)
|
||
validate_conversion_output <- function(ci_data_python) {
|
||
cat(sprintf("\nValidation:\n"))
|
||
cat(sprintf(" Unique fields: %d\n", n_distinct(ci_data_python$field)))
|
||
cat(sprintf(" Total daily rows: %d\n", nrow(ci_data_python)))
|
||
cat(sprintf(" Date range: %s to %s\n",
|
||
min(ci_data_python$Date, na.rm = TRUE),
|
||
max(ci_data_python$Date, na.rm = TRUE)))
|
||
cat(sprintf(" FitData range: %.2f to %.2f\n",
|
||
min(ci_data_python$FitData, na.rm = TRUE),
|
||
max(ci_data_python$FitData, na.rm = TRUE)))
|
||
cat(sprintf(" Raw measurements: %d\n", sum(!is.na(ci_data_python$value))))
|
||
cat(sprintf(" Interpolated values: %d\n", sum(is.na(ci_data_python$value) & !is.na(ci_data_python$FitData))))
|
||
|
||
invisible(ci_data_python)
|
||
}
|
||
|
||
#' Print next steps message
|
||
print_next_steps <- function() {
|
||
cat("\nNext steps for Python harvest detection:\n")
|
||
cat(" 1. Read this CSV file in Python\n")
|
||
cat(" 2. Group by field to identify seasons\n")
|
||
cat(" 3. Run LSTM model to detect harvest dates\n")
|
||
cat(" 4. Save predicted harvest dates to Excel\n")
|
||
cat(" 5. Use output in script 03 for interpolation\n")
|
||
}
|
||
|
||
# ============================================================================
|
||
# MAIN FUNCTION
|
||
# ============================================================================
|
||
|
||
main <- function() {
|
||
# Process command line arguments
|
||
args <- commandArgs(trailingOnly = TRUE)
|
||
|
||
# Get project directory
|
||
if (length(args) >= 1 && !is.na(args[1])) {
|
||
project_dir <- as.character(args[1])
|
||
} else if (exists("project_dir", envir = .GlobalEnv)) {
|
||
project_dir <- get("project_dir", envir = .GlobalEnv)
|
||
} else {
|
||
project_dir <- "angata"
|
||
}
|
||
|
||
# Make available globally
|
||
assign("project_dir", project_dir, envir = .GlobalEnv)
|
||
|
||
cat(sprintf("Converting CI RDS to CSV: project=%s\n", project_dir))
|
||
|
||
# Initialize project configuration and centralized paths
|
||
tryCatch({
|
||
source("parameters_project.R")
|
||
}, error = function(e) {
|
||
warning("Default parameters_project.R not found. Attempting from 'r_app' directory.")
|
||
tryCatch({
|
||
source(here::here("r_app", "parameters_project.R"))
|
||
}, error = function(e) {
|
||
stop("Failed to source parameters_project.R from both default and 'r_app' directories.")
|
||
})
|
||
})
|
||
|
||
# Load centralized path structure (creates all directories automatically)
|
||
paths <- setup_project_directories(project_dir)
|
||
|
||
# Use centralized paths (no need for dir.create - already handled)
|
||
ci_data_source_dir <- paths$cumulative_ci_vals_dir
|
||
ci_data_output_dir <- paths$ci_for_python_dir
|
||
|
||
input_file <- file.path(ci_data_source_dir, "combined_CI_data.rds")
|
||
output_file <- file.path(ci_data_output_dir, "ci_data_for_python.csv")
|
||
|
||
# Check if input file exists
|
||
if (!file.exists(input_file)) {
|
||
stop(paste("Input file not found:", input_file))
|
||
}
|
||
|
||
cat(sprintf("Loading: %s\n", input_file))
|
||
|
||
# Load RDS file
|
||
ci_data_wide <- readRDS(input_file) %>%
|
||
as_tibble()
|
||
|
||
cat(sprintf(" Loaded %d rows\n", nrow(ci_data_wide)))
|
||
cat(sprintf(" Format: WIDE (field, sub_field, then dates as columns)\n"))
|
||
cat(sprintf(" Sample columns: %s\n", paste(names(ci_data_wide)[1:6], collapse = ", ")))
|
||
|
||
# Step 1: Convert from WIDE to LONG format
|
||
cat("\nStep 1: Converting from wide to long format...\n")
|
||
ci_data_long <- wide_to_long_ci_data(ci_data_wide)
|
||
|
||
# Step 2: Create complete daily sequences with interpolation
|
||
cat("\nStep 2: Creating complete daily sequences with interpolation...\n")
|
||
ci_data_python <- create_interpolated_daily_sequences(ci_data_long)
|
||
|
||
# Step 3: Validate output
|
||
cat("\nStep 3: Validating output...")
|
||
validate_conversion_output(ci_data_python)
|
||
|
||
# Step 4: Save to CSV
|
||
cat(sprintf("\nStep 4: Saving to CSV...\n"))
|
||
cat(sprintf(" Output: %s\n", output_file))
|
||
write_csv(ci_data_python, output_file)
|
||
|
||
cat(sprintf("\n✓ Successfully created CSV with %d rows\n", nrow(ci_data_python)))
|
||
print_next_steps()
|
||
}
|
||
|
||
if (sys.nframe() == 0) {
|
||
main()
|
||
}
|