#######################################################################################################
####

library(dplyr)
library(lubridate)
library(stringr)

get_active_jobs <- function(subfolder) {
  # Get all relevant files, excluding today's files
  all_files <- list.files(subfolder, full.names = TRUE, pattern = "\\.csv$")
  today <- Sys.Date()
  relevant_files <- all_files[!grepl(as.character(today), all_files)]
  
  # Find the latest 'full' dataset and subsequent incremental files
  full_files <- relevant_files[grepl("_full.csv", relevant_files)]
  if (length(full_files) == 0) {
    stop("No full dataset available.")
  }
  dates <- sapply(full_files, function(x) as.Date(str_extract(x, "\\d{4}-\\d{2}-\\d{2}")))
  latest_full_file <- full_files[which.max(dates)]
  latest_full_date <- max(dates)
  
  # Load the most recent full dataset
  current_jobs <- read.csv(latest_full_file)
  
  # Apply all subsequent updates
  updates <- relevant_files[grepl("_new.csv|_old.csv", relevant_files) & sapply(relevant_files, function(x) as.Date(str_extract(x, "\\d{4}-\\d{2}-\\d{2}")) > latest_full_date)]
  for (file in updates) {
    update_data <- read.csv(file)
    if (grepl("_new.csv", file)) {
      current_jobs <- bind_rows(current_jobs, update_data)
    } else if (grepl("_old.csv", file)) {
      current_jobs <- filter(current_jobs, !refnr %in% update_data$refnr)
    }
  }
  
  return(current_jobs$refnr)  # Return only active job IDs
}


#right now at raw but can also be applied on downloads
active_ids <- get_active_jobs("/Users/alexanderunger/Desktop/AIDAHO_IDS_THAS/01_data/raw")


