#source("/Users/alexanderunger/Desktop/AIDAHO_IDS_THAS/02_code/R/api/docker_file.R")

library(dplyr)
library(purrr)
library(readr)



# Read and filter datasets by active IDs
filter_active_jobs <- function(directory) {
  ids <- active_ids
  
  # Define consistent column types for all data frames
  cols_types <- cols(
    beruf = col_character(),
    titel = col_character(),
    refnr = col_character(),
    arbeitsort.ort = col_character(),
    arbeitsort.region = col_character(),
    arbeitsort.land = col_character(),
    arbeitsort.strasse = col_character(),
    arbeitsort.ortsteil = col_character(),  # Ensure this is character even if logical in some datasets
    arbeitsort.adresszusatz = col_character(),
    studiengang = col_logical(),
    aktuelleVeroeffentlichungsdatum = col_date(),
    eintrittsdatum = col_date(),
    modifikationsTimestamp = col_datetime(),
    arbeitsort.koordinaten.lat = col_double(),
    arbeitsort.koordinaten.lon = col_double(),
    arbeitsort.entfernung = col_double(),
    arbeitsort.plz = col_double()
  )
  
  files <- list.files(path = directory, pattern = "\\.csv$", full.names = TRUE)
  job_dfs <- map(files, ~ read_csv(.x, col_types = cols_types) %>% 
                   filter(refnr %in% ids))
  combined_df <- bind_rows(job_dfs)
  return(as.data.frame(combined_df))
}

# Retrieve jobs by specific date or the most recent available
get_active_jobs_on_date <- function(df, date) {
  df$aktuelleVeroeffentlichungsdatum <- as.Date(df$aktuelleVeroeffentlichungsdatum)
  filtered_df <- filter(df, aktuelleVeroeffentlichungsdatum == as.Date(date))
  
  if (nrow(filtered_df) == 0) {
    max_date <- max(filter(df, aktuelleVeroeffentlichungsdatum < as.Date(date))$aktuelleVeroeffentlichungsdatum)
    filtered_df <- filter(df, aktuelleVeroeffentlichungsdatum == max_date)
  }
  
  return(filtered_df)
}

# Usage (the path might have to be changed)
all_active_jobs <- filter_active_jobs("/Users/alexanderunger/Desktop/AIDAHO_IDS_THAS/01_data/downloads")
jobs_on_date <- get_active_jobs_on_date(all_active_jobs, '2024-06-07')


#gets all the newest jobs



