##########
#Task 2a) get the data and store it and bind the rows so we get a dataset

library(httr)
library(dplyr)
library(jsonlite)





rm(list=ls())

#headers are set for the API request
headers <- c("Content-Type" = 'application/x-www-form-urlencoded')
auth_data <- list(
  client_id = 'c003a37f-024f-462a-b36d-b001be4cd24a',
  client_secret = '32a39620-32b3-4307-9aa1-511e3d7f48a8',
  grant_type = 'client_credentials'
)

# Post request to fetch access token
res <- POST(url = 'https://rest.arbeitsagentur.de/oauth/gettoken_cc',
            add_headers(.headers = headers),
            body = auth_data,
            encode = 'form')
token <- content(res)$access_token



base_url = "https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v4/jobs"
params <- list(
  "was" = "Daten",
  "wo" = "Stuttgart",
  size = 200,
  "umkreis" = "100"
)


initial_req <- GET(url = base_url, add_headers(OAuthAccessToken = token, Accept = "application/json"), query = params)
initial_data <- jsonlite::fromJSON(rawToChar(initial_req$content))
total_offers <- initial_data$maxErgebnisse
total_pages <- ceiling(total_offers / 200)


# List to hold each page's data frame
all_pages_data <- list()

# Loop through each page and fetch data
for (page in 1:total_pages) {
  params$page <- page
  response <- GET(url = base_url, add_headers(OAuthAccessToken = token, Accept = "application/json"), query = params)
  page_data <- jsonlite::fromJSON(rawToChar(response$content))
  all_pages_data[[page]] <- as.data.frame(page_data$stellenangebote)
}


# Combine all pages into one data frame
job_postings <- bind_rows(all_pages_data)

# Show structure of the combined data frame
str(job_postings)

