#source(" ..\R\fadntocapri\myfun_fadn.R ")
#---------------------------------
#          ---   functions
#---------------------------------

# group by different NUTS with YEAR, CROP/ANIM, VARIABLE,ORGANIC

fadn.filter <- function(data, group.by, type ) {
  if (group.by == "EU"){
    filtered <- data %>% filter(COUNTRY %in% EU_list) %>%
      group_by(YEAR,.data[[type]],ORGANIC,VARIABLE) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%
      as.data.table() %>%
      mutate(REGION = group.by,
             REG_TYPE = group.by)
  } else{
    filtered <- data %>%
      group_by(.data[[group.by]],
               YEAR,
               .data[[type]],
               VARIABLE,
               ORGANIC) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%
      as.data.table() %>%
      rename(REGION = .data[[group.by]]) %>%
      mutate(REG_TYPE = group.by)
  }
  return(filtered)
}

# convert csv in raw rds
convert.raw <- function(countries){
  csv_file_names <- list.files(path = fadn.data.dir, pattern= "*.csv$")
  if(countries == "all"){
    csv_selected = csv_file_names
  }else{
    toMacth.countries = paste(countries,collapse="|")
    csv_selected = grep(toMacth.countries, csv_file_names, value = TRUE)
  }

  cat("Create fadn.raw.rds files for: ", countries,"\n")
  for (file in csv_selected){
    # extract 4-7 char
    year = substr(file, 4, 7)
    countries = substr(file, 1, 3)
    convert.to.fadn.raw.rds(
      file.path = paste0(fadn.data.dir,file),
      sepS = ",",
      fadn.country = countries,
      fadn.year = year
      #keep.csv = T # copy csv file in csv.dir
    )
  }
}
# load fadn raw data
load.raw <- function(countries){

  # check if raw data exist
  fadn.raw.rds.avail <- get.available.fadn.raw.rds()[COUNTRY %in% countries]
  if(nrow(fadn.raw.rds.avail)==0){
    cat("Raw data does not exist, converting raw data ...\n")
    convert.raw(countries = countries)
  }
  
  # if (countries == "all") {countries = EU_list}
  # raw.rds.avail <- get.available.fadn.raw.rds()[COUNTRY %in% countries ]


  
  # countries.avail <- unique( raw.rds.avail$COUNTRY)
  raw_data <- load.fadn.raw.rds(countries = countries,years = "all")

  return(raw_data)
}
# convert raw rds in str data 
convert.str <- function(countries){
  # Convert FADN data, save the str data in path: ../output/restart/fadn/
  before2013.json = "../r/fadntocapri/corrected.json.full/corrected.2013_before.json"
  after2014.json = "../r/fadntocapri/corrected.json.full/corrected.2014_after.json"
  #Check if the str data already exists
  # extr.dirs = list.dirs(path = paste0(get.data.dir(),"/rds"), full.names = F, recursive = F)
  # extr.dirs.full = paste0(rds.dir,extr.dirs)
  # list.files(extr.dirs.full, pattern = paste0(fadn.countries,".rds") )

  if ( "all" %in% countries) {
    beforeyears = "before2013"
    afteryears = "after2014"
    # all countries and years 719.24s
    # convert raw data to structured data ---
    # before 2013 and 2013
    convert.to.fadn.str.rds(countries,
                            beforeyears,
                            raw_str_map.file = before2013.json,
                            str.name = "forcapri",
                            force_external_raw_str_map = T)# 413.25 for all countries
    # after 2014 and 2014
    convert.to.fadn.str.rds(countries,
                            afteryears,
                            raw_str_map.file = after2014.json,
                            str.name = "forcapri",
                            force_external_raw_str_map = T)# 305.99 for all countries
    }else{
    beforeyears = c(2004:2013)
    afteryears = c(2014:2018)
    # before 2014
    # only DEU 84s
    # BEL and DEU 107.26s
    for (country in countries ){

      sapply(seq_along(beforeyears), function(i)
        convert.to.fadn.str.rds(country,
                                beforeyears[i],
                                raw_str_map.file = after2014.json,
                                str.name = "forcapri",
                                force_external_raw_str_map = T) )
      # after 2013
      sapply(seq_along(afteryears), function(i)
        convert.to.fadn.str.rds(country,
                                afteryears[i],
                                raw_str_map.file = after2014.json,
                                str.name = "forcapri",
                                force_external_raw_str_map = T) )
    }
  }
}
# load str crops data
load.str <- function(countries ) {

    # check if str data exist
  fadn.str.rds.avail <- get.available.fadn.str.rds(extract_dir = "forcapri")[COUNTRY %in% countries]
  if(nrow(fadn.str.rds.avail)==0){
    cat("Str data does not exist, converting str data ...\n")
    convert.str(countries = countries)
  }
  # load crops str data
  fadn.str.data <- load.fadn.str.rds("forcapri",countries,"all")
  # fadn.str.cro <- fadn.str.data[[filter]]