# group by different NUTS with YEAR, CROP/ANIM, VARIABLE,ORGANIC
fadn.filter <- function(data, group.by, type ) {

  # filtered <- data %>%
  #   group_by({{group.by}},YEAR,{{type}},VARIABLE,ORGANIC) %>%
  #   summarise(sum_Value = sum(value2), .groups ="drop") %>%
  #   as.data.table() %>% rename(REGION = {{group.by}})

  if (group.by == "EU"){
    filtered <- data %>% filter(COUNTRY %in% EU_list) %>%
      group_by(YEAR,.data[[type]],ORGANIC,VARIABLE) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%

      as.data.table() %>%

      mutate(REGION = group.by,
             REG_TYPE = group.by)

  } else{

    filtered <- data %>%
      group_by(.data[[group.by]],
               YEAR,
               .data[[type]],
               VARIABLE,
               ORGANIC) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%
      as.data.table() %>%
      rename(REGION = .data[[group.by]]) %>%
      mutate(REG_TYPE = group.by)

  }

  return(filtered)

}

convert.raw <- function(countries){
  
  csv_file_names <- list.files(path = fadn.data.dir, pattern= "*.csv$")
  
  if(countries == "all"){
    csv_selected = csv_file_names
  }else{
    toMacth.countries = paste(countries,collapse="|")
    csv_selected = grep(toMacth.countries, csv_file_names, value = TRUE)
  }
  cat("Create fadn.raw.rds files for: ", countries,"\n")
  
  for (file in csv_selected){
    # extract 4-7 char
    year = substr(file, 4, 7)
    countries = substr(file, 1, 3)

    convert.to.fadn.raw.rds(
      file.path = paste0(fadn.data.dir,file),
      sepS = ",",
      fadn.country = countries,
      fadn.year = year
      #keep.csv = T # copy csv file in csv.dir
    )

  }

}

load.raw <- function(countries){
  if (countries == "all") {countries = EU_list}
  raw.rds.avail <- get.available.fadn.raw.rds()[COUNTRY %in% countries ]
  countries.avail <- unique( raw.rds.avail$COUNTRY)
  load.fadn.raw.rds(countries = countries.avail,years = "all")
}

convert.str.crops <- function(countries){

  # Convert FADN data, save the str data in path: ../output/restart/fadn/

 
  before2013.json = "../r/fadntocapri/corrected.json.full/corrected.2013_before.json"
  after2014.json = "../r/fadntocapri/corrected.json.full/corrected.2014_after.json"
  #Check if the str data already exists
  # extr.dirs = list.dirs(path = paste0(get.data.dir(),"/rds"), full.names = F, recursive = F)
  # extr.dirs.full = paste0(rds.dir,extr.dirs)
  # list.files(extr.dirs.full, pattern = paste0(fadn.countries,".rds") )
  

  if ( "all" %in% countries) {
    beforeyears = "before2013"
    afteryears = "after2014"
    # all countries and years 719.24s
    # convert raw data to structured data ---
    # before 2013 and 2013
    convert.to.fadn.str.rds(countries,
                            beforeyears,
                            raw_str_map.file = before2013.json,

                            str.name = "alex",

                            force_external_raw_str_map = T)# 413.25 for all countries

    # after 2014 and 2014
    convert.to.fadn.str.rds(countries,
                            afteryears,
                            raw_str_map.file = after2014.json,

                            str.name = "alex",

                            force_external_raw_str_map = T)# 305.99 for all countries

    
    }else{

    beforeyears = c(2004:2013)
    afteryears = c(2014:2018)
    # before 2014
    # only DEU 84s
    # BEL and DEU 107.26s
    for (country in countries ){

      sapply(seq_along(beforeyears), function(i)

        convert.to.fadn.str.rds(country,
                                beforeyears[i],
                                raw_str_map.file = after2014.json,

                                str.name = "alex",

                                force_external_raw_str_map = T) )
      # after 2013

      sapply(seq_along(afteryears), function(i)

        convert.to.fadn.str.rds(country,
                                afteryears[i],
                                raw_str_map.file = after2014.json,

                                str.name = "alex",

                                force_external_raw_str_map = T) )

    }

}

}
## year specific translation from FADN to CAPRI code
load.str.crops <- function(countries ) {
  
  # load crops str data
  fadn.str.data <- load.fadn.str.rds("alex",countries,"all")
  fadn.str.crops <- fadn.str.data$crops

  

  return(fadn.str.crops)

}

# get animals data

get.ifm_cap.animals = function(column="AN", years.eff=2010:2013, data.cur=TABLE_J.all) {

  #Livestock, number of animals
  cols = c("ID","YEAR","ANIM",column)
  tmp1 = data.table::dcast(data.cur[YEAR%in%years.eff,..cols],ID+YEAR~ANIM,value.var=column,fill=0)
  tmp1 = merge(tmp1,BOV1_PERC,all.x=T,by="ID")[is.na(LBOV0.perc),LBOV0.perc:=0]
  # setnames(tmp1,"ID","FD")

  #check no columns are missing. If yes, create one

  cols.used = c("LBOVFAT","LBOV0","LHEIFBRE","LHEIFFAT","LBOV1_2F","LCOWDAIR","LBUFDAIRPRS",
  "LCOWBUFDAIR","LCOWOTH","LEWEBRE","LGOATBRE","LSHEPOTH","LGOATOTH","LSOWBRE", 
  "LPIGFAT","LPIGOTH","LPLTRBROYL","LPLTROTH","LHENSLAY","LEQD","LBOV1_2M","LBOV2","LRABBRE")

  for(col.used in cols.used) {
    if(!col.used%in%names(tmp1)){
      warning(paste0("nCreating column ",col.used))
      tmp1[,(col.used):=0]
    }
  }

  cat("column",column)

  #If LBOV0+LBOVFAT are not present and LBOV1 is present, calculate the share
  tmp1[LBOV1>0 & LBOVFAT==0 & LBOV0==0,":="(LBOV0=LBOV0.perc*LBOV1,LBOVFAT=(1-LBOV0.perc)*LBOV1)]

  tmp2 = tmp1[,.(

    ID,

    YEAR,
    variable=column,
    CAMF = pmax(0.5*LBOVFAT),
    CAFF = pmax(0.5*LBOVFAT),
    CAMR = pmax(0,LBOV0-LHEIFBRE),
    CAFR = pmin(LHEIFBRE,LBOV0),
    HEIF = LHEIFFAT+pmax(0,LBOV1_2F-LHEIFBRE),
    BULF = LBOV1_2M+LBOV2,
    HEIR = LHEIFBRE+pmin(LBOV1_2F,LHEIFBRE),
    DCOW = LCOWBUFDAIR+LCOWCUL+LCOWDAIR+LBUFDAIRPRS,
    SCOW = LCOWOTH,
    SHGM = LEWEBRE+ LGOATBRE,
    SHGF = LSHEPOTH + LGOATOTH,
    SOWS = LSOWBRE ,

    PIGF = LPIGFAT         + LPIGOTH,
    POUF = LPLTRBROYL      + LPLTROTH,
    HENS = LHENSLAY        ,

    OANI = LRABBRE+LEQD    #LANIMOTH is not there
  )]

  return(
    melt(tmp2,id.vars = c("ID","YEAR","variable"),variable.name = "ANIM")[]
  )