# group by different NUTS with YEAR, CROP/ANIM, VARIABLE,ORGANIC
fadn.filter <- function(data, group.by, type ) {

  # filtered <- data %>%
  #   group_by({{group.by}},YEAR,{{type}},VARIABLE,ORGANIC) %>%
  #   summarise(sum_Value = sum(value2), .groups ="drop") %>%
  #   as.data.table() %>% rename(REGION = {{group.by}})

  if (group.by == "EU"){
    filtered <- data %>% filter(COUNTRY %in% EU_list) %>%
      group_by(YEAR,.data[[type]],ORGANIC,VARIABLE) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%

      as.data.table() %>%

      mutate(REGION = group.by,
             REG_TYPE = group.by)

  } else{

    filtered <- data %>%
      group_by(.data[[group.by]],
               YEAR,
               .data[[type]],
               VARIABLE,
               ORGANIC) %>%
      summarise(sum_Value = sum(value2), .groups ="drop") %>%
      as.data.table() %>%
      rename(REGION = .data[[group.by]]) %>%
      mutate(REG_TYPE = group.by)

  }

  return(filtered)

}


convert.load.str.crops <- function(countries ) {
  before2013.json = paste0(getwd(), "/corrected.json.full/corrected.2013_before.json")
  after2014.json = paste0(getwd(), "/corrected.json.full/corrected.2014_after.json")

  if ( "all" %in% countries) {
    beforeyears = "before2013"
    afteryears = "after2014"
    # all countries and years 719.24s
    # convert raw data to structured data ---
    # before 2013 and 2013
    convert.to.fadn.str.rds(countries,
                            beforeyears,
                            raw_str_map.file = before2013.json,

                            str.name = "alex",

                            force_external_raw_str_map = T)# 413.25 for all countries

    # after 2014 and 2014
    convert.to.fadn.str.rds(countries,
                            afteryears,
                            raw_str_map.file = after2014.json,

                            str.name = "alex",

                            force_external_raw_str_map = T)# 305.99 for all countries

    # load

    after2014 <- readRDS(paste0(rds.dir, "/alex/fadn.str.after2014.all.rds"))
    before2013 <- readRDS(paste0(rds.dir, "/alex/fadn.str.before2013.all.rds"))

    fadn.str.crops <- bind_rows(before2013$crops,after2014$crops)

    }
  else{
    beforeyears = c(2004:2013)
    afteryears = c(2014:2018)
    # before 2014
    # only DEU 84s
    # BEL and DEU 107.26s
    for (country in countries ){

      sapply(seq_along(beforeyears), function(i)

        convert.to.fadn.str.rds(country,
                                beforeyears[i],
                                raw_str_map.file = after2014.json,

                                str.name = "alex",

                                force_external_raw_str_map = T) )
      # after 2013

      sapply(seq_along(afteryears), function(i)

        convert.to.fadn.str.rds(country,
                                afteryears[i],
                                raw_str_map.file = after2014.json,

                                str.name = "alex",

                                force_external_raw_str_map = T) )

    }

    # load crops str data

    fadn.str.data <- load.fadn.str.rds("alex",countries,"all")
    fadn.str.crops <- fadn.str.data$crops

  }

  return(fadn.str.crops)

}

# get animals data

get.ifm_cap.animals = function(column="AN", years.eff=2010:2013, data.cur=TABLE_J.all) {

  #Livestock, number of animals
  cols = c("ID","YEAR","ANIM",column)
  tmp1 = data.table::dcast(data.cur[YEAR%in%years.eff,..cols],ID+YEAR~ANIM,value.var=column,fill=0)
  tmp1 = merge(tmp1,BOV1_PERC,all.x=T,by="ID")[is.na(LBOV0.perc),LBOV0.perc:=0]
  # setnames(tmp1,"ID","FD")

  #check no columns are missing. If yes, create one
  cols.used = c("LBOVFAT","LBOV0","LHEIFBRE","LHEIFFAT","LBOV1_2F","LCOWBUFDAIR","LCOWOTH","LEWEBRE","LGOATBRE","LSHEPOTH","LGOATOTH","LSOWBRE", "LPIGFAT","LPIGOTH","LPLTRBROYL","LPLTROTH","LHENSLAY","LEQD","LBOV1_2M","LBOV2","LRABBRE")
  for(col.used in cols.used) {
    if(!col.used%in%names(tmp1)){
      warning(paste0("nCreating column ",col.used))
      tmp1[,(col.used):=0]
    }
  }

  #If LBOV0+LBOVFAT are not present and LBOV1 is present, calculate the share
  tmp1[LBOV1>0 & LBOVFAT==0 & LBOV0==0,":="(LBOV0=LBOV0.perc*LBOV1,LBOVFAT=(1-LBOV0.perc)*LBOV1)]

  tmp2 = tmp1[,.(

    ID,

    YEAR,
    variable=column,
    CAMF = pmax(0.5*LBOVFAT),
    CAFF = pmax(0.5*LBOVFAT),
    CAMR = pmax(0,LBOV0-LHEIFBRE),
    CAFR = pmin(LHEIFBRE,LBOV0),
    HEIF = LHEIFFAT+pmax(0,LBOV1_2F-LHEIFBRE),
    BULF = LBOV1_2M+LBOV2,
    HEIR = LHEIFBRE+pmin(LBOV1_2F,LHEIFBRE),
    DCOW = LCOWBUFDAIR+LCOWCUL+LCOWDAIR+LBUFDAIRPRS,
    SCOW = LCOWOTH,
    SHGM = LEWEBRE+ LGOATBRE,
    SHGF = LSHEPOTH + LGOATOTH,
    SOWS = LSOWBRE ,
    PIGF = LPIGFAT	+ LPIGOTH,
    POUF = LPLTRBROYL	+ LPLTROTH,
    HENS = LHENSLAY	,
    OANI = LRABBRE+LEQD    #LANIMOTH is not there
  )]

  return(
    melt(tmp2,id.vars = c("ID","YEAR","variable"),variable.name = "ANIM")[]
  )