# 
rm(list =ls())

# load libraries ----
requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',
                     'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw')
for(p in requiredPackages){
  if(!require(p,character.only = TRUE)) install.packages(p)
  library(p,character.only = TRUE)
}
# set gams path ----
igdx("d:/gams/win64/34.3")

# load functions----
source("D:/data/fadn/lieferung_20210414/yang/FadntoCapri/myfun_fadn.R")

# set FADN project directory ---
CurrentProjectDirectory = "D:/data/fadn/lieferung_20210414/yang/fadn_work_space"

# ceate a data.dir
create.data.dir(folder.path = CurrentProjectDirectory)

# Once the data.dir is created, we must declare that we are working with it
set.data.dir(CurrentProjectDirectory)

rds.dir = paste0(get.data.dir(),"/rds/")

EU_list <- c("AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK",
        "EST", "FIN", "FRA", "DEU", "GRC", "HUN", "IRL",
        "ITA", "LVA", "LTU", "LUX", "MLT", "NLD", "POL",
        "PRT", "ROU", "SVK", "SVN", "ESP", "SWE")

# crops -----
# convert and load FADN data, save the str data in path: D:\data\fadn\lieferung_20210414\yang\fadn_work_space\rds\crops
# DEU and NED took 106s
# only DEU took 77s
fadn.str.data <- convert.load.str.crops(countries ="DEU")

# filter the crops data 
fadn.str.crops <- fadn.str.data$crops




# Export crops gdx file ----
    
# step 1: Convert str data to NUTS 2016 ---

fadn.str.crops <-  fadnUtils::NUTS.convert.all(fadn.str.crops,"DEU", 2016)

# filter LEVL 
fadn.str.crops.levl <- fadn.str.crops %>%
  mutate(ORGANIC=case_when(
    ORGANIC=="org-2" ~ "Organic",
    TRUE ~ "Conventional"))%>%
  mutate(value2 = WEIGHT*VALUE) %>%
  filter(VARIABLE == "LEVL" ) %>%
  select(-(TF8:SIZ6),-WEIGHT, -ALTITUDE)





# step 2: export a gdx 
# export DEU gdx took 1.25 mins
time.begin <- Sys.time()
export.crops = TRUE

if(export.crops){
  

group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final", "NUTS2_final", "EU")
crops.groupby <-  lapply(seq_along(group_by_lst), 
                         function(i) fadn.filter(fadn.str.crops.levl,
                                                 group_by_lst[i],
                                                 "CROP")) %>% 
  bind_rows() %>% 
  mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
                              REG_TYPE == "REGION" ~ "FADN_REGION",
                              REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
                              REG_TYPE =="NUTS2"~ "NUTS2_ORG",
                              REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
                              REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",
                              TRUE ~ REG_TYPE)) %>% 
  mutate_if(is.factor, as.character)



col_names_crops <- colnames(crops.groupby)
# library(gdxrrw)
# igdx("d:/gams/win64/34.3")
cat("Export the gdx: ", getwd(), "/gdx/crops_LEVL_new.gdx",sep = "")

# write gdx: levl: value*WEIGHT
writegdx(dt = crops.groupby,
         gdx = paste0(getwd(), "/gdx/crops_LEVL_new.gdx"),
         name = "DataOut",
         valcol= "sum_Value",
         uelcols= col_names_crops[!col_names_crops %in% "sum_Value"],
         type="parameter")



}
time.diff <- Sys.time() - time.begin

# animals ----

convert.str.animal = FALSE 
if (convert.str.animal){
  
  system.time(source("D:/data/fadn/lieferung_20210414/yang/FadntoCapri/animals.R"))
  
} 
# load animals str data 
# loading the animals data took 316.89s
system.time(fadn.str.animals.df <- readRDS(paste0(get.data.dir(),"/rds/str_dir/fadn.str.animal.rds")))


# Convert str data to NUTS 2016 ---
# 
fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.str.animals.df,"all",2016)    

fadn.str.animal.an <- fadn.str.animals %>%
  mutate(ORGANIC=case_when(
    ORGANIC=="org-2" ~ "Organic",
    TRUE ~ "Conventional"
  ),YEAR=as.factor(YEAR)) %>%
  mutate(value2 = WEIGHT*value) %>%
  filter(variable=="AN") %>% 
  rename(VARIABLE= variable) %>% 
  select(-(TF8:SIZ6),-WEIGHT)

export.animals = TRUE
system.time({
  
  group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final", "NUTS2_final", "EU")
  animals.groupby <-  lapply(seq_along(group_by_lst), 
                           function(i) fadn.filter(fadn.str.animal.an,
                                                   group_by_lst[i],
                                                   "ANIM")) %>% 
    bind_rows() %>% 
    mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
                                REG_TYPE == "REGION" ~ "FADN_REGION",
                                REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
                                REG_TYPE =="NUTS2"~ "NUTS2_ORG",
                                REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
                                REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",
                                TRUE ~ REG_TYPE)) %>% 
    mutate_if(is.factor, as.character)
  
  
  
  col_names_animals <- colnames(animals.groupby)
  # library(gdxrrw)
  # igdx("d:/gams/win64/34.3")
  cat("Export gdx: ",paste0(getwd(), "/gdx/animals_LEVL_converted.gdx") )
  # write gdx: levl: value*WEIGHT
  writegdx(dt = animals.groupby,
           gdx = paste0(getwd(), "/gdx/animals_LEVL_converted.gdx"),
           name = "DataOut",
           valcol= "sum_Value",
           uelcols= col_names_animals[!col_names_animals %in% "sum_Value"],
           type="parameter")
  
  
  
}) #10.75s