rm(list =ls())

# Setup -------------------------------------------------------------------

## Install fadnutils from thuenen gitlab
# devtools::install_git("https://git-dmz.thuenen.de/mindstep/fadnutilspackages", force = TRUE)

## load libraries

requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',

                     'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw','Hmisc')

for(p in requiredPackages){
  if(!require(p,character.only = TRUE)) install.packages(p)
  library(p,character.only = TRUE)
}

## set gams path

igdx("d:/gams/win64/34.3")

## Check your current working directory
## Determin if Git directory is current working dir.

getwd()

setwd("D:/data/fadn/lieferung_20210414/yang/FadntoCapri")
## Set FADN project directory

CurrentProjectDirectory = "D:/data/fadn/lieferung_20210414/yang/fadn_work_space"

## Ceate a data.dir

create.data.dir(folder.path = CurrentProjectDirectory)

## Once the data.dir is created, we must declare that we are working with it

set.data.dir(CurrentProjectDirectory)

rds.dir = paste0(get.data.dir(),"/rds/")

## Set EU list

EU_list <- c("AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK",
        "EST", "FIN", "FRA", "DEU", "GRC", "HUN", "IRL",
        "ITA", "LVA", "LTU", "LUX", "MLT", "NLD", "POL",
        "PRT", "ROU", "SVK", "SVN", "ESP", "SWE")

## Load functions
source("D:/data/fadn/lieferung_20210414/yang/FadntoCapri/myfun_fadn.R")


# FADN to CAPRI -----------------------------------------------------------

## FADN Crops  -----
output_GDX_crops <- function (fadn.countries, filter.variable = "LEVL" ){

  # Convert and load FADN data, save the str data in path: D:\data\fadn\lieferung_20210414\yang\fadn_work_space\rds\crops
  cat("Convert and load FADN str data for:", fadn.countries)

  fadn.str.crops <- convert.load.str.crops(countries = fadn.countries)

  cat("Convert FADN str data to NUTS Version 2016 for country:", fadn.countries)
  # Export crops gdx file
  # step 1: Convert str data to NUTS 2016
  fadn.str.crops <-  fadnUtils::NUTS.convert.all(fadn.str.crops,fadn.countries, 2016)

  # filter

  fadn.str.crops.levl <- fadn.str.crops %>%
    mutate(ORGANIC=case_when(
      ORGANIC=="org-2" ~ "Organic",
      TRUE ~ "Conventional"))%>%
    mutate(value2 = WEIGHT*VALUE) %>%

    filter(VARIABLE == filter.variable ) %>%

    select(-(TF8:SIZ6),-WEIGHT, -ALTITUDE)

  # step 2: export a gdx

  # export DEU gdx took 1.25 mins

  group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final", "NUTS2_final", "EU")

  crops.groupby <-  lapply(seq_along(group_by_lst),

                           function(i) fadn.filter(fadn.str.crops.levl,
                                                   group_by_lst[i],

                                                   "CROP")) %>%
    bind_rows() %>%

    mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
                                REG_TYPE == "REGION" ~ "FADN_REGION",
                                REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
                                REG_TYPE =="NUTS2"~ "NUTS2_ORG",
                                REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
                                REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",

                                TRUE ~ REG_TYPE)) %>%

    mutate_if(is.factor, as.character)

  col_names_crops <- colnames(crops.groupby)
  # library(gdxrrw)
  # igdx("d:/gams/win64/34.3")

  # cat("Export the gdx: ", getwd(), "/gdx/crops_LEVL_new.gdx",sep = "")
  cat("Export the gdx: ", getwd(), paste0("/gdx/crops_",fadn.countries,"_", filter.variable,".gdx") ,sep = "")

  # write gdx: levl: value*WEIGHT
  writegdx(dt = crops.groupby,

           gdx = paste0(getwd(), "/gdx/crops_",fadn.countries, "_",filter.variable,".gdx"),

           name = "DataOut",
           valcol= "sum_Value",
           uelcols= col_names_crops[!col_names_crops %in% "sum_Value"],
           type="parameter")

}

## FADN animals ----
output_GDX_animals <- function( convert.str.animal = FALSE, filter.variable = "AN", fadn.countries = "all" ) {
  # load data ----
  if (filter.variable == "AN") {
    cat("Loading animal data ......\n")

    if (convert.str.animal) {
      # converting animal str data took 815.39s
      cat("Converting animal str data ....\n")
      system.time(source("animals.R"))

      fadn.str.animals.df <- animal.dt


    } else{
      # loading the animals data took 316.89s
      fadn.str.animals.df <-
        readRDS(paste0(get.data.dir(), "/rds/str_dir/fadn.str.animal.rds"))
    }

    fadn.animals.dt <- fadn.str.animals.df
  } else {
    cat("Loading animal product data ......\n")
    # load animal production
    if (!exists('ANIMALS.production.all')) {
      cat("Loading animal product data")
      source("total_production_animal.R")
      # ANIMALS.production.all <- fadn.tp.animals()
    } else{
      cat("loaded!\n")
    }
    fadn.animals.dt <- ANIMALS.production.all

  }

  if(!filter.variable %in% fadn.animals.dt$variable) {
    cat(filter.variable, "is not included\n")
    return(filter.variable, "is not included\n")}

  cat("Converting to NUTS 2016 ......\n")

  # Convert str data to NUTS 2016 ---

  fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.animals.dt, fadn.countries, 2016)

  fadn.str.animals.filtered <- fadn.str.animals %>%

    mutate(ORGANIC=case_when(
      ORGANIC=="org-2" ~ "Organic",
      TRUE ~ "Conventional"
    ),YEAR=as.factor(YEAR)) %>%
    mutate(value2 = WEIGHT*value) %>%

    filter(variable==filter.variable) %>%

    rename(VARIABLE= variable) %>%

    select(-(TF8:SIZ6),-WEIGHT)

  group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final","NUTS2_final", "EU")

  animals.groupby <-  lapply(seq_along(group_by_lst),

                             function(i) fadn.filter(fadn.str.animals.filtered,

                                                     group_by_lst[i],

                                                     "ANIM")) %>%
    bind_rows() %>%

    mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
                                REG_TYPE == "REGION" ~ "FADN_REGION",
                                REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
                                REG_TYPE =="NUTS2"~ "NUTS2_ORG",
                                REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
                                REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",

                                TRUE ~ REG_TYPE)) %>%

    mutate_if(is.factor, as.character)

  col_names_animals <- colnames(animals.groupby)

  cat("Export gdx: ",paste0(getwd(), "/gdx/animals_",filter.variable,"_",fadn.countries,"_converted.gdx\n"))

  # write gdx: levl: value*WEIGHT
  writegdx(dt = animals.groupby,

           gdx = paste0(getwd(), "/gdx/animals_",filter.variable,"_",fadn.countries,"_converted.gdx"),

           name = "DataOut",
           valcol= "sum_Value",
           uelcols= col_names_animals[!col_names_animals %in% "sum_Value"],
           type="parameter")

}

## Call functions --------------------------------------------------------
### crops ----

# LEVL----

# only for DEU took 108s

output_GDX_crops(fadn.countries = "DEU", filter.variable = "LEVL")

# NED and ROU took 119.87

output_GDX_crops(fadn.countries = c("DEU","ROU"),filter.variable = "LEVL")

# all countries took 731.40s