# load libraries
requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',
                     'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw','Hmisc')
for(p in requiredPackages){
  if(!require(p,character.only = TRUE)) install.packages(p)
  library(p,character.only = TRUE)
}
```

-   Setting GAMS path

It is prepare for exporting GDX files.

``` r
igdx("d:/gams/win64/34.3")
```

    ## GDX API loaded from gamsSysDir=d:/gams/win64/34.3
    ## The GDX library has been loaded
    ## GDX library load path: d:/gams/win64/34.3

-   Setting FADN project directory and list of Europe country codes

You can set the path for `fadnUtils` package if you have not set, then you can create a new path using `create.data.dir()` command. The European Union has 28 member countries, CAPRI project focuses on EU countries, so we have to set all of the country codes for Europe, including 3-letter codes as below.

``` r
# Set FADN project directory ----
CurrentProjectDirectory = "D:/data/fadn/lieferung_20210414/yang/fadn_work_space"

# Ceate a data.dir
create.data.dir(folder.path = CurrentProjectDirectory)

# Once the data.dir is created, we must declare that we are working with it
set.data.dir(CurrentProjectDirectory)

rds.dir = paste0(get.data.dir(),"/rds/")

# Set EU list
EU_list <- c("AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK",
        "EST", "FIN", "FRA", "DEU", "GRC", "HUN", "IRL",
        "ITA", "LVA", "LTU", "LUX", "MLT", "NLD", "POL",
        "PRT", "ROU", "SVK", "SVN", "ESP", "SWE")
```

-   Loading functions from R script `myfun_fadn.R`

This script is responsible for grouping by different variables (NUTS versions, organic or non-orgnic, etc.), converting the `crops` data and getting `animals` data.

``` r
# Load functions
source("myfun_fadn.R")
```

### `Crops` data for CAPRI

Using `output_GDX_crops` function loads raw data, extracts crops data, computes orgnic and non-org, aggregtes different variables and exports `crops` FADN data.

``` r
output_GDX_crops <- function (fadn.countries ){
  # Convert and load FADN data, save the str data in path: D:\data\fadn\lieferung_20210414\yang\fadn_work_space\rds\crops
  cat("Convert and load FADN str data for:", fadn.countries)
  fadn.str.crops <- convert.load.str.crops(countries = fadn.countries)

  cat("Convert FADN str data to NUTS Version 2016 for country:", fadn.countries)
  # Export crops gdx file
  # step 1: Convert str data to NUTS 2016
  fadn.str.crops <-  fadnUtils::NUTS.convert.all(fadn.str.crops,fadn.countries, 2016)

  # filter LEVL
  fadn.str.crops.levl <- fadn.str.crops %>%
    mutate(ORGANIC=case_when(
      ORGANIC=="org-2" ~ "Organic",
      TRUE ~ "Conventional"))%>%
    mutate(value2 = WEIGHT*VALUE) %>%
    filter(VARIABLE == "LEVL" ) %>%
    select(-(TF8:SIZ6),-WEIGHT, -ALTITUDE)
  # step 2: export a gdx
  # export DEU gdx took 1.25 mins

  group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final", "NUTS2_final", "EU")
  crops.groupby <-  lapply(seq_along(group_by_lst),
                           function(i) fadn.filter(fadn.str.crops.levl,
                                                   group_by_lst[i],
                                                   "CROP")) %>%
    bind_rows() %>%
    mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
                                REG_TYPE == "REGION" ~ "FADN_REGION",
                                REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
                                REG_TYPE =="NUTS2"~ "NUTS2_ORG",
                                REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
                                REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",
                                TRUE ~ REG_TYPE)) %>%
    mutate_if(is.factor, as.character)



  col_names_crops <- colnames(crops.groupby)
  # library(gdxrrw)
  # igdx("d:/gams/win64/34.3")
  cat("Export the gdx: ", getwd(), "/gdx/crops_LEVL_new.gdx",sep = "")

  # write gdx: levl: value*WEIGHT
  writegdx(dt = crops.groupby,
           gdx = paste0(getwd(), "/gdx/crops_LEVL_new.gdx"),
           name = "DataOut",
           valcol= "sum_Value",
           uelcols= col_names_crops[!col_names_crops %in% "sum_Value"],
           type="parameter")
}
```

Call funtion to export GDX files for crops of EU country.

``` r
output_GDX_crops(fadn.countries = "all")
```

### `Animals` data for CAPRI

All the information about `animals` comes from table J (collection of tables are provided by FADN). `output_GDX_animals` includes loading raw animals data from FADN data, cleaning raw data, and exporting final animals data as GDX format.

``` r
output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "all" ) {
  if (convert.str.animal){
    # converting animal str data took 815.39s
    system.time(source("animals.R"))

    fadn.str.animals.df <- animal.dt

  } else{
  # loading the animals data took 316.89s
  fadn.str.animals.df <- readRDS(paste0(get.data.dir(),"/rds/str_dir/fadn.str.animal.rds"))
  }


  # Convert str data to NUTS 2016 ---

  fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.str.animals.df,fadn.countries,2016)

  fadn.str.animal.an <- fadn.str.animals %>%
    mutate(ORGANIC=case_when(
      ORGANIC=="org-2" ~ "Organic",
      TRUE ~ "Conventional"
    ),YEAR=as.factor(YEAR)) %>%
    mutate(value2 = WEIGHT*value) %>%
    filter(variable=="AN") %>%
    rename(VARIABLE= variable) %>%
    select(-(TF8:SIZ6),-WEIGHT)