.gitignore

+ani_product.R
+check_tp.R
+extraction_animal_product.R
+test_time.R
+
+
+dimitrios_load_str_crops.R
+dimitrios_load_str_livestocks.R
+
+
+gdx/*.gdx
+
+corrected.json.full/test.json
+corrected.json.full/rewrite_corrected.2013_before.json
+corrected.json.full/rewrite_corrected.2014_after.json
\ No newline at end of file

README.Rmd

@@ -36,16 +36,25 @@ git clone https://git-dmz.thuenen.de/mindstep/fadntocapri.git
 
 It is also possible to create a new Rstudio project with git version control. In Rstudio Select File -> New Project -> Version Control -> Git, then provide the repository HTTPS link: https://git-dmz.thuenen.de/mindstep/fadntocapri.git, select the R workspace folder and create the project. RStudio now copies (clone in Git terms) the content of the repository to your project folder. The content of the Gitlab repository should now appear in the Files pane of RStudio.
 
+Attention: You can also use the IIASA GitLab:
+``` bash
+git clone https://https://gitlab.iiasa.ac.at/mind-step/fadntocapri
+```
+
 ## Usage
 ### Setup and load Libraries
 *  Loading libraries 
 
-Make sure you have `fadnutils` installed. 
+Make sure you have `fadnutils` installed. If the package is public, you should not need password and username.
+
 Using `devtools` to install `fadnutils`. If you did not install this package, please just run the following command at the root of your project: 
 ``` {r, results='hide', message=FALSE}
 # Install fadnutils from thuenen internal gitlab 
 devtools::install_git("https://git-dmz.thuenen.de/mindstep/fadnutilspackages", 
-                      credentials = git2r::cred_user_pass("yang", "Thu_Y2021"))
+                      credentials = git2r::cred_user_pass("yang", "your_password")) ## "your_password" must be your password, if needed
+## Also possible for IIASA GitLab
+# devtools::install_git("https://gitlab.iiasa.ac.at/mind-step/fadnutilspackage")
+
 # load libraries
 requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',
                      'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw','Hmisc')
@@ -94,7 +103,7 @@ source("myfun_fadn.R")
 ```
 ### `Crops` data for CAPRI
 Using `output_GDX_crops` function loads raw data, extracts crops data, computes orgnic and non-org, aggregtes different variables and exports `crops` FADN data.
-```{r}
+```{r eval=FALSE}
 output_GDX_crops <- function (fadn.countries ){
   # Convert and load FADN data, save the str data in path: D:\data\fadn\lieferung_20210414\yang\fadn_work_space\rds\crops
   cat("Convert and load FADN str data for:", fadn.countries)
@@ -151,7 +160,7 @@ output_GDX_crops <- function (fadn.countries ){
 Call funtion to export GDX files for crops of EU country.
 
 
-```{r}
+```{r eval=FALSE}
 output_GDX_crops(fadn.countries = "all")
 ```
 
@@ -159,7 +168,7 @@ output_GDX_crops(fadn.countries = "all")
 
 All the information about `animals` comes from table J (collection of tables are provided by FADN). `output_GDX_animals` includes loading raw animals data from FADN data, cleaning raw data, and exporting final animals data as GDX format.
 
-```{r}
+```{r eval=FALSE}
 output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "all" ) {
   if (convert.str.animal){
     # converting animal str data took 815.39s
@@ -221,7 +230,7 @@ output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "al
 
 ```
 Call funtion to export GDX files for animals of Germany.
-```{r}
+```{r eval=FALSE}
 output_GDX_animals(convert.str.animal = FALSE, fadn.countries = "DEU")
 ```
 

README.md

@@ -25,18 +25,29 @@ git clone https://git-dmz.thuenen.de/mindstep/fadntocapri.git
 
 It is also possible to create a new Rstudio project with git version control. In Rstudio Select File -&gt; New Project -&gt; Version Control -&gt; Git, then provide the repository HTTPS link: <https://git-dmz.thuenen.de/mindstep/fadntocapri.git>, select the R workspace folder and create the project. RStudio now copies (clone in Git terms) the content of the repository to your project folder. The content of the Gitlab repository should now appear in the Files pane of RStudio.
 
+Attention: You can also use the IIASA GitLab:
+
+``` bash
+git clone https://https://gitlab.iiasa.ac.at/mind-step/fadntocapri
+```
+
 ## Usage
 
 ### Setup and load Libraries
 
 -   Loading libraries
 
-Make sure you have `fadnutils` installed. Using `devtools` to install `fadnutils`. If you did not install this package, please just run the following command at the root of your project:
+Make sure you have `fadnutils` installed. If the package is public, you should not need password and username.
+
+Using `devtools` to install `fadnutils`. If you did not install this package, please just run the following command at the root of your project:
 
 ``` r
 # Install fadnutils from thuenen internal gitlab 
 devtools::install_git("https://git-dmz.thuenen.de/mindstep/fadnutilspackages", 
-                      credentials = git2r::cred_user_pass("yang", "Thu_Y2021"))
+                      credentials = git2r::cred_user_pass("yang", "your_password")) ## "your_password" must be your password, if needed
+## Also possible for IIASA GitLab
+# devtools::install_git("https://gitlab.iiasa.ac.at/mind-step/fadnutilspackage")
+
 # load libraries
 requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',
                      'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw','Hmisc')
@@ -154,25 +165,6 @@ Call funtion to export GDX files for crops of EU country.
 output_GDX_crops(fadn.countries = "all")
 ```
 
-    ## Convert and load FADN str data for: all
-    ##  D:/data/fadn/lieferung_20210414/yang/FadntoCapri/corrected.json.full/corrected.2013_before.json  copied to  D:/data/fadn/lieferung_20210414/yang/fadn_work_space/rds//alex/raw_str_map.json
-    ## [1] "Doing id ..."
-    ## [1] "Doing info ..."
-    ## [1] "Doing crops ..."
-    ## ...........................................
-    ## 
-    ##  D:/data/fadn/lieferung_20210414/yang/FadntoCapri/corrected.json.full/corrected.2014_after.json  copied to  D:/data/fadn/lieferung_20210414/yang/fadn_work_space/rds//alex/raw_str_map.json
-    ## [1] "Doing id ..."
-    ## [1] "Doing info ..."
-    ## [1] "Doing crops ..."
-    ## ..........................................
-    ## Convert FADN str data to NUTS Version 2016 for country: allCountries:  NED BEL BGR CZE DAN DEU ELL ESP EST FRA ITA LUX LVA OST POR ROU SVE SVK CYP HRV MLT 
-    ## Converting......
-    ## Non-special country finished.
-    ## Sepcial country finished!
-    ## Combination special and non-countries finished!
-    ## Export the gdx: D:/data/fadn/lieferung_20210414/yang/FadntoCapri/gdx/crops_LEVL_new.gdx
-
 ### `Animals` data for CAPRI
 
 All the information about `animals` comes from table J (collection of tables are provided by FADN). `output_GDX_animals` includes loading raw animals data from FADN data, cleaning raw data, and exporting final animals data as GDX format.
@@ -243,8 +235,3 @@ Call funtion to export GDX files for animals of Germany.
 ``` r
 output_GDX_animals(convert.str.animal = FALSE, fadn.countries = "DEU")
 ```
-
-    ## Countries:  DEU 
-    ## Converting......
-    ## Non-special country finished.
-    ## Export gdx:  D:/data/fadn/lieferung_20210414/yang/FadntoCapri/gdx/animals_LEVL_converted.gdx

animals.R

 # Based on the work of Dimitrios Kremmydas (JRC) and Xinxin Yang (THÜNEN)
-# livestocks
+# livestocks(animals)
 
 source("raw_data_codes_livestock.R")
 
@@ -8,11 +8,12 @@ source("raw_data_codes_livestock.R")
 # raw table K: production
 # col.codes.anim_products.dt
 
-# load raw data ----
-raw.file <- paste0(get.data.dir(),"/rds","/fadn.raw.all.rds")
-raw_data_test <- readRDS(file=raw.file) #82.42s
 
 
+# Animals  ----------------------------------------------------------------
+## load raw data ===========================================================
+raw.file <- paste0(get.data.dir(),"/rds","/fadn.raw.all.rds")
+raw_data_test <- readRDS(file=raw.file) #82.42s
 #read thw variables of a worksheet into R df
 FADN_data_request_file <- "D:/data/fadn/lieferung_20210414/yang/FADN data request forms_2020_02_update - 20201212.xlsx"
 
@@ -38,7 +39,7 @@ missing.names <- unique(missing.names)
 ## add missing variables with 0 value
 raw_data_test[,c(missing.names) := 0]
 
-# load livestocks from raw data ----
+# load livestocks from raw data ===========================================================
 cols.to.load = c("ID",col.codes.livestock.dt[!code.num=="",common.name])
 
 #  only average number of animals
@@ -239,3 +240,4 @@ animal.dt <- ANIMALS_FADN[animal.dt , on = c("variable", "ANIM", "ID", "YEAR")]
 
 
 # all.equal(animal.dt1, animal.dt)
+

corrected.json.full/corrected.2013_before.json

@@ -53,8 +53,7 @@
 			"description": "Organic farming Code (1=No Organic, 2=Only organic, 3=Both, 4=In conversion)"
 		}
 	},
-  // comment info 
-  /*  "info": {
+	"info": {
 		"SIZEUR": {
 			"source": "SIZEUR",
 			"description": "Total economic size in EUR, Calculated by DG AGRI (cf Typology Regulation)"
@@ -127,8 +126,8 @@
 			"source": "NA",
 			"description": ""
 		}
-  }, */
-/*   "costs": {
+	},
+	"costs": {
 		"SEED": {
 			"source": "SE290+ISEDPUR_V",
 			"description": ""
@@ -309,7 +308,64 @@
 				}
 			}
 		},
-    "production": {}
+		"production": {
+			"COMI": {
+				"description": "Cows' milk",
+				"columns": {
+					"GROF": "PMLKCOWBUF_PRQ_X",
+					"EAAP": "NA"
+				}
+			},
+			"SGMI": {
+				"description": "Sheep's and Goat's milk",
+				"columns": {
+					"GROF": "PMLKGOAT_PRQ+PMLKSHEP_PRQ",
+					"EAAP": "NA"
+				}
+			},
+			"EGGS": {
+				"description": "Hens' eggs",
+				"columns": {
+					"GROF": "PEGGHENS_PRQ_X",
+					"EAAP": "NA"
+				}
+			},
+			"OANI": {
+				"description": "Other animal products",
+				"columns": {
+					"GROF": "PANIMPRDOTH_PRQ_X+PWOOL_PRQ+ODAIRPRS_PRQ_X+OSHEPDAIRPRS_PRQ+OGOATDAIRPRS_PRQ",
+					"EAAP": "NA"
+				}
+			},
+			"PCAT": {
+				"description": "Cattle Net output",
+				"columns": {
+					"EALP": "LBOV_NO",
+					"EAAP": "LBOV_CV+LBOV_FUV-LBOV_OV+LBOV_NO+LBOV_PV"
+				}
+			},
+			"PPIG": {
+				"description": "Pigs Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			},
+			"PSAG": {
+				"description": "Goats and sheeps Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			},
+			"POTH": {
+				"description": "Other Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			}
+		}
 	},
 	"subsidies": {
 		"SA1OTHSUB_2_V": {
@@ -1912,7 +1968,7 @@
 			"source": "SYF_V",
 			"description": ""
 		}
-  }, */
+	},
 	"crops": {
 		"SWHE": {
 			"description": "Soft wheat",

corrected.json.full/corrected.2014_after.json

@@ -53,7 +53,7 @@
 			"description": "Organic farming Code (1=No Organic, 2=Only organic, 3=Both, 4=In conversion)"
 		}
 	},
-/*   "info": {
+	"info": {
 		"SIZEUR": {
 			"source": "SIZEUR",
 			"description": "Total economic size in EUR, Calculated by DG AGRI (cf Typology Regulation)"
@@ -126,8 +126,8 @@
 			"source": "NA",
 			"description": ""
 		}
-  }, */
- /*  "costs": {
+	},
+	"costs": {
 		"SEED": {
 			"source": "SE290+ISEDPUR_V",
 			"description": ""
@@ -308,7 +308,64 @@
 				}
 			}
 		},
-    "production": {}
+		"production": {
+			"COMI": {
+				"description": "Cows' milk",
+				"columns": {
+					"GROF": "PMLKCOW_PRQ+PMLKCOW_PRQ",
+					"EAAP": "PMLKBUF_TO+PMLKCOW_TO"
+				}
+			},
+			"SGMI": {
+				"description": "Sheep's and Goat's milk",
+				"columns": {
+					"GROF": "PMLKGOAT_PRQ + PMLKSHEP_PRQ",
+					"EAAP": "NA"
+				}
+			},
+			"EGGS": {
+				"description": "Hens' eggs",
+				"columns": {
+					"GROF": "PEGGC_PRQ+PEGGH_PRQ",
+					"EAAP": "PEGGH_TO + PEGGC_TO"
+				}
+			},
+			"OANI": {
+				"description": "Other animal products",
+				"columns": {
+					"GROF": "NA",
+					"EAAP": "NA"
+				}
+			},
+			"PCAT": {
+				"description": "Cattle Net output or Total output",
+				"columns": {
+					"EALP": "LBOV_NO",
+					"EAAP": "LBOV_CV+LBOV_FUV-LBOV_OV+LBOV_NO+LBOV_PV"
+				}
+			},
+			"PPIG": {
+				"description": "Pigs Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			},
+			"PSAG": {
+				"description": "Goats and sheeps Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			},
+			"POTH": {
+				"description": "Other Net output or Total output",
+				"columns": {
+					"EALP": "NA",
+					"EAAP": "NA"
+				}
+			}
+		}
 	},
 	"subsidies": {
 		"SA1OTHSUB_2_V": {
@@ -1911,7 +1968,7 @@
 			"source": "SYF_V",
 			"description": ""
 		}
-  }, */
+	},
 	"crops": {
 		"SWHE": {
 			"description": "Soft wheat",

fadn_to_capri.Rmd

+---
+title: "Fadn to capri"
+author: "Xinxin Yang"
+date: "13 Januar 2022"
+output: html_document
+---
+
+```{r setup, include=FALSE}
+knitr::opts_chunk$set(echo = TRUE)
+```
+
+## Setup, libraries
+
+## Introduction
+
+## Crops 
+
+## Animals
+Save data into `crops_LEVL_new.gdx` for CAPRI
+##

fadntocapri.R

 rm(list =ls())
 
-# Install fadnutils from thuenen gitlab
-devtools::install_git("https://git-dmz.thuenen.de/mindstep/fadnutilspackages", force = TRUE)
+# Setup -------------------------------------------------------------------
 
-# load libraries ----
+## Install fadnutils from thuenen gitlab
+# devtools::install_git("https://git-dmz.thuenen.de/mindstep/fadnutilspackages", force = TRUE)
+
+## load libraries
 requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2',
                      'gdxdt', 'tidyverse', 'xlsx', 'gdxrrw','Hmisc')
 for(p in requiredPackages){
   if(!require(p,character.only = TRUE)) install.packages(p)
   library(p,character.only = TRUE)
 }
-# set gams path ----
+## set gams path
 igdx("d:/gams/win64/34.3")
 
-# Check your current working directory ----
-# Determin if Git directory is current working dir.
+## Check your current working directory
+## Determin if Git directory is current working dir.
 getwd()
-
-# Load functions
-source("myfun_fadn.R")
-
-
-# Set FADN project directory ----
+setwd("D:/data/fadn/lieferung_20210414/yang/FadntoCapri")
+## Set FADN project directory
 CurrentProjectDirectory = "D:/data/fadn/lieferung_20210414/yang/fadn_work_space"
 
-# Ceate a data.dir
+## Ceate a data.dir
 create.data.dir(folder.path = CurrentProjectDirectory)
 
-# Once the data.dir is created, we must declare that we are working with it
+## Once the data.dir is created, we must declare that we are working with it
 set.data.dir(CurrentProjectDirectory)
 
 rds.dir = paste0(get.data.dir(),"/rds/")
 
-# Set EU list
+## Set EU list
 
 EU_list <- c("AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK",
         "EST", "FIN", "FRA", "DEU", "GRC", "HUN", "IRL",
         "ITA", "LVA", "LTU", "LUX", "MLT", "NLD", "POL",
         "PRT", "ROU", "SVK", "SVN", "ESP", "SWE")
 
-# FADN Crops -----
-output_GDX_crops <- function (fadn.countries ){
+## Load functions
+source("D:/data/fadn/lieferung_20210414/yang/FadntoCapri/myfun_fadn.R")
+
+
+# FADN to CAPRI -----------------------------------------------------------
+
+## FADN Crops  -----
+output_GDX_crops <- function (fadn.countries, filter.variable = "LEVL" ){
   # Convert and load FADN data, save the str data in path: D:\data\fadn\lieferung_20210414\yang\fadn_work_space\rds\crops
   cat("Convert and load FADN str data for:", fadn.countries)
   fadn.str.crops <- convert.load.str.crops(countries = fadn.countries)
@@ -50,13 +54,13 @@ output_GDX_crops <- function (fadn.countries ){
   # step 1: Convert str data to NUTS 2016
   fadn.str.crops <-  fadnUtils::NUTS.convert.all(fadn.str.crops,fadn.countries, 2016)
 
-  # filter LEVL
+  # filter
   fadn.str.crops.levl <- fadn.str.crops %>%
     mutate(ORGANIC=case_when(
       ORGANIC=="org-2" ~ "Organic",
       TRUE ~ "Conventional"))%>%
     mutate(value2 = WEIGHT*VALUE) %>%
-    filter(VARIABLE == "LEVL" ) %>%
+    filter(VARIABLE == filter.variable ) %>%
     select(-(TF8:SIZ6),-WEIGHT, -ALTITUDE)
   # step 2: export a gdx
   # export DEU gdx took 1.25 mins
@@ -81,11 +85,12 @@ output_GDX_crops <- function (fadn.countries ){
   col_names_crops <- colnames(crops.groupby)
   # library(gdxrrw)
   # igdx("d:/gams/win64/34.3")
-  cat("Export the gdx: ", getwd(), "/gdx/crops_LEVL_new.gdx",sep = "")
+  # cat("Export the gdx: ", getwd(), "/gdx/crops_LEVL_new.gdx",sep = "")
+  cat("Export the gdx: ", getwd(), paste0("/gdx/crops_",fadn.countries,"_", filter.variable,".gdx") ,sep = "")
 
   # write gdx: levl: value*WEIGHT
   writegdx(dt = crops.groupby,
-           gdx = paste0(getwd(), "/gdx/crops_LEVL_new.gdx"),
+           gdx = paste0(getwd(), "/gdx/crops_",fadn.countries, "_",filter.variable,".gdx"),
            name = "DataOut",
            valcol= "sum_Value",
            uelcols= col_names_crops[!col_names_crops %in% "sum_Value"],
@@ -97,31 +102,56 @@ output_GDX_crops <- function (fadn.countries ){
 }
 
 
-# FADN animals ----
-output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "all" ) {
+## FADN animals ----
+output_GDX_animals <- function( convert.str.animal = FALSE, filter.variable = "AN", fadn.countries = "all" ) {
+  # load data ----
+  if (filter.variable == "AN") {
+    cat("Loading animal data ......\n")
+
     if (convert.str.animal) {
       # converting animal str data took 815.39s
+      cat("Converting animal str data ....\n")
       system.time(source("animals.R"))
 
       fadn.str.animals.df <- animal.dt
 
+
     } else{
       # loading the animals data took 316.89s
-  fadn.str.animals.df <- readRDS(paste0(get.data.dir(),"/rds/str_dir/fadn.str.animal.rds"))
+      fadn.str.animals.df <-
+        readRDS(paste0(get.data.dir(), "/rds/str_dir/fadn.str.animal.rds"))
+    }
+
+    fadn.animals.dt <- fadn.str.animals.df
+  } else {
+    cat("Loading animal product data ......\n")
+    # load animal production
+    if (!exists('ANIMALS.production.all')) {
+      cat("Loading animal product data")
+      source("total_production_animal.R")
+      # ANIMALS.production.all <- fadn.tp.animals()
+    } else{
+      cat("loaded!\n")
+    }
+    fadn.animals.dt <- ANIMALS.production.all
   }
 
+  if(!filter.variable %in% fadn.animals.dt$variable) {
+    cat(filter.variable, "is not included\n")
+    return(filter.variable, "is not included\n")}
 
+  cat("Converting to NUTS 2016 ......\n")
   # Convert str data to NUTS 2016 ---
 
-  fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.str.animals.df,fadn.countries,2016)
+  fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.animals.dt, fadn.countries, 2016)
 
-  fadn.str.animal.an <- fadn.str.animals %>%
+  fadn.str.animals.filtered <- fadn.str.animals %>%
     mutate(ORGANIC=case_when(
       ORGANIC=="org-2" ~ "Organic",
       TRUE ~ "Conventional"
     ),YEAR=as.factor(YEAR)) %>%
     mutate(value2 = WEIGHT*value) %>%
-    filter(variable=="AN") %>%
+    filter(variable==filter.variable) %>%
     rename(VARIABLE= variable) %>%
     select(-(TF8:SIZ6),-WEIGHT)
 
@@ -129,7 +159,7 @@ output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "al
 
   group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final","NUTS2_final", "EU")
   animals.groupby <-  lapply(seq_along(group_by_lst),
-                             function(i) fadn.filter(fadn.str.animal.an,
+                             function(i) fadn.filter(fadn.str.animals.filtered,
                                                      group_by_lst[i],
                                                      "ANIM")) %>%
     bind_rows() %>%
@@ -142,35 +172,75 @@ output_GDX_animals <- function( convert.str.animal = FALSE, fadn.countries = "al
                                 TRUE ~ REG_TYPE)) %>%
     mutate_if(is.factor, as.character)
 
-
-
   col_names_animals <- colnames(animals.groupby)
-  # library(gdxrrw)
-  # igdx("d:/gams/win64/34.3")
-  cat("Export gdx: ",paste0(getwd(), "/gdx/animals_LEVL_converted.gdx") )
+
+  cat("Export gdx: ",paste0(getwd(), "/gdx/animals_",filter.variable,"_",fadn.countries,"_converted.gdx\n"))
   # write gdx: levl: value*WEIGHT
   writegdx(dt = animals.groupby,
-           gdx = paste0(getwd(), "/gdx/animals_LEVL_converted.gdx"),
+           gdx = paste0(getwd(), "/gdx/animals_",filter.variable,"_",fadn.countries,"_converted.gdx"),
            name = "DataOut",
            valcol= "sum_Value",
            uelcols= col_names_animals[!col_names_animals %in% "sum_Value"],
            type="parameter")
-
-
-
-
 }
 
 
-# crops ----
+## Call functions --------------------------------------------------------
+### crops ----
+
+# LEVL----
 # only for DEU took 108s
+output_GDX_crops(fadn.countries = "DEU", filter.variable = "LEVL")
 # NED and ROU took 119.87
+output_GDX_crops(fadn.countries = c("DEU","ROU"),filter.variable = "LEVL")
 # all countries took 731.40s
-system.time(output_GDX_crops(fadn.countries = "all"))
-
-# animals ----
-# 332.30s
-system.time(output_GDX_animals(convert.str.animal = FALSE, fadn.countries = "DEU"))
-
-
-
+output_GDX_crops(fadn.countries = "all",filter.variable = "LEVL")
+
+
+# GROF----
+output_GDX_crops(fadn.countries = "DEU", filter.variable = "GROF")
+output_GDX_crops(fadn.countries = "all", filter.variable = "GROF")
+# EAAP----
+output_GDX_crops(fadn.countries = "all", filter.variable = "EAAP")
+
+
+### animals ----
+# AN ----
+# took 332.30s
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "AN",
+                               fadn.countries = "DEU"))
+# GROF: gross production ----
+# GROF: product quantity of germany
+# took 6.55s
+fadn.countries = "DEU"
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "GROF",
+                               fadn.countries = "DEU"))
+# GROF: product quantity of EU took 1110.34s
+fadn.countries = "all"
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "GROF",
+                               fadn.countries = "all"))
+# EAAP: European Assocication for Animal production ----
+# EAAP: total output
+# EAAP of EU
+# took  10s
+fadn.countries = "all"
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "EAAP",
+                               fadn.countries = "all"))
+# EALP ----
+# EALP: net output
+# EALP of EU
+# took 9.39s
+fadn.countries = "all"
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "EALP",
+                               fadn.countries = "all"))
+# EALP of germany
+# took 120.74s
+fadn.countries = "DEU"
+system.time(output_GDX_animals(convert.str.animal = FALSE,
+                               filter.variable = "EALP",
+                               fadn.countries = "DEU"))

plot_animal_product.R

+# plots for animal production
+#
+cat("Loading animal product data ......\n")
+
+fadn.countries = "all"
+# load animal production
+source("total_production_animal.R")
+fadn.animals.dt <- ANIMALS.production.all %>% distinct()
+str(fadn.animals.dt )
+
+cat("Converting NUTS 2016....\n")
+fadn.str.animals <- fadnUtils::NUTS.convert.all(fadn.animals.dt, "all", 2016)
+
+
+# check NUTS, if converted
+fadn.str.animals %>% filter(COUNTRY == "DEU") %>% count(NUTS2,NUTS2_final)
+
+fadn.str.animals.organic <- fadn.str.animals %>%
+  mutate(ORGANIC=case_when(
+    ORGANIC=="org-2" ~ "Organic",
+    TRUE ~ "Conventional"
+  ),YEAR = as.factor(YEAR)) %>%
+  mutate(value2 = WEIGHT*value) %>%
+  # filter(variable == filter.variable) %>%
+  rename(VARIABLE= variable) %>%
+  select(-(TF8:SIZ6),-WEIGHT)
+
+
+
+group_by_lst <- c("COUNTRY","REGION","NUTS1","NUTS2","NUTS1_final","NUTS2_final", "EU")
+animals.groupby <-  lapply(seq_along(group_by_lst),
+                           function(i) fadn.filter(fadn.str.animals.organic,
+                                                   group_by_lst[i],
+                                                   "ANIM")) %>%
+  bind_rows() %>%
+  mutate(REG_TYPE = case_when(REG_TYPE == "COUNTRY" ~"MS",
+                              REG_TYPE == "REGION" ~ "FADN_REGION",
+                              REG_TYPE == "NUTS1" ~ "NUTS1_ORG",
+                              REG_TYPE =="NUTS2"~ "NUTS2_ORG",
+                              REG_TYPE =="NUTS1_final"~ "Xinxin_NUTS1",
+                              REG_TYPE =="NUTS2_final" ~"Xinxin_NUTS2",
+                              TRUE ~ REG_TYPE)) %>%
+  mutate_if(is.factor, as.character)
+
+
+# plots ----
+# MS ----
+options(scipen = 999)
+for (var in c("EALP", "EAAP", "GROF")) {
+  for (country in fadn.countries.sel) {
+    # country= "DEU"
+    p <-  animals.groupby %>%
+      filter(REG_TYPE == "MS" &
+               VARIABLE == var & REGION == country) %>%
+      ggplot(aes(
+        x = YEAR,
+        y = sum_Value,
+        color = ORGANIC,
+        group = ORGANIC
+      )) +
+      geom_point() +
+      geom_line() +
+      facet_wrap(vars(ANIM),
+                 # space="free",
+                 ncol = 4,
+                 scale = "free_y") +
+      ggtitle(paste0(var, " of country: ", country)) +
+      ylim(c(0, NA)) +
+      theme(axis.text.x = element_text(angle = 45, hjust = 1))
+    options(scipen = 999)
+
+
+    ggsave(
+      plot = p ,
+      filename = paste0(
+        CurrentProjectDirectory,
+        "/plots/animal/TP/",
+        var,
+        "/",
+        country,
+        "_",
+        var,
+        ".png"
+      ),
+      width = 18,
+      height = 8
+    )
+
+  }
+}
+
+# EU ------------------------------------------------------------------
+for (var in c("EALP", "EAAP", "GROF") ){
+p <- animals.groupby %>%
+  filter(REG_TYPE == "Xinxin_NUTS2" &
+           VARIABLE == var ) %>%
+  ggplot(aes(
+    x = YEAR,
+    y = sum_Value,
+    color = ORGANIC,
+    group = ORGANIC
+  )) +
+  geom_point() +
+  geom_line() +
+  facet_wrap(vars(ANIM),
+             # space="free",
+             ncol = 4,
+             scale = "free_y") +
+  ggtitle(paste0(var, " of EU ")) +
+  ylim(c(0, NA)) +
+  theme(axis.text.x = element_text(angle = 45, hjust = 1))
+
+
+ggsave(
+  plot = p ,
+  filename = paste0(
+    CurrentProjectDirectory,
+    "/plots/animal/TP/EU/",
+    var,
+    ".png"
+  ),
+  width = 18,
+  height = 8
+)
+}
+
+# NUTS2 -------------------------------------------------------------------
+
+

raw_data_codes_livestock.R

@@ -7602,4 +7602,3 @@ col.codes.anim_products.dt = data.table(  #convert the previous list to a DT, us
   units=list.mapv(col.codes.anim_products.list,units)
 )
 
-

total_production_animal.R

+# extraction total production of animals ----------------------------------
+
+fadn.tp.animals <- function( ) {
+  # fadn countries
+  # if (fadn.countries == "all") {
+    fadn.countries.sel = c(
+      "NED",
+      "BEL",
+      "BGR",
+      "CYP",
+      "CZE",
+      "DAN",
+      "DEU",
+      "ELL",
+      "ESP",
+      "EST",
+      "FRA",
+      "HUN",
+      "IRE",
+      "ITA",
+      "LTU",
+      "LUX",
+      "LVA",
+      "MLT",
+      "OST",
+      "POL",
+      "POR",
+      "ROU",
+      "SUO",
+      "SVE",
+      "SVK",
+      "SVN",
+      "UKI",
+      "HRV"
+    )
+  # } else{
+  #   fadn.countries.sel = fadn.countries
+  # }
+  # fadn years
+  fadn.years.sel = c(2004:2018)
+  # fadn info
+  fadn.info.sel = c(
+    "ID",
+    "YEAR",
+    "COUNTRY",
+    "REGION",
+    "NUTS0",
+    "NUTS1",
+    "NUTS2",
+    "NUTS3",
+    "SYS02",
+    "TF14",
+    "TF8",
+    "SIZ6",
+    "ORGANIC"
+  )
+  # extraction variables for animal product
+
+  fadn.tp.sel <-
+    c(
+      "LBOV_CV",
+      "LBOV_FUV",
+      "LBOV_NO",
+      "LBOV_OV",
+      "LBOV_PV",
+      "LBOV_SV",
+      "PMLKGOAT_PRQ",
+      "PMLKSHEP_PRQ",
+      "PMLKBUF_TO",
+      "PMLKCOW_TO",
+      "PMLKCOWBUF_PRQ_X",
+      "PMLKBUF_PRQ",
+      "PMLKCOW_PRQ",
+      "PEGGHENS_PRQ_X",
+      "PEGGC_PRQ",
+      "PEGGH_PRQ",
+      "PEGGH_TO",
+      "PEGGC_TO"
+    )
+
+
+  # fadn.str.data <- load.fadn.str.rds("alex",,"all")
+
+  # after2014 <- readRDS(paste0(rds.dir, "/alex/fadn.str.after2014.all.rds"))
+  # before2013 <- readRDS(paste0(rds.dir, "/alex/fadn.str.before2013.all.rds"))
+  # fadn.str.crops <- bind_rows(before2013$crops,after2014$crops)
+
+  # load
+  cols.to.load.anim_prod = c(fadn.info.sel, fadn.tp.sel)
+
+  # load raw fadn data using fadnutils package
+  tmp.LOAD.raw = fadnUtils::load.fadn.raw.rds(years = fadn.years.sel,
+                                              countries = fadn.countries.sel,
+                                              col.filter = cols.to.load.anim_prod)
+
+  #
+  tmp.LOAD.raw <- tmp.LOAD.raw %>% select(-c(load.COUNTRY, load.YEAR))
+
+
+
+
+  tmp.LOAD <- tmp.LOAD.raw %>% mutate(
+    COMI_GROF = ifelse (
+      YEAR %in% c(2004:2013),
+      PMLKCOWBUF_PRQ_X,
+      (PMLKBUF_PRQ + PMLKCOW_PRQ)
+    ),
+    COMI_EAAP = PMLKBUF_TO + PMLKCOW_TO,
+    EGGS_GROF = ifelse (
+      YEAR %in% c(2004:2013),
+      PEGGHENS_PRQ_X,
+      as.numeric (PEGGC_PRQ  + PEGGH_PRQ)
+    ) ,
+    EGGS_EAAP = PEGGH_TO + PEGGC_TO,
+    SGMI_GROF = PMLKGOAT_PRQ + PMLKSHEP_PRQ,
+    PCAT_EAAP = LBOV_CV + LBOV_FUV - LBOV_OV + LBOV_NO + LBOV_PV,
+    PCAT_EALP = LBOV_NO
+  )
+
+  tmp.LOAD.raw.long = melt(tmp.LOAD %>% select(-fadn.tp.sel),
+                           id.vars = fadn.info.sel)
+
+  tmp.LOAD.raw.long[grep("_", variable),
+                    ":="(
+                      PRODUCT = gsub("^(.+?)_(.+?)(_.*)?$", "\\1", variable),
+                      VAR = gsub("^(.+?)_(.+?)(_.*)?$", "\\2", variable)
+                    )]
+
+  # rename and set the type of objects
+  ANIMALS.production.all <- tmp.LOAD.raw.long %>%
+    select(-c("NUTS0")) %>%
+    mutate_if(is.character, as.factor) %>%
+    mutate_at(c("COUNTRY", "REGION", "NUTS3", "NUTS2"), as.character) %>%
+    mutate(ORGANIC =
+             paste0('org-', ORGANIC)) %>%
+    select(-variable)%>%
+    rename(WEIGHT = SYS02, variable = VAR, ANIM = PRODUCT )
+
+  return(ANIMALS.production.all)
+
+}
+
+ANIMALS.production.all <- fadn.tp.animals()