GitLab at IIASA

FADN_USE_CASE.R 27.2 KiB
Newer Older
Xinxin Yang's avatar
Xinxin Yang committed
############################################################################
###                                                                      ###
###                              FADNUTILS:                              ###
###         AN R PACKAGE TO EASILY LOAD AND MANIPULATE FADN DATA         ###
###                                                                      ###
############################################################################


##  In order to use fadnUtils, we must load fadnUtils and other packages.

## R version >=3.6.1 and < 4.0.0
Xinxin Yang's avatar
Xinxin Yang committed

Xinxin Yang's avatar
Xinxin Yang committed

############################################################################
###                                                                      ###
###                                SETUP:                                ###
###                 DIRECTORY NAMES AND CONDITIONAL FILE                 ###
###                                                                      ###
############################################################################
# fadnUtils always work with a user defined data.dir
# Let's assume that the user has not created one yet.
# The following line creates a data.dir folder somewhere in our computer
# We must also have created the raw_str_map.file and pass it as an argument
# to the function. This file is copied to the data.dir folder. Thus, we can
# see the structure of the data contained in a data.dir folder by inspecting
# the raw_str_map.file residing in it.

##################################################################
##                  Install and load packages                   ##
##################################################################
requiredPackages = c('fadnUtils','data.table', 'devtools','jsonlite', 'ggplot2')
for(p in requiredPackages){
  if(!require(p,character.only = TRUE)) install.packages(p)
  library(p,character.only = TRUE)
}


#################################################################
##                       DIRECTORY NAMES                       ##
#################################################################

CurrentProjectDirectory = "D:/public/yang/MIND_STEP/New_test_fadnUtils"

##################################################################
##                        Required files                        ##
##################################################################

# the path of the fadn files for loading
fadn.data.dir = "D:/public/data/fadn/lieferung_20210414/csv/"

# A json file for extraction

# ceate a data.dir
create.data.dir(folder.path = CurrentProjectDirectory)

# Once the data.dir is created, we must declare that we are working with it
set.data.dir(CurrentProjectDirectory)

get.data.dir()

# After you create a data dir, below is a list of "real-world" example files:
# CurrentProjectDirectory/
# +-- csv
# +-- fadnUtils.metadata.json
# +-- rds
# \-- spool
#     \-- readme.txt


############################################################################
###                                                                      ###
###                              SECTION 1:                              ###
###                         IMPORT CSV FADN DATA                         ###
###                                                                      ###
############################################################################


# .............. IMPORT DATA IN TWO STEPS ..........................................#

# However, you can import the file in two steps, one for converting
# the csv to fadn.raw.str (csv-data to raw r-data) and
# one for converting the fadn.raw.rds to fadn.str.rds (raw r-data
# to structured r-data).


#################################################################
##             STEP 1: CONVERT CSV TO FADN.RAW.RDS             ##
#################################################################

##-----------------------------
## load each file separately
##-----------------------------
# load for a specific country "DEU" and from a specific year "2009"
convert.to.fadn.raw.rds(
  file.path = paste0(fadn.data.dir ,"DEU2009.csv"),
  sepS = ",",
  fadn.country = "DEU",
  fadn.year = 2009
  #keep.csv = T # copy csv file in csv.dir
Xinxin Yang's avatar
Xinxin Yang committed
)

##-----------------------------
## load all csv files in a folder
##-----------------------------
"csv2raw function takes csv files in a folder and converts them into raw data"
allcsv2raw <- function(LocationofCSVFiles){
Xinxin Yang's avatar
Xinxin Yang committed
  csv_file_names <- list.files(path = LocationofCSVFiles, pattern= "*.csv$")
Xinxin Yang's avatar
Xinxin Yang committed
  #csv_file_names <- "DEU, BEL"
  for (file in csv_file_names){
    # extract first 3 char
    country = substr(file, 1, 3)
Xinxin Yang's avatar
Xinxin Yang committed
    # extract 4-7 char
    year = substr(file, 4, 7)
    #year = as.numeric(gsub("\\D+", "", file))
Xinxin Yang's avatar
Xinxin Yang committed
    convert.to.fadn.raw.rds(
      file.path = paste0(fadn.data.dir,file),
      sepS = ",",
      fadn.country = country,
      fadn.year = year
      #keep.csv = T # copy csv file in csv.dir
Xinxin Yang's avatar
Xinxin Yang committed
    )
Xinxin Yang's avatar
Xinxin Yang committed
  }
}


allcsv2raw(fadn.data.dir)

##-----------------------------
## load specific year and country
##-----------------------------


"C.Y2raw function takes selected countries and years, then converts them into raw data"
C.Y2raw <- function(countries, years){
  for (country in countries){
    for (year in years){
Xinxin Yang's avatar
Xinxin Yang committed
      file = paste0(country,year,".csv")
Xinxin Yang's avatar
Xinxin Yang committed
      convert.to.fadn.raw.rds(
        file.path = paste0(fadn.data.dir,file),
        sepS = ",",
        fadn.country = country,
        fadn.year = year
        #keep.csv = T # copy csv file in csv.dir
Xinxin Yang's avatar
Xinxin Yang committed
      )
Xinxin Yang's avatar
Xinxin Yang committed
    }
  }
}

# load countries: BEL, DEU and NED
countriesList = c("BEL", "DEU", "NED")
yearsList = c(2009,2010,2011,2018)
C.Y2raw(countries = countriesList, years =yearsList )



show.data.dir.contents()


# If you converted the csv to raw r-data successfully, raw r-data files are saved in "rds" folder,
# the project's files and folders look like this:
Xinxin Yang's avatar
Xinxin Yang committed

# New_test_fadnUtils/
# +-- csv
# +-- fadnUtils.metadata.json
# +-- rds
# |   +-- fadn.raw.2009.BEL.compressed.rds
# |   +-- fadn.raw.2009.BEL.rds
# |   +-- fadn.raw.2010.BEL.compressed.rds
# |   +-- fadn.raw.2010.BEL.rds
# |   +-- fadn.raw.2011.BEL.compressed.rds
# |   +-- fadn.raw.2011.BEL.rds
# |   +-- fadn.raw.2012.BEL.compressed.rds
# |   \-- fadn.raw.2012.BEL.rds
# \-- spool
#     \-- readme.txt


##################################################################
##         STEP 2: CONVERT FADN.RAW.RDS TO FADN.STR.RDS         ##
##################################################################

#######################################################################################################
# Notices:#
###########
## Before converting raw r-data into str r-data, it is recommended to use check.column() method
## so that all variables in this json file can be converted.
Xinxin Yang's avatar
Xinxin Yang committed
## The conversion of the raw r-data file to a structured r-data file is driven by a human-readable file,
## called raw_str_map.json.
## This json file is saved in extraction_dir by default.
## if you want to use raw_str_map.json by default, please put this file in extraction_dir.
## Or the user can define a external json file where it is
## and how to caculate the str r-data.
#######################################################################################################

rds.dir = paste0(get.data.dir(),"/rds/")

# set a str name for for saving the str r-data in rds.dir
new.str.name = "test"

# set a extraction_dir
dir.create(paste0(rds.dir, new.str.name))
new.extraction.dir = paste0(rds.dir, new.str.name)

##-----------------------------------------------------------------------------
##  Step 2.0: Check the variables of loaded a  raw rds data and a json file  --
##-----------------------------------------------------------------------------

# Save the modifed json file
list_vars = check.column(importfilepath = paste0(rds.dir, "fadn.raw.2009.BEL.rds"), # a rds file or a csv file
                         jsonfile = "C:/Users/yang_x/Desktop/new-Version/inst/examples/2014_after - Kopie.json", # a json file
Xinxin Yang's avatar
Xinxin Yang committed
                         rewrite_json = TRUE, # write a new json file without unmatched variables
                         extraction_dir = new.extraction.dir # save the new json in extraction_dir
)

# Let's see the unmathcted variables in this json file
print(list_vars)

Xinxin Yang's avatar
Xinxin Yang committed
##---------------------------------------------------------------
##  Step 2.1: convert convert the raw r-data into str r-data   --
##---------------------------------------------------------------

#check the default json file in extraction_dir
if ("raw_str_map.json" %in% list.files(new.extraction.dir, pattern = "\\.json$")){
  cat(new.extraction.dir, "has a raw_str_map.json.", "\n")
}else{warning("please put a raw_str_map.json in ", new.extraction.dir,"\n", "Or using a external json file (option 2)", "\n")}


##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
##  option 1: convert the file separately using a raw_str_map.json in extraction_dir
Xinxin Yang's avatar
Xinxin Yang committed
##  making sure that a raw_str_map.json is in extraction_dir
##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

convert.to.fadn.str.rds(fadn.country = "BEL",
                        fadn.year = 2009,
                        str.name = new.str.name # extraction_dir
)

##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
##  option 2: convert the file separately using a external json file
Xinxin Yang's avatar
Xinxin Yang committed
##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

# If force_external_raw_str_map is TRUE
# this external json file will be copied to extraction_dir as raw_str_map.json,
Xinxin Yang's avatar
Xinxin Yang committed
convert.to.fadn.str.rds(fadn.country = "BEL",
                        fadn.year = 2009,
                        raw_str_map.file = "D:/public/yang/MIND_STEP/new_sample/test01/raw_str_map.json", # a external json file
                        str.name = new.str.name, # extraction_dir
                        force_external_raw_str_map = T,
                        DEBUG = F
)

##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
##  option 3: convert mutilple raw r-data files in rds.dir into str r-data in extraction_dir
Xinxin Yang's avatar
Xinxin Yang committed
##::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

"raw2str function will help us to convert the raw r-data into the str r-data in rds.dir completely.
This function takes a user-defined raw_str_map.file and a logical constant which is FALSE by default,
and converting raw data to str data."
raw2str <- function(Current_raw_str_map.file = NULL, overwrite_external_json = F){
Xinxin Yang's avatar
Xinxin Yang committed
  rds.dir  = paste0(get.data.dir(), "/rds")
Xinxin Yang's avatar
Xinxin Yang committed
  raw_file_names <- dir(rds.dir, pattern ="\\.rds$" )
Xinxin Yang's avatar
Xinxin Yang committed
  for (file in raw_file_names){
Xinxin Yang's avatar
Xinxin Yang committed
    if (!grepl("compressed", file)){
Xinxin Yang's avatar
Xinxin Yang committed
      # extract first 3 char
      country = substr(file, 15, 17)
Xinxin Yang's avatar
Xinxin Yang committed
      # extract number
      year = as.numeric(gsub("\\D+", "", file))
Xinxin Yang's avatar
Xinxin Yang committed
      cat("converting the str data for country: ", country, " and year: ", year, "\n")
      tryCatch(
        expr = {
          convert.to.fadn.str.rds(fadn.country = country,
                                  fadn.year = year,
                                  raw_str_map.file = Current_raw_str_map.file,
                                  force_external_raw_str_map =  overwrite_external_json,
                                  str.name = new.str.name
          )
        },
        warning = function(w){
          message('Caught an warning!')
          print(w)
        },
        error = function(e) {
          message("Caught an error! Please check the objects in json file using check.column() (see more in USE_CASE_4.R).")
          #cat("Wrong, can't convert the str r-data!",sep = "\n")
          print(e)
        }
      )
Xinxin Yang's avatar
Xinxin Yang committed
    }
Xinxin Yang's avatar
Xinxin Yang committed
    else{cat("It's compressed data!",sep = "\n")}
Xinxin Yang's avatar
Xinxin Yang committed
  }
}
#---------------------------------------------------------------
# option 3.1: using a raw_str_map.json by defaut
Xinxin Yang's avatar
Xinxin Yang committed
#---------------------------------------------------------------
# convert str data with a default json file, make sure that a raw_str_map.json in extraction_dir
raw2str()
# or
raw2str(Current_raw_str_map.file = "D:/public/yang/MIND_STEP/new_sample/test01/raw_str_map.json", overwrite_external_json = F)

#---------------------------------------------------------------
# option 3.2: using a external json file
Xinxin Yang's avatar
Xinxin Yang committed
#---------------------------------------------------------------

# convert str data using a external json file
raw2str(Current_raw_str_map.file = "D:/public/yang/MIND_STEP/new_sample/test01/raw_str_map.json", overwrite_external_json = T)

show.data.dir.contents()


# If str r-data was converted, the str r-data is saved in "test"(new.str.name) folder as below.

# New_test_fadnUtils/
# +-- csv
# +-- fadnUtils.metadata.json
# +-- rds
# |   +-- fadn.raw.2009.BEL.compressed.rds
# |   +-- fadn.raw.2009.BEL.rds
# |   +-- fadn.raw.2010.BEL.compressed.rds
# |   +-- fadn.raw.2010.BEL.rds
# |   +-- fadn.raw.2011.BEL.compressed.rds
# |   +-- fadn.raw.2011.BEL.rds
# |   +-- fadn.raw.2012.BEL.compressed.rds
# |   +-- fadn.raw.2012.BEL.rds
# |   \-- test
# |       +-- fadn.str.2009.BEL.rds
# |       +-- raw_str_map.json
# |       \-- rewrite_2014_after_copy.json
# \-- spool
#     +-- my_logfile.txt
#     \-- readme.txt


###########################################################################
###                                                                     ###
###                              SECTION 2:                             ###
###                     LOAD R-DATA FROM A DATA.DIR                     ###
###                                                                     ###
###########################################################################


#################################################################
##                       LOAD RAW R-DATA                       ##
#################################################################
### We can either load raw r-data files (the original FADN csv in r-friendly format),
### or structured r-data files (the original data transformed into meaningful
### information)

# To load raw r-data, only for BEL and 2009
my.raw.data.BEL = load.fadn.raw.rds(
Xinxin Yang's avatar
Xinxin Yang committed
  countries = "BEL",
  years = 2009
)

# my.data is a single large data.table, with the original csv columns and rows
nrow(my.raw.data.BEL)  #Number of rows
names(my.raw.data.BEL)  #Column names
length(names(my.raw.data.BEL)) #Number of columns
str(my.raw.data.BEL) #Overall structure
Xinxin Yang's avatar
Xinxin Yang committed

##################################################################
##                    LOAD STRUCTURED R-DATA                    ##
##################################################################

#To load structured data, for BEL and 2009
my.str.data.2009.BEL = load.fadn.str.rds(
Xinxin Yang's avatar
Xinxin Yang committed
  countries = "BEL",
  years = 2009,
  extraction_dir = new.str.name # Location of the str r-data
)


# You can see that my.data is a list, with three elements: info, costs, crops
str(my.str.data.2009.BEL)
Xinxin Yang's avatar
Xinxin Yang committed

# You can access each individual element like this
str(my.str.data.2009.BEL$info)
str(my.str.data.2009.BEL$costs) # NULL
str(my.str.data.2009.BEL$crops)
Xinxin Yang's avatar
Xinxin Yang committed


# The first columns of each of the above elements (info, costs, crops)
#   are created according to the ID section of the raw_str_map
names(my.str.data.2009.BEL$info)
names(my.str.data.2009.BEL$costs) # NULL
names(my.str.data.2009.BEL$crops)
Xinxin Yang's avatar
Xinxin Yang committed


# info and costs data.tables are in wide-format (each observation in a single row,
#   all attributes of a single observation in different columns).
# crops element is in long format (one observation is in many rows,
#
#
# See https://seananderson.ca/2013/10/19/reshape/ for
#   discussion of the two types of data formats
head(my.str.data.2009.BEL$info)
head(my.str.data.2009.BEL$costs) # NULL
head(my.str.data.2009.BEL$crops)
Xinxin Yang's avatar
Xinxin Yang committed



# Also on the attributes section of each of the above elements, we can access
#   the column formulas and descriptions, as defined in the raw_str_map file.
View(
  attr(my.str.data.2009.BEL$info,"column.descriptions")
Xinxin Yang's avatar
Xinxin Yang committed
)
# View(
#   attr(my.str.data.2009.BEL$costs,"column.descriptions")
# ) # NULL
Xinxin Yang's avatar
Xinxin Yang committed
View(
  attr(my.str.data.2009.BEL$crops,"column.descriptions")
Xinxin Yang's avatar
Xinxin Yang committed
)


# Especially for the crops element, we can also see the description
#   CROP column
View(
  attr(my.str.data.2009.BEL$crops,"crops.descriptions")
Xinxin Yang's avatar
Xinxin Yang committed
)

#################################################################
##              LOAD COUNTRIES-YEARS COMBINATIONS              ##
#################################################################
###  In the previous examples, we showed how to load data for one country and
###  one year In the following examples we show more combinations.
Xinxin Yang's avatar
Xinxin Yang committed

#To load for DEU and NED for year 2015
my.data = load.fadn.str.rds(countries = c("DEU","NED"), years = c(2009,2010,2011), extraction_dir = new.str.name )


#To load for DEU and NED for all years
my.data = load.fadn.str.rds(countries = c("DEU","NED"),extraction_dir = new.str.name )


#To load all available countries for year 2015
my.data = load.fadn.str.rds(years = 2015, extraction_dir = new.str.name)


#To load all availabel data
my.data = load.fadn.str.rds(extraction_dir = new.str.name)

#################################################################
##                    HOW TO STORE THE LOAD                    ##
#################################################################

#TODOS

# Since loading data sometimes takes time and create big datasets
# fadUtils offers a way to save the dataset created from the load call

# The first step is to store the loaded data
# Provide the object to save, a name and a description
# store.rds.data(my.data,"everything","all countries and years are here")

############################################################################
############################################################################
###                                                                      ###
Xinxin Yang's avatar
Xinxin Yang committed
###                           PERFORM ANALYSIS                           ###
###                                                                      ###
############################################################################
############################################################################

#We load structured data for all available countries and years
my.str.data = load.fadn.str.rds(extraction_dir = "a")
Xinxin Yang's avatar
Xinxin Yang committed

##----------------------------------------------------------------
##         HOW MANY FARMS FOR EACH COUNTY AND EACH YEAR         --
##----------------------------------------------------------------
# we use the info DT, and group by YEAR-COUNTRY
my.str.data$info[,.N,by=list(YEAR,COUNTRY)]

#We can also use dcast, to show a more tabular format
dcast(
  my.str.data$info,
  YEAR~COUNTRY,
  fun.aggregate = length,
  value.var =
)

# We can also export to clipboard, using the write.excel utility function
#   After running the following command, open excel and paste. The result will appear.
write.excel(
  dcast(
    my.str.data$info,
    YEAR~COUNTRY,
    fun.aggregate = length,
    value.var =
  )
)

##---------------------------------------------------------------
##               ALL CROP AREAS PER COUNTRY-YEAR               --
##---------------------------------------------------------------

# First, calculate the weighted area
my.str.data$crops[
  VARIABLE=="LEVL",
  VALUE.w:=WEIGHT*VALUE/1000
  ]

# Then dcast that variable
dcast(
  my.str.data$crops[VARIABLE=="LEVL"],
  COUNTRY+CROP~YEAR,
  value.var = "VALUE.w",
  fun.aggregate = sum,
  na.rm = T
)


##---------------------------------------------------------------
##             ALL CROP PRODUCTION PER COUNTRY-YEAR            --
##---------------------------------------------------------------


dcast(
  my.str.data$crops[VARIABLE=="GROF",VALUE.w:=WEIGHT*VALUE/1000],
  COUNTRY+CROP~YEAR,
  value.var = "VALUE.w",
  fun.aggregate = sum,
  na.rm = T
)


##---------------------------------------------------------------
##             BARLEY PRODUCTION PER COUNTRY-YEAR            --
##---------------------------------------------------------------
dcast(
  my.str.data$crops[
    VARIABLE=="GROF" & CROP=="BARL",
    VALUE.w:=WEIGHT*VALUE/1000
    ],
  COUNTRY~YEAR,
  value.var = "VALUE.w",
  fun.aggregate = sum,
  na.rm = T
)

##----------------------------------------------------------------
##       DISTRIBUTION OF NUMBER OF CROPS PER COUNTRY-YEAR       --
##----------------------------------------------------------------

crops.data = my.str.data$crops #catering for easier access at next steps

#this contains the number of crops for each farm-country-year/
#   Be carefule, we hav to filter to count only the LEVL variable
crops.data.Ncrops = crops.data[VARIABLE=="LEVL",.N,by=list(COUNTRY,YEAR,ID)]

# This displays the quantiles of the number of crops
crops.data.Ncrops[,as.list(quantile(N)),by=list(YEAR,COUNTRY)][order(COUNTRY)]

# R excels on graphic representation of results
library(ggplot2)

ggplot(crops.data.Ncrops,aes(y=N,x=1)) +
  geom_boxplot() +
  facet_grid(YEAR~COUNTRY) +
  theme(axis.title.x=element_blank(),
        axis.text.x=element_blank(),
        axis.ticks.x=element_blank()
  )+
  ylab("Number of Crops")

##---------------------------------------------------------------------------
##  COLLECT COMMON ID FROM LOADED STRUCTURED R-DATA AND LOADED RAW R_DATA  --
##---------------------------------------------------------------------------

# Collection the common id from loaded str r-data
collected.common.id_str = collect.common.id(my.str.data)

# Collection the common id from loaded raw r-data
collected.common.id_raw = collect.common.id(my.data)


##---------------------------------------------------------------
##          CALCULATION BASED ON COLLECETED COMMON ID          --
##---------------------------------------------------------------

# sommaries for infos

# sample and representend number of farms
my.str.data$info[,list(Nobs_sample=.N,Nobs_represented=sum(WEIGHT)),
                 by=.(COUNTRY,YEAR)]

# only for full sample (common id over years in selected data)
my.str.data$info[ID %in% collected.common.id_str[[1]],
Xinxin Yang's avatar
Xinxin Yang committed
                 list(Nobs_sample=.N,
                      Nobs_represented=sum(WEIGHT)),
                 by=.(COUNTRY,YEAR)]


# some summaries for crops

# unweighted and weighted sum over countries, years, crops and variables (EAAP  GROF  INTF  LEVL SHARE  UVAG  UVSA)
# EAA: Economic Accounts of Agriculture
# EAAP: EAA value at producer prices
# GROF

my.str.data$crops[ ,list(VALUE=sum(VALUE),
                         VALUE_weighted=sum(VALUE*WEIGHT)),
                   by=.(COUNTRY,YEAR,CROP,VARIABLE)]

# only for full sample (common id over years in selected data)
my.str.data$crops[ID %in% collected.common.id_str[[1]],
                  list(VALUE=sum(VALUE),
                       VALUE_weighted=sum(VALUE*WEIGHT)),
                  by=.(COUNTRY,YEAR,CROP,VARIABLE)]


my.str.data$crops[ID %in% collected.common.id_str[,common_id],
                  list(VALUE=sum(VALUE),
                       VALUE_weighted=sum(VALUE*WEIGHT)),
                  by=.(COUNTRY,YEAR,CROP,VARIABLE,ID)]

Xinxin Yang's avatar
Xinxin Yang committed
##---------------------------------------------------------------
##  Load fadn raw data and search the the number of common id for adjacent combination years
##---------------------------------------------------------------


"find all adjacent combinations in a list"
Xinxin Yang's avatar
Xinxin Yang committed
myFun <- function(Data) {
  A <- lapply(1:(length(Data)), sequence)
  B <- lapply(rev(lengths(A))-1L, function(x) c(0, sequence(x)))
  unlist(lapply(seq_along(A), function(x) {
    lapply(B[[x]], function(y) Data[A[[x]]+y])
  }), recursive = FALSE, use.names = FALSE)
}

"add a string to the facet label text and split it in two lines"
Xinxin Yang's avatar
Xinxin Yang committed
label_facet <- function(original_var, custom_name){
  lev <- levels(as.factor(original_var))
  lab <- paste0(lev, " \n ",custom_name)
  names(lab) <- lev
  return(lab)
}

"multi-panel plots using facet_wrap() for dynamic choice"
Xinxin Yang's avatar
Xinxin Yang committed
figure <- function(country, df, n){
Xinxin Yang's avatar
Xinxin Yang committed
  p = df %>%
    # reorder by Num_id
    #ggplot( aes(x = reorder(Years, -Num_id) ,y=Num_id))
    ggplot( aes(x = Years ,y=Num_id))+
    geom_bar( stat="identity",
Xinxin Yang's avatar
Xinxin Yang committed
              position = position_dodge(width = 0.8),
              width=0.5,
              #fill = rainbow(n=length(df$Num_id))
Xinxin Yang's avatar
Xinxin Yang committed
              fill = "#00abff"
    ) +
    coord_flip()+
    #labs(title = paste0("Plot of the Number of common ID for country: ", country ), fill = "Years") +
    xlab("Years") +
    ylab("Number of common ID") +
    geom_text(aes(label=Num_id), vjust=0.5, colour="black", size=3.5)+
    theme(axis.text.x=element_text(color = "black", size=6, angle=0, vjust=.8, hjust=0.8)) +
    scale_x_discrete(labels = function(x) str_wrap(x, width = n)) +
    facet_wrap( ~ group, scales="free",
                #labeller=names
                labeller = labeller(group = label_facet(df$group, "adjacent combinations"))
    )+
    ggtitle(paste0("Number of common ID for country: ", country )) +
    theme_bw() +
    theme(plot.title = element_text(hjust = 0.5))
  p
}

"load raw data and get the number of common id for selected countries over all exist adjacent combinations years.
then save the number of common id to an excel sheet and plot"
output_common_id <- function(countries_list, saveExcel = TRUE, excelname , savePlots = TRUE){
Xinxin Yang's avatar
Xinxin Yang committed

  rds.dir = paste0(get.data.dir(),"/rds/")
  plots.dir = paste0(get.data.dir(), "/plots/")
  if (!dir.exists(plots.dir)) dir.create(plots.dir)


Xinxin Yang's avatar
Xinxin Yang committed
  library(xlsx)
  library(openxlsx)

  xlsx_file_dir <- paste0(get.data.dir(), "/spool/")


  if (saveExcel==TRUE) {wb <- createWorkbook(paste0(xlsx_file_dir, excelname))}


Xinxin Yang's avatar
Xinxin Yang committed
  outlist = list()
  for (country in countries_list){

Xinxin Yang's avatar
Xinxin Yang committed
    cat("Country:", country, '\n')
Xinxin Yang's avatar
Xinxin Yang committed
    raw_file_names <- dir(rds.dir, pattern = paste0(country, "\\.rds$") )
    years_list = as.numeric(gsub("\\D+", "", raw_file_names))
Xinxin Yang's avatar
Xinxin Yang committed
    adjacent_list = myFun(years_list)

Xinxin Yang's avatar
Xinxin Yang committed
    my.data = list()
Xinxin Yang's avatar
Xinxin Yang committed
    for (year_items in adjacent_list) {
Xinxin Yang's avatar
Xinxin Yang committed
      name = toString(year_items)
Xinxin Yang's avatar
Xinxin Yang committed
      data = load.fadn.raw.rds(countries = country, years = year_items)
Xinxin Yang's avatar
Xinxin Yang committed
    }

    Big.Num.Common.id = list()
Xinxin Yang's avatar
Xinxin Yang committed
    for (data_list in names(my.data)){
Xinxin Yang's avatar
Xinxin Yang committed
      common.id = collect.common.id(my.data[[data_list]])

      Big.Num.Common.id[[data_list]] = nrow(common.id)
    }
    DF = do.call(rbind, Big.Num.Common.id)
Xinxin Yang's avatar
Xinxin Yang committed
    DF = data.frame(DF)
    colnames(DF) <-  "Num_id"
    DF$Years <-  row.names(DF)
Xinxin Yang's avatar
Xinxin Yang committed
    outlist[[country]] = DF

    if (!is.null(wb)) {
      if (!(country %in% names(wb))) {
        addWorksheet(wb, country)}
      writeData(wb,country, DF)
      }
Xinxin Yang's avatar
Xinxin Yang committed
    if (savePlots == TRUE){
      library(ggplot2)
      library(stringr)
      DF$Length <- str_count(DF$Years)
      DF$group <- cut(DF$Length, breaks=c(1,5,14,18,25,30,35,40,48,55,58,70,Inf))
      levels(DF$group) <- c("1 year","2 years","3 years","4 years","5 years","6 years","7 years",
                                      "8 years","9 years","10 years","11, 12 years",">12 years")
Xinxin Yang's avatar
Xinxin Yang committed
      if (length(years_list) >=15){
Xinxin Yang's avatar
Xinxin Yang committed
        p <- figure(country,DF, 35)}
      else{p <- figure(country,DF, 20)}
Xinxin Yang's avatar
Xinxin Yang committed
      ggsave(plot = p,
             filename = paste0(plots.dir,country ,"_plot.png"),
             width = 18, height = 8)
      }
    }
    if (saveExcel == TRUE) {
      saveWorkbook(wb, paste0(xlsx_file_dir, excelname), overwrite = T)
      cat(excelname," is saved in ",xlsx_file_dir, "\n")}
    if (savePlots == TRUE) cat("plots are saved in", plots.dir, "\n")

Xinxin Yang's avatar
Xinxin Yang committed
  return(outlist)
Xinxin Yang's avatar
Xinxin Yang committed

# get all countires in fadn str data
countries = unique(my.str.data$info$COUNTRY)
Xinxin Yang's avatar
Xinxin Yang committed

ID_list <- output_common_id(countries_list = countries)
Xinxin Yang's avatar
Xinxin Yang committed

# get Germany: DEU and Kroatien: HRV
DEU_list <- output_common_id(c("HRV", "DEU"), saveExcel = TRUE, excelname = "HRV_DEU.xlsx", savePlots = TRUE)