Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
# This script provides some sample commands of fadnUtils
#
# The functions of the package are deliberately written as "fadnUtils::<FUNCTION>", in order to be
# easily identified as such. However one can write them skipping the "fadnUtils::" part
#
# For any of the functions of the package, you can see its documentation
# by either writing ?<function name> or by hovering the mouse on its name and pressing F1 (in rstudio)
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
#
# LOAD THE PACKAGE -----------------------------------------------------------------------------------------------------
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
library(data.table) #required
library(fadnUtils)
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
#
# IMPORT FADN CSV FILES INTO raw.rds -----------------------------------------------------------------------------------
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
# Step 1, Create a data.dir
fadnUtils::create.data.dir("E:/test_data_dir") #replace the path with the one you want your data.dir to reside to
# after running the command, go to this directory to see its structure
# Step 2, Set the data.dir
fadnUtils::set.data.dir("E:/test_data_dir") # The package needs to know the location of the data.dir it works with
# if you do not define, any following fadnUtil command will fail with an error
# Step 2, Import the csv files
countries = c("ELL","ITA","ESP") # select the countries to import data for
years = c(2013,2014,2015) # select the years to import data for
path.to.csv.files = "<PUT THE FOLDER WHERE CSV FILE RESIDE>" # We assume that the CSV files are are located in an external directory
path.to.csv.files ="U:/SCIENTIFIC/IFM CAP/70-Data/FADN/10_Requests/IFM-CAP/2019/Baseyear/Data/SO"
for(c in countries) { #Loop countries and years and import, one by one
for (y in years) { # after the loop, check the data.dir/rds directory to see the rds files there
file.path.cur = paste0(path.to.csv.files,'/',c,y,".csv")
if(file.exists(file.path.cur)) {
print(paste0("doing year:",y," and country:",c))
fadnUtils::convert.to.fadn.raw.rds(file.path = file.path.cur, fadn.year = y, fadn.country = c)
}
else {
print(paste0("DOES NOT EXIST for year:",y," and country:",c, "/ ", file.path.cur))
}
}
}
fadnUtils::show.data.dir.contents() #It will show the country-year pairs available
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
#
# WORK WITH raw.rds ----------------------------------------------------------------------------------------------------
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
# Step 1, set data.dir (in this case, this step is not necessary, since data.dir is already set with a previous command)
fadnUtils::set.data.dir("E:/test_data_dir")
# Step 2, load data
#To load raw r-data, only for Spain (ESP) for 2015
my.data = load.fadn.raw.rds(
countries = "ESP",
years = 2015
)
# my.data is a single large data.table, with the original csv columns and rows
nrow(my.data) #Number of rows
names(my.data) #Column names
length(names(my.data)) #Number of columns
str(my.data) #Overall structure
# You can also load for combinations of COUNTRY-YEAR
my.data = load.fadn.raw.rds(
countries = c("ELL", "ESP"),
years = c(2014,2015)
)
#If you do not define country, year, all data of the data.dir is loaded
my.data = load.fadn.raw.rds()
# You can also load a subset of the columns
my.data = load.fadn.raw.rds(
col.filter = c("ID","NUTS0","NUTS2", "SYS02","TF8","IELE_V") #Load only id, nuts2, weight, tf8 and the electricity expenses for all data.dir
)
# You can also load a subset of the rows
# Here, load only farms that belong to TF8-16, for
# all years and countries
my.data = load.fadn.raw.rds(
row.filter = "TF8==16"
)
str(my.data) #gave a look at the structure of the data. It contains only a few columns
# see that always there are two columns: load.YEAR and load.COUNTRY. They denote the loading point of the row
#Step 3, work with the data.table structure as you like.
# See example below that calculate the expenditure of electricity per country per year
my.data[,list(ELECTRICITY.th=sum(IELE_V* SYS02*0.001)),by=list(load.COUNTRY,load.YEAR)] #See comment above for load.COUNTRY,load.YEAR
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
#
# AGGREGATE raw.rds into str.rds------------------------------------------------------------------------------------------
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
# Step 1, set data.dir (in this case, this step is not necessary, since data.dir is already set with a previous command)
fadnUtils::set.data.dir("E:/test_data_dir")
# Step 2, create the raw_rds_map file
# You have to create it inside the "<data.dir>/raw_str_maps" folder. Name it as you like
# Let's suppose that there exist two such files: "2014_after.json" and "2013_before.json"
# The first (2014_after.json) is a full representation of the calculations for FADN data from 2014 and after
# The second (2013_before.json) has only the fields that must be calculated differently for the period of 2013 and before
# Step 3, aggregate the data
countries = c("ELL","ITA","ESP") # select the countries to import data for
for(c in countries) {
for (y in c(2014:2015)) {
print(paste0("doing year:",y," and country:",c))
convert.to.fadn.str.rds(fadn.year = y,
fadn.country = c,
raw.f = "2014_after.json") # we use one json for 2014 and after
}
}
# we need to merge the 2013 and 2014 and then copy it to a new json (2013 is differentnail to 2014)
merge_raw_f.path = fadnUtils::raw_str_map.merge(
source.raw_str_map.file = "2014_after.json",
new.raw_str_map.file = "2013_before.json",
return.file = T
)
file.copy(merge_raw_f.path,paste0(fadnUtils::get.data.dir(),"/raw_str_maps/merged.json"))
for(c in countries) {
for (y in c(2013)) {
print(paste0("doing year:",y," and country:",c))
convert.to.fadn.str.rds(fadn.year = y,
fadn.country = c,
raw.f = "merged.json") # we use another one json for 2013 (projection of 2013 on 2014)
}
}
fadnUtils::show.data.dir.contents() #It will show the country-year pairs available
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
#
# WORK WITH str.rds ----------------------------------------------------------------------------------------------------
#
#nununununununununununununununununununununununununununununununnunununununununununununununununununununununununununununununun#
# Step 1, set data.dir (in this case, this step is not necessary, since data.dir is already set with a previous command)
fadnUtils::set.data.dir("E:/test_data_dir")
# Step 2, Load str.rds data
my.data = fadnUtils::load.fadn.str.rds(countries = "ELL", years = c(2013:2015)) #The calculated variables are consistend for 2013 and 2014/15
str(my.data) # the loaded data is a list that contains data.tables
str(my.data$crops) #crops are in long format
# Step 3, Work with this data as regularly with r-scripting
dcast(
my.data$crops[VARIABLE=="GROF",list(GROF.mil=sum(0.001*VALUE*WEIGHT)),by=list(YEAR,COUNTRY,CROP)],
COUNTRY+CROP~YEAR, value.var="GROF.mil"
)