|
|
|
# find all adjacent combinations in a list
|
|
|
|
myFun <- function(Data) {
|
|
|
|
A <- lapply(2:(length(Data)-1L), sequence)
|
|
|
|
B <- lapply(rev(lengths(A))-1L, function(x) c(0, sequence(x)))
|
|
|
|
unlist(lapply(seq_along(A), function(x) {
|
|
|
|
lapply(B[[x]], function(y) Data[A[[x]]+y])
|
|
|
|
}), recursive = FALSE, use.names = FALSE)
|
|
|
|
}
|
|
|
|
# load fadn raw data
|
|
|
|
# search the number of common id for adjacent combination years
|
|
|
|
output_common_id <- function(countires_list){
|
|
|
|
|
|
|
|
rds.dir = paste0(get.data.dir(),"/rds/")
|
|
|
|
|
|
|
|
# set a str name for for saving the str r-data in rds.dir
|
|
|
|
|
|
|
|
for (country in countires_list){
|
|
|
|
|
|
|
|
new.str.name = country
|
|
|
|
cat("Country:", country, '\n')
|
|
|
|
|
|
|
|
# set a extraction_dir
|
|
|
|
dir.create(paste0(rds.dir, new.str.name))
|
|
|
|
new.extraction.dir = paste0(rds.dir, new.str.name)
|
|
|
|
|
|
|
|
# count the number of the years for one country
|
|
|
|
|
|
|
|
years = df[df$country== country, ]$Year
|
|
|
|
|
|
|
|
adjacent_list = myFun(years)
|
|
|
|
adjacent_list[[length(adjacent_list)+1]] = years
|
|
|
|
|
|
|
|
my.data = list()
|
|
|
|
for (year_items in adjacent_list) {
|
|
|
|
name = toString(year_items)
|
|
|
|
|
|
|
|
data = load.fadn.raw.rds(countries = country, years = year_items)
|
|
|
|
my.data[[name]] = data
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Big.Num.Common.id = list()
|
|
|
|
|
|
|
|
for (data_list in 1:length(my.data)){
|
|
|
|
|
|
|
|
data = my.data[data_list]
|
|
|
|
|
|
|
|
# Retrieving column names
|
|
|
|
name = names(data)
|
|
|
|
|
|
|
|
print("******************************")
|
|
|
|
|
|
|
|
#colnames(data[[name]])[which( names(data[[name]]) == "ID")] <- "id"
|
|
|
|
|
|
|
|
common.id = collect.common.id(data[[name]])
|
|
|
|
|
|
|
|
|
|
|
|
Big.Num.Common.id[[name]] = nrow(common.id)
|
|
|
|
}
|
|
|
|
DF = do.call(rbind, Big.Num.Common.id)
|
|
|
|
|
|
|
|
DF = data.frame(DF)
|
|
|
|
colnames(DF) <- "number_of_common_id"
|
|
|
|
|
|
|
|
|
|
|
|
DF$Years = rownames(DF)
|
|
|
|
|
|
|
|
# # write xlsx
|
|
|
|
# write.xlsx(DF,
|
|
|
|
# file="D:/public/yang/MIND_STEP/new_sample/DEUData_common_id.xlsx",
|
|
|
|
# sheetName = country,
|
|
|
|
# col.names= TRUE,
|
|
|
|
# row.names = TRUE,
|
|
|
|
# append = T)
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
output_common_id("BEL")
|
|
|
|
|
|
|
|
# convert fadn raw into str data
|
|
|
|
raw2str <- function(Current_raw_str_map.file = NULL, overwrite_external_json = F, countires_list){
|
|
|
|
|
|
|
|
for (country in countires_list){
|
|
|
|
print("**********************************")
|
|
|
|
new.str.name = country
|
|
|
|
|
|
|
|
|
|
|
|
rds.dir = paste0(get.data.dir(), "/rds")
|
|
|
|
|
|
|
|
raw_file_names = dir(rds.dir, pattern = paste0(country,".","rds$"))
|
|
|
|
|
|
|
|
unlink(paste0(rds.dir,"/", country), recursive=TRUE)
|
|
|
|
|
|
|
|
for (file in raw_file_names){
|
|
|
|
|
|
|
|
# extract first 3 char
|
|
|
|
country = substr(file, 15, 17)
|
|
|
|
|
|
|
|
# extract number
|
|
|
|
year = as.numeric(gsub("\\D+", "", file))
|
|
|
|
|
|
|
|
|
|
|
|
cat("converting the str data for country: ", country, " and year: ", year, "\n")
|
|
|
|
tryCatch(
|
|
|
|
expr = {
|
|
|
|
convert.to.fadn.str.rds(fadn.country = country,
|
|
|
|
fadn.year = year,
|
|
|
|
raw_str_map.file = Current_raw_str_map.file,
|
|
|
|
force_external_raw_str_map = overwrite_external_json,
|
|
|
|
str.name = country)
|
|
|
|
},
|
|
|
|
warning = function(w){
|
|
|
|
message('Caught an warning!')
|
|
|
|
print(w)
|
|
|
|
},
|
|
|
|
error = function(e) {
|
|
|
|
message("Caught an error! Please check the objects in json file using check.column() (see more in USE_CASE_4.R).")
|
|
|
|
#cat("Wrong, can't convert the str r-data!",sep = "\n")
|
|
|
|
print(e)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
raw2str(Current_raw_str_map.file ="D:/public/yang/MIND_STEP/new_sample/raw_str_maps/rewrite_2014_after.json",
|
|
|
|
countires_list = countires) |