diff --git a/R/xfaostat_L102_ProductionArea.R b/R/xfaostat_L102_ProductionArea.R index 9de75453..3e78bcc0 100644 --- a/R/xfaostat_L102_ProductionArea.R +++ b/R/xfaostat_L102_ProductionArea.R @@ -23,11 +23,11 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { "FBSH_CBH_wide") MODULE_OUTPUTS <- - c("QCL_PROD", - "QCL_AN_LIVEANIMAL", - "QCL_AN_PRIMARY_MILK", - "QCL_AN_LIVEANIMAL_MEATEQ", - "QCL_CROP_PRIMARY") + c("L102.QCL_PROD", + "L102.QCL_AN_LIVEANIMAL", + "L102.QCL_AN_PRIMARY_MILK", + "L102.QCL_AN_LIVEANIMAL_MEATEQ", + "L102.QCL_CROP_PRIMARY") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -378,7 +378,7 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { - # Bind QCL_ALL and save RDS ---- + # Bind L102.QCL_ALL and save RDS ---- #******************************************* QCL_CROP_PRIMARY %>% mutate(item_set = "QCL_COMM_CROP_PRIMARY") %>% @@ -392,10 +392,10 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { bind_rows(QCL_AN_LIVEANIMAL %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL")) %>% bind_rows(QCL_AN_LIVEANIMAL_MEATEQ %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL_MEATEQ")) %>% bind_rows(FBS_FISH %>% mutate(item_set = "FBS_COMM_FISH"))-> - QCL_ALL + L102.QCL_ALL - QCL_ALL %>% + L102.QCL_ALL %>% filter(item_set %in% c("QCL_COMM_AN_PRIMARY_MEAT1", "QCL_COMM_AN_PRIMARY_MEAT2", "QCL_COMM_AN_PRIMARY_EGG", @@ -408,60 +408,60 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { add_units("various") %>% add_comments("Detailed FAO QCL data processing for live animal and production") %>% add_precursors("QCL_wide", "FBS_wide", "FBSH_CBH_wide") -> - QCL_AN_LIVEANIMAL + L102.QCL_AN_LIVEANIMAL QCL_AN_PRIMARY_MILK %>% add_title("FAO milk animal stock and production") %>% add_units("various") %>% add_comments("Detailed FAO QCL data processing for dairy animal and production") %>% - same_precursors_as(QCL_AN_LIVEANIMAL) -> - QCL_AN_PRIMARY_MILK + same_precursors_as(L102.QCL_AN_LIVEANIMAL) -> + L102.QCL_AN_PRIMARY_MILK QCL_AN_LIVEANIMAL_MEATEQ %>% add_title("FAO live animal stock meat equivalent") %>% add_units("various") %>% add_comments("Detailed FAO QCL data processing for live animal stock meat equivalent") %>% - same_precursors_as(QCL_AN_LIVEANIMAL) -> - QCL_AN_LIVEANIMAL_MEATEQ + same_precursors_as(L102.QCL_AN_LIVEANIMAL) -> + L102.QCL_AN_LIVEANIMAL_MEATEQ QCL_CROP_PRIMARY %>% add_title("FAO primary crop area and production") %>% add_units("various") %>% add_comments("Detailed FAO QCL data processing for crop area and production") %>% - same_precursors_as(QCL_AN_LIVEANIMAL) -> - QCL_CROP_PRIMARY + same_precursors_as(L102.QCL_AN_LIVEANIMAL) -> + L102.QCL_CROP_PRIMARY # Production only - QCL_ALL %>% filter(element_code == 5510) -> - QCL_PROD + L102.QCL_ALL %>% filter(element_code == 5510) -> + L102.QCL_PROD - QCL_PROD %>% + L102.QCL_PROD %>% add_title("FAO primary production") %>% add_units("tonnes") %>% add_comments("FAO primary production") %>% - same_precursors_as(QCL_AN_LIVEANIMAL) -> - QCL_PROD + same_precursors_as(L102.QCL_AN_LIVEANIMAL) -> + L102.QCL_PROD # No NA - assertthat::assert_that(QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0) + assertthat::assert_that(L102.QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0) - #QCL_ALL %>% FF_check_count_plot -> p; p + #L102.QCL_ALL %>% FF_check_count_plot -> p; p # ggsave(file.path(DIR_DATAPROC_PLOT, "QCL_ALL.png"), # plot = p + ggtitle("gcamdata-FAOSTAT (QCL & FBS) production data over time"), # dpi = 200, width = 9, height = 5 ) # rm(p) - QCL_ALL %>% distinct(year); # 53 years - QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code - QCL_ALL %>% distinct(item) # 158 primary crop + 45 primary an + 57 others + 17 +12 + L102.QCL_ALL %>% distinct(year); # 53 years + L102.QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code + L102.QCL_ALL %>% distinct(item) # 158 primary crop + 45 primary an + 57 others + 17 +12 - # QCL_ALL %>% + # L102.QCL_ALL %>% # add_title("FAO crop and livestock production and crop area") %>% # add_units("various") %>% # add_comments("Detailed FAO QCL data processing. FBS fish data is used") -> - # QCL_ALL + # L102.QCL_ALL return_data(MODULE_OUTPUTS) diff --git a/R/xfaostat_L103_ProducerPrices.R b/R/xfaostat_L103_ProducerPrices.R index 3a6b4de5..8799d001 100644 --- a/R/xfaostat_L103_ProducerPrices.R +++ b/R/xfaostat_L103_ProducerPrices.R @@ -18,11 +18,11 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) { MODULE_INPUTS <- - c("QCL_PROD", + c("L102.QCL_PROD", "PP_wide") MODULE_OUTPUTS <- - c("QCL_PRIMARY_PROD_PV") + c("L103.QCL_PRIMARY_PROD_PV") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -32,7 +32,7 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) { year <- value <- Year <- Value <- FAO_country <- iso <- NULL # silence package check. PP_wide <- element_code <- element <- area_code <- item_code <- area <- - item <- unit <- QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <- + item <- unit <- L102.QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <- PP_item <- Production <- Prod_Value <- PP_index <- World_PP_index <- PP_Multiplier <- PP_item_world <- NULL @@ -50,15 +50,15 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) { # Primary crops - QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>% + L102.QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>% distinct(item, item_code) -> QCL_COMM_CROP_PRIMARY # Primary animal products, including fat hides etc. 46 - QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>% + L102.QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>% distinct(item, item_code) -> QCL_COMM_AN_PRIMARY QCL_COMM_CROP_PRIMARY %>% bind_rows(QCL_COMM_AN_PRIMARY) %>% - left_join(QCL_PROD, by = c("item_code", "item")) -> + left_join(L102.QCL_PROD, by = c("item_code", "item")) -> QCL_PRIMARY QCL_PRIMARY %>% distinct(item, element, item_set, unit) @@ -129,17 +129,17 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) { left_join(UnitMap %>% bind_rows(UnitMap %>% mutate(element = "Prod_Value", unit = "USD")), by = "element") -> - QCL_PRIMARY_PROD_PV + L103.QCL_PRIMARY_PROD_PV rm(QV1, QV) - QCL_PRIMARY_PROD_PV %>% + L103.QCL_PRIMARY_PROD_PV %>% add_title("FAO crop and livestock production and crop area") %>% add_units("USD and tonne") %>% add_comments("Detailed FAO QCL data processing. FBS fish data is used") %>% - add_precursors("QCL_PROD", + add_precursors("L102.QCL_PROD", "PP_wide") -> - QCL_PRIMARY_PROD_PV + L103.QCL_PRIMARY_PROD_PV return_data(MODULE_OUTPUTS) diff --git a/R/xfaostat_L104_ProductionAreaAddFodder.R b/R/xfaostat_L104_ProductionAreaAddFodder.R index 3083b0e9..bda2429b 100644 --- a/R/xfaostat_L104_ProductionAreaAddFodder.R +++ b/R/xfaostat_L104_ProductionAreaAddFodder.R @@ -22,7 +22,7 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) { "QCL_area_code_map") MODULE_OUTPUTS <- - c("QCL_FODDERCROP") + c("L104.QCL_FODDERCROP") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -95,17 +95,17 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) { # Remove area x year that should no exist FAOSTAT_AREA_RM_NONEXIST %>% left_join(UnitMap, by = "element") -> - QCL_FODDERCROP + L104.QCL_FODDERCROP rm(FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011, UnitMap) - QCL_FODDERCROP %>% + L104.QCL_FODDERCROP %>% add_title("Processed fodder crop production and area") %>% add_units("tonne and ha") %>% add_comments("Data is from old GCAM data v5.4") %>% add_precursors(file.path(DIR_RAW_DATA_FAOSTAT, "Other_supplementary/FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011"), "QCL_area_code_map")-> - QCL_FODDERCROP + L104.QCL_FODDERCROP # FAO_ag_items_PRODSTAT is not read in anymore diff --git a/R/xfaostat_L105_DataConnectionToSUA.R b/R/xfaostat_L105_DataConnectionToSUA.R index 183c4c02..5d740ab9 100644 --- a/R/xfaostat_L105_DataConnectionToSUA.R +++ b/R/xfaostat_L105_DataConnectionToSUA.R @@ -19,8 +19,8 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { MODULE_INPUTS <- c(FILE = file.path(DIR_RAW_DATA_FAOSTAT, "FAO_items"), - "QCL_PROD", - "QCL_AN_LIVEANIMAL_MEATEQ", + "L102.QCL_PROD", + "L102.QCL_AN_LIVEANIMAL_MEATEQ", "TCL_wide", "TM_bilateral_wide", "FBSH_CBH_wide", @@ -28,7 +28,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { "SCL_wide") MODULE_OUTPUTS <- - c("Bal_new_all") + c("L105.Bal_new_all") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -39,10 +39,10 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { year <- value <- Year <- Value <- FAO_country <- iso <- NULL # silence package check. SCL_wide <- element_code <- element <- area_code <- item_code <- area <- item <- unit <- FBS_wide <- FBSH_CBH_wide <- TCL_wide <- TM_bilateral_wide <- - QCL_PROD <- FAO_items <- tier <- QCL <- oil <- + L102.QCL_PROD <- FAO_items <- tier <- QCL <- oil <- cake <- SCL_item_oil <- SCL_item_cake <- cake_rate <- cake_rate_world <- DS_key_coproduct_item <- Production <- Import <- Export <- DS_demand <- - DS_production <- CoproductRate <- QCL_AN_LIVEANIMAL_MEATEQ <- `Closing stocks` <- + DS_production <- CoproductRate <- L102.QCL_AN_LIVEANIMAL_MEATEQ <- `Closing stocks` <- `Opening stocks` <- `Stock Variation` <- NULL all_data <- list(...)[[1]] @@ -78,7 +78,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { # Get area code in QCL that is consistent with FBS e.g., after 2010 only - QCL_PROD %>% filter(year >= min (FAOSTAT_Hist_Year_FBS)) %>% distinct(area_code) %>% pull -> + L102.QCL_PROD %>% filter(year >= min (FAOSTAT_Hist_Year_FBS)) %>% distinct(area_code) %>% pull -> QCL_area_code_FBS ## 1.2. Get FAO supply-utilization SCL ready ---- @@ -161,7 +161,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { # Merge Sudan regions to be consistent with data # Mainly for storage data concerns # And only keep data > min(FAOSTAT_Hist_Year_FBS) - for (.DF in c("SCL", "TCL_TM", "TCL_gross", "FBSH_CBH", "FBS", "QCL_PROD")) { + for (.DF in c("SCL", "TCL_TM", "TCL_gross", "FBSH_CBH", "FBS", "L102.QCL_PROD")) { get(.DF) %>% filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% # merge Sudan and South Sudan FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION_ALL(SUDAN2012_MERGE = T) %>% @@ -169,7 +169,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { # Update area code in QCL - QCL_PROD %>% filter(year %in% FAOSTAT_Hist_Year_FBS) %>% distinct(area_code) %>% pull -> + L102.QCL_PROD %>% filter(year %in% FAOSTAT_Hist_Year_FBS) %>% distinct(area_code) %>% pull -> QCL_area_code_FBS # 2. Create helper functions to simplify join by data set ---- @@ -214,19 +214,19 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { ## Start SUA_TEMPLATE_LEFT_JOIN ---- - ## a. Join QCL_PROD when .DS == "QCL" ---- + ## a. Join L102.QCL_PROD when .DS == "QCL" ---- if (.DS == "QCL") { - # assert QCL_PROD exist - assertthat::assert_that(is.data.frame(QCL_PROD)) + # assert L102.QCL_PROD exist + assertthat::assert_that(is.data.frame(L102.QCL_PROD)) # assert items exist in joined DF assertthat::assert_that(.DF %>% distinct(item_code) %>% pull %>% - setdiff(QCL_PROD %>% distinct(item_code) %>% pull) %>% + setdiff(L102.QCL_PROD %>% distinct(item_code) %>% pull) %>% length() == 0 ) # Join .DF %>% left_join( - QCL_PROD %>% + L102.QCL_PROD %>% select(area_code, item_code, element, year, QCL = value), by = c("area_code", "item_code", "element", "year") ) -> .DF1 @@ -237,7 +237,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { ## b. Join TCL_TM when .DS == "TM" ---- if (.DS == "TM") { - # assert QCL_PROD exist + # assert L102.QCL_PROD exist assertthat::assert_that(is.data.frame(TCL_TM)) if (.DS_TM_Assert_Item == T) { @@ -259,7 +259,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { ## c. Join TCL_gross when .DS == "TCL_gross" ---- if (.DS == "TCL_gross") { - # assert QCL_PROD exist + # assert TCL exist assertthat::assert_that(is.data.frame(TCL_gross)) if (.DS_TM_Assert_Item == T) { @@ -280,7 +280,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { ## d. Join SCL when .DS == "SCL" ---- if (.DS == "SCL") { - # assert QCL_PROD exist + # assert SCL exist assertthat::assert_that(is.data.frame(SCL)) # assert items exist in joined DF @@ -580,7 +580,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { ### 3.8.1 Process the live animal meat equivalent data APE_live_an_MeatEQ ---- - # read in QCL_AN_LIVEANIMAL_MEATEQ live animal meat equivalent + # read in L102.QCL_AN_LIVEANIMAL_MEATEQ live animal meat equivalent # Treat live animal as stock and adjust using production or other demand # Milk cattle is not included # Note that only stock variation is used @@ -588,7 +588,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { # But accounting delta allows more accurate estimate of feed uses # E.g., additional feed demand due to animal expansion - QCL_AN_LIVEANIMAL_MEATEQ %>% + L102.QCL_AN_LIVEANIMAL_MEATEQ %>% select(area_code, item_code, year, value) %>% #mutate(item = gsub("Meat", "AnMeatEq", item)) %>% # convert units back to tonne!!! And adjust item_code @@ -622,14 +622,14 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { Bal_new_tier8 assert_FBS_balance(.DF = Bal_new_tier8) - rm(QCL_AN_LIVEANIMAL_MEATEQ, APE_live_an_MeatEQ) + rm(L102.QCL_AN_LIVEANIMAL_MEATEQ, APE_live_an_MeatEQ) # 4. Bind all to get Bal_new_all ---- #[ToDo loop not working in package] # lapply(paste0("Bal_new_tier", 1:9), get) %>% bind_rows() %>% # # Add area_code - # left_join(QCL_PROD %>% distinct(area, area_code), by = "area_code") -> + # left_join(L102.QCL_PROD %>% distinct(area, area_code), by = "area_code") -> # Bal_new_all Bal_new_tier1 %>% @@ -641,28 +641,28 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { bind_rows(Bal_new_tier7) %>% bind_rows(Bal_new_tier8) %>% # Add area_code - left_join(QCL_PROD %>% distinct(area, area_code), by = "area_code")-> - Bal_new_all + left_join(L102.QCL_PROD %>% distinct(area, area_code), by = "area_code")-> + L105.Bal_new_all - assert_FBS_balance(.DF = Bal_new_all) + assert_FBS_balance(.DF = L105.Bal_new_all) rm(TCL_gross, TCL_TM, SCL, FBS, FBSH_CBH, FAO_items) rm(list = ls(pattern = "Bal_new_tier*")) - Bal_new_all %>% - add_title("Bal_new_all") %>% + L105.Bal_new_all %>% + add_title("L105.Bal_new_all") %>% add_units("Ktonne") %>% add_comments("Preprocessed FAO SUA 2010 - 2021") %>% add_precursors(file.path(DIR_RAW_DATA_FAOSTAT, "FAO_items"), - "QCL_PROD", - "QCL_AN_LIVEANIMAL_MEATEQ", + "L102.QCL_PROD", + "L102.QCL_AN_LIVEANIMAL_MEATEQ", "TCL_wide", "TM_bilateral_wide", "FBSH_CBH_wide", "FBS_wide", "SCL_wide")-> - Bal_new_all + L105.Bal_new_all return_data(MODULE_OUTPUTS) @@ -690,7 +690,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) { # # TCL_TM %>% mutate(item = 1) -> TCL_TM1 # -# FF_join_checkmap(c("SCL", "QCL_PROD", "TCL_TM1", "FBSH_CB"), COL_by = c("item_code"), COL_rename = "item" ) -> +# FF_join_checkmap(c("SCL", "L102.QCL_PROD", "TCL_TM1", "FBSH_CB"), COL_by = c("item_code"), COL_rename = "item" ) -> # JoinItemMap # # diff --git a/R/xfaostat_L106_FoodMacroNutrient.R b/R/xfaostat_L106_FoodMacroNutrient.R index aca1a8f5..53017d7e 100644 --- a/R/xfaostat_L106_FoodMacroNutrient.R +++ b/R/xfaostat_L106_FoodMacroNutrient.R @@ -25,7 +25,7 @@ module_xfaostat_L106_FoodMacroNutrient <- function(command, ...) { FILE = file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_FAO_FBS_SUA")) MODULE_OUTPUTS <- - c("SUA_food_macronutrient_rate") + c("L106.SUA_food_macronutrient_rate") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -235,11 +235,11 @@ module_xfaostat_L106_FoodMacroNutrient <- function(command, ...) { left_join(FAO_an_items_cal_SUA %>% select(item_code, calperg = Mcal_t,fatperc = fat_Perc, proteinperc = protein_Perc), by = "item_code" ) - ) -> SUA_food_macronutrient_rate + ) -> L106.SUA_food_macronutrient_rate - unique(SUA_food_macronutrient_rate$area_code) %>% length() - unique(SUA_food_macronutrient_rate$item_code) %>% length() - SUA_food_macronutrient_rate %>% distinct(item, item_code) -> SUA_COMM_FOOD_NUTRIENT + unique(L106.SUA_food_macronutrient_rate$area_code) %>% length() + unique(L106.SUA_food_macronutrient_rate$item_code) %>% length() + L106.SUA_food_macronutrient_rate %>% distinct(item, item_code) -> SUA_COMM_FOOD_NUTRIENT # 76244 =179 area * (414 items + 12 fish items) # remove processing data @@ -252,7 +252,7 @@ module_xfaostat_L106_FoodMacroNutrient <- function(command, ...) { - SUA_food_macronutrient_rate %>% + L106.SUA_food_macronutrient_rate %>% add_title("FAO food calories and macronutrient rate") %>% add_units("rates") %>% add_comments("Detailed FAO food calories and macrotunitent info for 414 SUA items + 12 fish items") %>% @@ -261,7 +261,7 @@ module_xfaostat_L106_FoodMacroNutrient <- function(command, ...) { "OA", file.path(DIR_RAW_DATA_FAOSTAT, "FAO_an_items_cal_SUA"), file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_FAO_FBS_SUA")) -> - SUA_food_macronutrient_rate + L106.SUA_food_macronutrient_rate # P.S. ---- # China Wheat, bran for food? Need to fix in SUA later; no changes needed here for this diff --git a/R/yextension_L100_FoodBalanceSheet.R b/R/xfaostat_L107_FoodBalanceSheet.R similarity index 86% rename from R/yextension_L100_FoodBalanceSheet.R rename to R/xfaostat_L107_FoodBalanceSheet.R index 2d75cc2c..9f99c2c7 100644 --- a/R/yextension_L100_FoodBalanceSheet.R +++ b/R/xfaostat_L107_FoodBalanceSheet.R @@ -1,13 +1,13 @@ # Copyright 2019 Battelle Memorial Institute; see the LICENSE file. -#' module_yextension_L100_FoodBalanceSheet +#' module_xfaostat_L107_FoodBalanceSheet #' -#' Generate supply utilization balance in primary equivalent +#' Generate FBS supply utilization balance in primary equivalent #' #' @param command API command to execute #' @param ... other optional parameters, depending on command #' @return Depends on \code{command}: either a vector of required inputs, a vector of output names, or (if -#' \code{command} is "MAKE") all the generated outputs: \code{FAO_Food_Macronutrient_All}, +#' \code{command} is "MAKE") all the generated outputs: \code{L107.FAO_Food_Macronutrient_All}, #' \code{FAO_Food_MacronutrientRate_2010_2019_MaxValue} #' @details This chunk aggregates supply utilization accounts into food balance sheet items #' @importFrom assertthat assert_that @@ -16,22 +16,19 @@ #' @importFrom tibble tibble is_tibble #' @importFrom tidyr complete drop_na gather nesting spread replace_na #' @author XZ 2024 -module_yextension_L100_FoodBalanceSheet <- function(command, ...) { +module_xfaostat_L107_FoodBalanceSheet <- function(command, ...) { MODULE_INPUTS <- - c("FBS_wide", - "TM_bilateral_wide", - "Bal_new_all", - "QCL_CROP_PRIMARY", - "QCL_PROD", - "SUA_food_macronutrient_rate", + c("TM_bilateral_wide", + "L105.Bal_new_all", + "L106.SUA_food_macronutrient_rate", FILE = file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_SUA_PrimaryEquivalent"), FILE = file.path(DIR_RAW_DATA_FAOSTAT, "SUA_item_code_map"), FILE = file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_FAO_iso_reg") ) MODULE_OUTPUTS <- - c("GCAM_APE_after2010", - "FAO_Food_Macronutrient_All") + c("L107.APE_after2010", + "L107.FAO_Food_Macronutrient_All") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -47,26 +44,6 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { get_data_list(all_data, MODULE_INPUTS, strip_attributes = TRUE) - for (.DF in c("QCL_CROP_PRIMARY", "QCL_PROD")) { - get(.DF) %>% - # merge Sudan and South Sudan - FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION_ALL(SUDAN2012_MERGE = T) %>% - assign(x = .DF, envir = Curr_Envir) - } - - ## *NonFodderProdArea ---- - QCL_CROP_PRIMARY %>% - filter(element == "Area harvested") %>% - mutate(item_set = "QCL_COMM_CROP_PRIMARY") %>% - bind_rows(QCL_PROD %>% - filter(!is.na(year))) %>% - select(-element_code) %>% - mutate(value = value / 1000, - unit = if_else(unit == "tonnes", "1000 tonnes", "1000 ha")) %>% - spread(year, value) -> - GCAMFAOSTAT_NonFodderProdArea - - TM_bilateral_wide %>% gather_years() %>% @@ -91,36 +68,37 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { ) %>% filter(value != 0.0) %>% transmute(area_code, item_code, source_code, year, value) -> - GCAMFAOSTAT_BiTrade + FAO_BiTrade_Kt - Bal_new_all %>% filter(value != 0.0) %>% + # GCAMFAOSTAT_SUA + L105.Bal_new_all %>% + filter(value != 0.0) %>% transmute(area_code, item_code, element, year, value) -> - GCAMFAOSTAT_SUA + FAO_SUA_Kt - SUA_food_macronutrient_rate -> GCAMFAOSTAT_MacroNutrientRate # Fix Sudan with code 206 and 276 after 2012 - if (206 %in% GCAMFAOSTAT_MacroNutrientRate$area_code == F) { - GCAMFAOSTAT_MacroNutrientRate %>% + if (206 %in% L106.SUA_food_macronutrient_rate$area_code == F) { + L106.SUA_food_macronutrient_rate %>% filter(area_code == 276) %>% mutate(area_code = 206) %>% - bind_rows(GCAMFAOSTAT_MacroNutrientRate) -> - GCAMFAOSTAT_MacroNutrientRate + bind_rows(L106.SUA_food_macronutrient_rate) -> + L106.SUA_food_macronutrient_rate } # Get Supply-utilization account (SUA) elements and use as factor - All_Bal_element <- levels(GCAMFAOSTAT_SUA$element) + All_Bal_element <- levels(FAO_SUA_Kt$element) All_Bal_element <- factor(All_Bal_element, levels = All_Bal_element) # Bilateral trade item indicator is added to SUA_item_code_map - # filter GCAMFAOSTAT_BiTrade to only include bilateral trade item to be - # consistent with GCAMFAOSTAT_SUA + # filter FAO_BiTrade_Kt to only include bilateral trade item to be + # consistent with FAO_SUA_Kt BilaterialTrade_ItemCode <- SUA_item_code_map %>% filter(TM == TRUE) %>% distinct(item_code) %>% pull - GCAMFAOSTAT_BiTrade %>% + FAO_BiTrade_Kt %>% filter(item_code %in% BilaterialTrade_ItemCode) -> - GCAMFAOSTAT_BiTrade + FAO_BiTrade_Kt SUA_item_code_map %>% select(item, item_code) -> SUA_item_code_map # Section1: [2010-2019] Region aggregation of supply-utilization-accounting data ---- @@ -133,26 +111,28 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # trap doors where we need to be extra careful to complete / refill zeros or risk loosing # rows of legitimate data. - # create a complete area / iso / GCAM region mapping - GCAMFAOSTAT_NonFodderProdArea %>% - distinct(area_code, area) %>% + # create a complete area / iso region mapping + FAO_SUA_Kt %>% + distinct(area_code) %>% left_join_error_no_match(Mapping_FAO_iso_reg %>% distinct(area_code, iso, region_ID), by = c("area_code")) -> Area_Region_Map # 1.1 Regional aggregation for SUA ---- + # [this section is not doing anything if no regional aggregation is needed] - # Aggregate SUA to GCAM regions + # Aggregate SUA regions if needed # Intra regional trade is removed when bilateral trade data is available - GCAMFAOSTAT_SUA %>% - left_join_error_no_match(Area_Region_Map %>% select(area_code, region_ID), by="area_code") %>% + FAO_SUA_Kt %>% + left_join_error_no_match( + Area_Region_Map %>% select(area_code, region_ID), by="area_code") %>% group_by(region_ID, item_code, element, year) %>% summarize(value = sum(value), .groups = "drop") %>% ungroup() -> DF_SUA_Agg # Calculate intra regional trade - GCAMFAOSTAT_BiTrade %>% + FAO_BiTrade_Kt %>% left_join_error_no_match(Area_Region_Map %>% select(area_code, region_ID), by="area_code") %>% left_join_error_no_match(Area_Region_Map %>% select(source_code = area_code, source_region_ID = region_ID), by="source_code") %>% filter(region_ID == source_region_ID) %>% @@ -193,18 +173,20 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { group_by(region_ID, item_code, element, year) %>% summarize(value = sum(value), .groups = "drop") %>% ungroup() -> - FAO_SUA_Kt_2010to2019_R + FAO_SUA_Kt_R + # complete years and fill in zeros + FAO_SUA_Kt_R %>% + complete(nesting(region_ID, item_code, element), year) %>% + replace_na(list(value = 0)) -> + FAO_SUA_Kt_R - Min_SUA_Year <- min(FAO_SUA_Kt_2010to2019_R$year) - FAO_SUA_Kt_2010to2019 <- GCAMFAOSTAT_SUA ## Clean up - rm(GCAMFAOSTAT_SUA) - rm(GCAMFAOSTAT_BiTrade) + rm(FAO_BiTrade_Kt) ## Done Section1 ---- #****************************---- - # Section2: [2010-2019] Primary equivalent aggregation to GCAM commodities ---- + # Section2: [2010-2019] Primary equivalent aggregation to PCe commodities ---- Mapping_SUA_PrimaryEquivalent %>% left_join_error_no_match(SUA_item_code_map %>% rename(sink_item_code = item_code), by=c("sink_item" = "item")) %>% @@ -231,6 +213,7 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # @return A data frame including regional, traded, and world extraction rates of a processing Get_GROSS_EXTRACTION_RATE <- function(DF_CURR_NEST, DF_ALL) { + curr_sink_items = unique(DF_CURR_NEST$item_code) Mapping_SUA_PrimaryEquivalent_ID %>% filter(sink_item_code %in% curr_sink_items) -> @@ -267,12 +250,13 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # Regional extraction rate = prod of an aggregated processed item / Processed use of an aggregated primary item # Use world average to fill in NA or zero extraction_rate = if_else(is.na(extraction_rate) | extraction_rate == 0, extraction_rate_world, extraction_rate), - # Using minimum extraction rate here + # Using minimum extraction rate here as an upper threshold to avoid extremely large scaling later extraction_rate = pmax(extraction_rate, minimium_extraction_rate), + # Set rate to Inf when Processed == 0 + PositiveProd_ZeroProc = Production > 0 & Processed == 0, + extraction_rate = if_else(PositiveProd_ZeroProc == TRUE, Inf, extraction_rate), extraction_rate_trade = sum(Export) / sum(Export / extraction_rate), - extraction_rate_trade = if_else(is.na(extraction_rate_trade), extraction_rate, extraction_rate_trade), - # both processed and production > 0 - positive_prod = Production > 0 & Processed > 0) %>% + extraction_rate_trade = if_else(is.na(extraction_rate_trade), extraction_rate, extraction_rate_trade)) %>% ungroup() %>% group_by(APE_comm, region_ID) %>% # Calculate lagged extraction_rate but replace NA with current rate (first period) @@ -426,11 +410,13 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # Primary equivalent aggregation # @param DF_ALL Input supply-utilization accounting data frame with all levels of data nested which need to be primarized - # @return A supply-utilization accounting data frame with all levels processed and aggregated to GCAM_commodity + # @return A supply-utilization accounting data frame with all levels processed and aggregated to APE_comm_Agg Proc_primarize <- function(DF_ALL){ + MaxNest = max(DF_ALL$nest_level) MinNest = 1 + for(curr_nest in MaxNest:MinNest) { # get the current tier to process DF_ALL %>% @@ -495,9 +481,9 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { complete(region_ID = unique(Area_Region_Map$region_ID), nesting(APE_comm, item_code, nest_level, year), fill=list(value=0)) %>% complete(.df2 %>% distinct(APE_comm, item_code, nest_level), nesting(region_ID, year), fill=list(value=0)) %>% # Use global value (year is specified now!) - group_by(APE_comm, item_code, year) %>% - mutate(value = sum(value)) %>% - ungroup() %>% + # group_by(APE_comm, item_code, year) %>% + # mutate(value = sum(value)) %>% + # ungroup() %>% group_by(APE_comm, region_ID, year) %>% mutate(share = value/ sum(value), share = if_else(is.finite(share), share, dplyr::n()/sum(dplyr::n()))) %>% @@ -572,28 +558,28 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { gather(element, value, -region_ID, -APE_comm, -year) -> APE_AGG - # Aggregate by GCAM_commodity + # Aggregate by APE_comm_Agg # At this point we ditch the ID codes and factors as we return the data and # make it available for the rest of the original processing APE_AGG %>% - left_join_error_no_match(Mapping_SUA_PrimaryEquivalent %>% select(GCAM_commodity, APE_comm) %>% distinct(), + left_join_error_no_match(Mapping_SUA_PrimaryEquivalent %>% select(APE_comm_Agg, APE_comm) %>% distinct(), by = c("APE_comm")) %>% - group_by(region_ID, GCAM_commodity, element, year) %>% + group_by(region_ID, APE_comm_Agg, element, year) %>% summarize(value = sum(value), .groups = "drop") %>% ungroup() %>% mutate(element = as.character(element)) %>% - select(region_ID, year, GCAM_commodity, element, value) -> - GCAM_APE_after2010 + select(region_ID, year, APE_comm_Agg, element, value) -> + .df_APE - return(GCAM_APE_after2010) + return(.df_APE) } # 2.2. Execution: process data into APE ---- - ## Loop through all GCAM_commodity with available data ---- + ## Loop through all APE_comm_Agg with available data ---- - FAO_SUA_Kt_2010to2019_R %>% + FAO_SUA_Kt_R %>% left_join(Mapping_SUA_PrimaryEquivalent_ID %>% select(APE_comm, item_code = sink_item_code, nest_level) %>% distinct(), by = c("item_code")) %>% left_join(Mapping_SUA_PrimaryEquivalent_ID %>% @@ -610,12 +596,11 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # although we end up needed to unnest at times as well so ultimately, # it likely makes little difference in performance tidyr::nest(data = -nest_level) %>% - # we are now ready to recursively primarize APE commodities then aggregate - # to GCAM commodities + # we are now ready to recursively primarize APE commodities then aggregate to APE Proc_primarize() -> - GCAM_APE_after2010 + L107.APE_after2010 - rm(FAO_SUA_Kt_2010to2019_R) + rm(FAO_SUA_Kt_R) ## Done Section2 ---- #****************************---- @@ -632,15 +617,15 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # about 486 items (out of 530) used in GCAM Mapping_SUA_PrimaryEquivalent %>% - select(GCAM_commodity, item = source_item) %>% + select(APE_comm_Agg, item = source_item) %>% bind_rows(Mapping_SUA_PrimaryEquivalent %>% - select(GCAM_commodity, item = sink_item)) %>% + select(APE_comm_Agg, item = sink_item)) %>% distinct() %>% left_join_error_no_match(SUA_item_code_map, by = "item") -> - SUA_Items_GCAM + SUA_Items_APE assertthat::assert_that( - SUA_Items_GCAM %>% distinct(item_code) %>% nrow() == SUA_Items_GCAM %>% nrow(), + SUA_Items_APE %>% distinct(item_code) %>% nrow() == SUA_Items_APE %>% nrow(), msg = "Check duplicates in Mapping_SUA_PrimaryEquivalent SUA items" ) @@ -648,16 +633,16 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # (e.g., wine, infant food, or other nonfood items etc.) SUA_item_code_map %>% - filter(!item_code %in% unique(SUA_Items_GCAM$item_code)) -> SUA_Items_NonGCAM + filter(!item_code %in% unique(SUA_Items_APE$item_code)) -> SUA_Items_NEC - # b. There are 426 FAO food items, all included in FAO_SUA_Kt_2010to2019 (530 items) + # b. There are 426 FAO food items, all included in FAO_SUA_Kt_2010to2022 (530 items) # SUA_Items_Food includes both GCAM and NonGCAM(NEC) SUA_item_code_map %>% - filter(item_code %in% unique(GCAMFAOSTAT_MacroNutrientRate$item_code)) %>% - left_join(SUA_Items_GCAM %>% select(-item), by = "item_code") %>% - # For NA GCAM_commodity: not elsewhere classified (NEC) + filter(item_code %in% unique(L106.SUA_food_macronutrient_rate$item_code)) %>% + left_join(SUA_Items_APE %>% select(-item), by = "item_code") %>% + # For NA APE_comm_Agg: not elsewhere classified (NEC) # So we would know % of food calories not included in GCAM commodities - mutate(GCAM_commodity = if_else(is.na(GCAM_commodity), "NEC", GCAM_commodity)) -> + mutate(APE_comm_Agg = if_else(is.na(APE_comm_Agg), "NEC", APE_comm_Agg)) -> SUA_Items_Food @@ -666,7 +651,7 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { ### a. Get world average macronutrient ---- # For filling in missing values - GCAMFAOSTAT_MacroNutrientRate %>% + L106.SUA_food_macronutrient_rate %>% tidyr::gather(macronutrient, macronutrient_value, calperg:proteinperc) %>% group_by(item, item_code, macronutrient) %>% summarise(macronutrient_value_World = mean(macronutrient_value), .groups = "drop") %>% @@ -676,7 +661,7 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { ### b. Calculate SUA food Calories consumption by joining macronutrient rates and SUA food ---- - FAO_SUA_Kt_2010to2019 %>% + FAO_SUA_Kt %>% filter(element == "Food", item_code %in% SUA_Items_Food$item_code) %>% # ensure we at least have a complete series across time otherwise it may # throw off moving avg calculations @@ -687,7 +672,7 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { repeat_add_columns( tibble(macronutrient = c("calperg", "fatperc", "proteinperc"))) %>% left_join( - GCAMFAOSTAT_MacroNutrientRate %>% + L106.SUA_food_macronutrient_rate %>% select(-item) %>% tidyr::gather(macronutrient, macronutrient_value, calperg:proteinperc), by = c("area_code", "item_code", "macronutrient") @@ -709,20 +694,8 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { macronutrient == "fatperc" ~ "MtFat", macronutrient == "proteinperc" ~ "MtProtein" )) %>% left_join_error_no_match(Area_Region_Map, by = "area_code") -> - FAO_Food_Macronutrient_All - - - # Compare recompiled FBS with the original FBS ---- - - FBS_wide %>% gather_years() %>% - filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% - FAOSTAT_AREA_RM_NONEXIST() -> FBS + L107.FAO_Food_Macronutrient_All - for (.DF in c("FBS")) { - get(.DF) %>% filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% - # merge Sudan and South Sudan - FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION_ALL(SUDAN2012_MERGE = T) %>% - assign(x = .DF, envir = parent.env(environment())) } #****************************---- # Produce outputs ---- @@ -732,28 +705,18 @@ module_yextension_L100_FoodBalanceSheet <- function(command, ...) { # add_title("GCAM_AgLU_SUA_APE_1973_2019") %>% # add_units("kton") %>% # add_comments("Supply utilization balance for GCAM commodities and regions in primary equivalent") %>% - # add_precursors("aglu/AGLU_ctry", - # "common/iso_GCAM_regID", - # "aglu/FAO/GCAMFAOSTAT_SUA", - # "aglu/FAO/GCAMFAOSTAT_BiTrade", - # "aglu/FAO/Mapping_SUA_PrimaryEquivalent", - # "aglu/FAO/SUA_item_code_map", - # "aglu/FAO/GCAMFAOSTAT_NonFodderProdArea") -> + # add_precursors(MODULE_INPUTS) -> # GCAM_AgLU_SUA_APE_1973_2019 # - # FAO_Food_Macronutrient_All %>% + # L107.FAO_Food_Macronutrient_All %>% # add_title("GCAM_AgLU_SUA_APE_1973_2019") %>% # add_units("MKcal, MtFat, MtProtein") %>% # add_comments("Macronutrient consumption values connected to food consumption in GCAM_AgLU_SUA_APE_1973_2019") %>% - # add_precursors("aglu/AGLU_ctry", - # "common/iso_GCAM_regID", - # "aglu/FAO/GCAMFAOSTAT_SUA", - # "aglu/FAO/GCAMFAOSTAT_MacroNutrientRate", - # "aglu/FAO/Mapping_SUA_PrimaryEquivalent") -> - # FAO_Food_Macronutrient_All + # add_precursors(MODULE_INPUTS) -> + # L107.FAO_Food_Macronutrient_All # return_data(MODULE_OUTPUTS) diff --git a/R/xfaostat_L201_Forestry.R b/R/xfaostat_L201_Forestry.R index 9815d4ba..a3d0a07c 100644 --- a/R/xfaostat_L201_Forestry.R +++ b/R/xfaostat_L201_Forestry.R @@ -21,8 +21,8 @@ module_xfaostat_L201_Forestry <- function(command, ...) { c("FO_RoundwoodProducts") MODULE_OUTPUTS <- - c("For_Balance", - "FO_RoundwoodProducts_Export_Q_V") + c("L201.For_Balance", + "L201.FO_RoundwoodProducts_Export_Q_V") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -186,21 +186,19 @@ module_xfaostat_L201_Forestry <- function(command, ...) { # clean up rm(FO_area_item, UnitMap, L201.FO_RoundwoodProducts, L201.FO_RoundwoodProducts_Prod, L201.FO_RoundwoodProducts_Import_Q_V) - L201.For_Balance %>% add_title("FAO forestry consumption, production, export, and import (roundwood total) by country_year") %>% add_units("m3 ") %>% add_comments("Data is preprocessed and generated by gcamdata-FAOSTAT. Gross trade is balanced") %>% add_precursors("FO_RoundwoodProducts") -> - For_Balance + L201.For_Balance L201.FO_RoundwoodProducts_Export_Q_V %>% add_title("FAO forests export qunatity and export value by country_year") %>% add_units("m3 and 1000 USD") %>% add_comments("Data is generated by gcamdata-FAOSTAT for deriving export prices") %>% add_precursors("FO_RoundwoodProducts") -> - FO_RoundwoodProducts_Export_Q_V - + L201.FO_RoundwoodProducts_Export_Q_V return_data(MODULE_OUTPUTS) diff --git a/R/xfaostat_L301_LandCover.R b/R/xfaostat_L301_LandCover.R index af8a6d10..6038f21e 100644 --- a/R/xfaostat_L301_LandCover.R +++ b/R/xfaostat_L301_LandCover.R @@ -21,7 +21,7 @@ module_xfaostat_L301_LandCover <- function(command, ...) { c("RL") MODULE_OUTPUTS <- - c("RL_LandCover") + c("L301.RL_LandCover") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -125,14 +125,14 @@ module_xfaostat_L301_LandCover <- function(command, ...) { RL1_ArableLand %>% bind_rows(RL2_TemporaryCrop) %>% bind_rows(RL3_FallowLand) -> - RL_LandCover + L301.RL_LandCover - RL_LandCover %>% + L301.RL_LandCover %>% add_title("FAO cropland, Temporary Crop (harvested cropland), and fallow area by country_year") %>% add_units("1000 Ha") %>% add_comments("Data is cleaned and generated by gcamdata-FAOSTAT") %>% add_precursors("RL") -> - RL_LandCover + L301.RL_LandCover return_data(MODULE_OUTPUTS) diff --git a/R/xfaostat_L401_Fertilizer.R b/R/xfaostat_L401_Fertilizer.R index 43b8517f..7fdc735b 100644 --- a/R/xfaostat_L401_Fertilizer.R +++ b/R/xfaostat_L401_Fertilizer.R @@ -21,7 +21,7 @@ module_xfaostat_L401_Fertilizer <- function(command, ...) { c("RFN") MODULE_OUTPUTS <- - c("RFN_ProdDemand") + c("L401.RFN_ProdDemand") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -110,14 +110,14 @@ module_xfaostat_L401_Fertilizer <- function(command, ...) { RFN1_Production %>% bind_rows(RFN1_Demand) -> - RFN_ProdDemand + L401.RFN_ProdDemand - RFN_ProdDemand %>% + L401.RFN_ProdDemand %>% add_title("FAO fertilizer production by country_year") %>% add_units("tonnes N") %>% add_comments("Data is preprocessed and generated by gamdata-FAOSTAT") %>% add_precursors("RFN") -> - RFN_ProdDemand + L401.RFN_ProdDemand return_data(MODULE_OUTPUTS) diff --git a/R/yfaostat_SUA_FBS_CSVExport.R b/R/xfaostat_L999_DataExport.R similarity index 89% rename from R/yfaostat_SUA_FBS_CSVExport.R rename to R/xfaostat_L999_DataExport.R index ac706488..ba280c68 100644 --- a/R/yfaostat_SUA_FBS_CSVExport.R +++ b/R/xfaostat_L999_DataExport.R @@ -1,6 +1,6 @@ # Copyright 2019 Battelle Memorial Institute; see the LICENSE file. -#' module_yfaostat_SUA_FBS_CSVExport +#' module_xfaostat_L999_DataExport #' #' Generate supply utilization balance in primary equivalent #' @@ -15,16 +15,16 @@ #' @importFrom tibble tibble #' @importFrom tidyr complete drop_na gather nesting spread replace_na #' @author XZ Sep2024 -module_yfaostat_SUA_FBS_CSVExport <- function(command, ...) { +module_xfaostat_L999_DataExport <- function(command, ...) { MODULE_INPUTS <- c(FILE = file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_SUA_PrimaryEquivalent"), FILE = file.path(DIR_RAW_DATA_FAOSTAT, "SUA_item_code_map"), FILE = file.path(DIR_RAW_DATA_FAOSTAT, "Mapping_FAO_iso_reg"), - "GCAM_APE_after2010", - "Bal_new_all", + "L107.APE_after2010", + "L105.Bal_new_all", "FBS_wide", - "FAO_Food_Macronutrient_All") + "L107.FAO_Food_Macronutrient_All") MODULE_OUTPUTS <- c(CSV = "GCAMFAOSTAT_DataArchive_SUA") @@ -53,12 +53,12 @@ module_yfaostat_SUA_FBS_CSVExport <- function(command, ...) { # SUA and FBS ---- ## *SUA ---- - Bal_new_all %>% filter(value != 0.0) %>% + L105.Bal_new_all %>% filter(value != 0.0) %>% transmute(area_code, item_code, element, year, value) %>% add_title("GCAMFAOSTAT_DataArchive_SUA") %>% add_units("1000 tonnes") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("Bal_new_all") -> + add_precursors("L105.Bal_new_all") -> GCAMFAOSTAT_DataArchive_SUA @@ -73,7 +73,7 @@ module_yfaostat_SUA_FBS_CSVExport <- function(command, ...) { out_dir = DIR_OUTPUT_CSV, GZIP = F) - rm(Bal_new_all) + rm(L105.Bal_new_all) } diff --git a/R/yfaostat_GCAM_CSVExport.R b/R/yfaostat_GCAM_CSVExport.R index a4198ef7..14d888c3 100644 --- a/R/yfaostat_GCAM_CSVExport.R +++ b/R/yfaostat_GCAM_CSVExport.R @@ -21,20 +21,20 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { c("CS", "PD", "MK", - "Bal_new_all", "FBSH_CBH_wide", - "QCL_PROD", - "QCL_AN_LIVEANIMAL", - "QCL_AN_PRIMARY_MILK", - "QCL_CROP_PRIMARY", - "QCL_FODDERCROP", - "QCL_PRIMARY_PROD_PV", + "L102.QCL_PROD", + "L102.QCL_AN_LIVEANIMAL", + "L102.QCL_AN_PRIMARY_MILK", + "L102.QCL_CROP_PRIMARY", + "L104.QCL_FODDERCROP", + "L103.QCL_PRIMARY_PROD_PV", + "L105.Bal_new_all", "TM_bilateral_wide", - "SUA_food_macronutrient_rate", - "For_Balance", - "FO_RoundwoodProducts_Export_Q_V", - "RL_LandCover", - "RFN_ProdDemand") + "L106.SUA_food_macronutrient_rate", + "L201.For_Balance", + "L201.FO_RoundwoodProducts_Export_Q_V", + "L301.RL_LandCover", + "L401.RFN_ProdDemand") MODULE_OUTPUTS <- c("yfaostat_GCAM_GCAMFAOSTAT_CSV", @@ -61,9 +61,9 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { } else if(command == driver.MAKE) { year <- value <- Year <- Value <- FAO_country <- iso <- NULL # silence package check. - area_code <- source_code <- area_code1 <- item_code <- Bal_new_all <- element <- FBSH_CBH_wide <- - element_code <- QCL_CROP_PRIMARY <- QCL_PROD <- unit <- QCL_FODDERCROP <- QCL_AN_LIVEANIMAL <- - QCL_AN_PRIMARY_MILK <- QCL_PRIMARY_PROD_PV <- area <- PD <- SUA_food_macronutrient_rate <- + area_code <- source_code <- area_code1 <- item_code <- L105.Bal_new_all <- element <- FBSH_CBH_wide <- + element_code <- L102.QCL_CROP_PRIMARY <- L102.QCL_PROD <- unit <- L104.QCL_FODDERCROP <- L102.QCL_AN_LIVEANIMAL <- + L102.QCL_AN_PRIMARY_MILK <- L103.QCL_PRIMARY_PROD_PV <- area <- PD <- L106.SUA_food_macronutrient_rate <- TM_bilateral_wide <- NULL all_data <- list(...)[[1]] @@ -160,12 +160,12 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # SUA and FBS ---- ## *SUA ---- - Bal_new_all %>% filter(value != 0.0) %>% + L105.Bal_new_all %>% filter(value != 0.0) %>% transmute(area_code, item_code, element, year, value) %>% add_title("GCAMFAOSTAT_SUA") %>% add_units("1000 tonnes") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("Bal_new_all", + add_precursors("L105.Bal_new_all", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_SUA @@ -182,7 +182,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { out_dir = DIR_OUTPUT_CSV, GZIP = T) - rm(Bal_new_all) + rm(L105.Bal_new_all) ## *FBSH and CB ---- FBSH_CBH_wide %>% @@ -224,7 +224,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # Production and Area harvested ---- - for (.DF in c("QCL_CROP_PRIMARY", "QCL_FODDERCROP", "QCL_PROD")) { + for (.DF in c("L102.QCL_CROP_PRIMARY", "L104.QCL_FODDERCROP", "L102.QCL_PROD")) { get(.DF) %>% # merge Sudan and South Sudan FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION_ALL(SUDAN2012_MERGE = T) %>% @@ -232,10 +232,10 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { } ## *NonFodderProdArea ---- - QCL_CROP_PRIMARY %>% + L102.QCL_CROP_PRIMARY %>% filter(element == "Area harvested") %>% mutate(item_set = "QCL_COMM_CROP_PRIMARY") %>% - bind_rows(QCL_PROD %>% + bind_rows(L102.QCL_PROD %>% filter(!is.na(year))) %>% select(-element_code) %>% mutate(value = value / 1000, @@ -244,8 +244,8 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { add_title("GCAMFAOSTAT_NonFodderProdArea") %>% add_units("1000 tonnes or 1000 ha") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("QCL_CROP_PRIMARY", - "QCL_PROD", + add_precursors("L102.QCL_CROP_PRIMARY", + "L102.QCL_PROD", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_NonFodderProdArea @@ -263,7 +263,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { ) ## *FodderProdArea ---- - QCL_FODDERCROP %>% + L104.QCL_FODDERCROP %>% filter(!is.na(year)) %>% select(-element_code) %>% mutate(value = value / 1000, @@ -272,7 +272,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { add_title("GCAMFAOSTAT_FodderProdArea") %>% add_units("1000 tonnes or 1000 ha") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("QCL_FODDERCROP", + add_precursors("L104.QCL_FODDERCROP", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_FodderProdArea @@ -289,17 +289,17 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { GZIP = T ) - rm(QCL_CROP_PRIMARY, QCL_FODDERCROP, QCL_PROD) + rm(L102.QCL_CROP_PRIMARY, L104.QCL_FODDERCROP, L102.QCL_PROD) ##*AnimalStocks ---- # Laying is not needed for now; live animal stocks and milk animals are used for water demand calculation in GCAM # 1171 "Animals live nes" and 1083 "Pigeons, other birds" are not available - QCL_AN_LIVEANIMAL %>% filter(element == "Stocks") %>% + L102.QCL_AN_LIVEANIMAL %>% filter(element == "Stocks") %>% spread(year, value) -> FAO_an_Stocks - QCL_AN_PRIMARY_MILK %>% filter(element == "Milk Animals") %>% + L102.QCL_AN_PRIMARY_MILK %>% filter(element == "Milk Animals") %>% spread(year, value) -> FAO_an_Dairy_Stocks FAO_an_Stocks %>% @@ -329,13 +329,13 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # Producer prices in quantity and value ---- ##*Producer Price ---- - QCL_PRIMARY_PROD_PV %>% + L103.QCL_PRIMARY_PROD_PV %>% filter(year >= MIN_HIST_PP_YEAR) %>% spread(year, value) %>% add_title("GCAMFAOSTAT_ProdPrice") %>% add_units("Quantity in tonne; Value in US$") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("QCL_PRIMARY_PROD_PV", + add_precursors("L103.QCL_PRIMARY_PROD_PV", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_ProdPrice @@ -352,7 +352,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { out_dir = DIR_OUTPUT_CSV, GZIP = T ) - rm(QCL_PRIMARY_PROD_PV) + rm(L103.QCL_PRIMARY_PROD_PV) ##*GDP deflators ---- @@ -384,7 +384,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { ## *MacroNutrientRate---- - SUA_food_macronutrient_rate -> + L106.SUA_food_macronutrient_rate -> GCAMFAOSTAT_MacroNutrientRate @@ -400,7 +400,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { add_title("GCAMFAOSTAT_MacroNutrientRate") %>% add_units("calories per g or (fat or protein) percentage") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("SUA_food_macronutrient_rate", + add_precursors("L106.SUA_food_macronutrient_rate", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_MacroNutrientRate @@ -421,13 +421,13 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # Forestry data output ---- ##*ForProdTrade ---- - For_Balance %>% + L201.For_Balance %>% filter(element != "Demand") %>% spread(year, value) %>% add_title("GCAMFAOSTAT_ForProdTrade for forest products") %>% add_units("m3") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("For_Balance", + add_precursors("L201.For_Balance", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_ForProdTrade @@ -445,12 +445,12 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { ) ##*ForExpPrice ---- - FO_RoundwoodProducts_Export_Q_V %>% + L201.FO_RoundwoodProducts_Export_Q_V %>% spread(year, value) %>% add_title("GCAMFAOSTAT_ForExpPrice") %>% add_units("Export Quantity in m3; Export Value in 1000 US$") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("FO_RoundwoodProducts_Export_Q_V", + add_precursors("L201.FO_RoundwoodProducts_Export_Q_V", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_ForExpPrice @@ -471,12 +471,12 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # Land Cover data output ---- ##*LandCover ---- - RL_LandCover %>% + L301.RL_LandCover %>% spread(year, value) %>% add_title("GCAMFAOSTAT_LandCover") %>% add_units("1000 Ha") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("RL_LandCover", + add_precursors("L301.RL_LandCover", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_LandCover @@ -497,12 +497,12 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) { # Fertilizer data output---- ##* N Fertilizer ---- - RFN_ProdDemand %>% + L401.RFN_ProdDemand %>% spread(year, value) %>% add_title("GCAMFAOSTAT_NFertilizer") %>% add_units("tonnes N") %>% add_comments("gcamfaostat Export CSV") %>% - add_precursors("RFN_ProdDemand", + add_precursors("L401.RFN_ProdDemand", "yfaostat_GCAM_GCAMFAOSTAT_CSV") -> GCAMFAOSTAT_NFertilizer diff --git a/inst/extdata/FAOSTAT/Mapping_SUA_PrimaryEquivalent.csv b/inst/extdata/FAOSTAT/Mapping_SUA_PrimaryEquivalent.csv index 828f5948..106d464a 100644 --- a/inst/extdata/FAOSTAT/Mapping_SUA_PrimaryEquivalent.csv +++ b/inst/extdata/FAOSTAT/Mapping_SUA_PrimaryEquivalent.csv @@ -20,7 +20,7 @@ # Last update: 06-09-2024 (XZ) # Column types: ccilccncnnl # ---------- -GCAM_commodity,APE_comm,nest_level,source_primary,source_item,sink_item,output_specific_extraction_rate,sink_FBS_item,extraction_rate_Q25,extraction_rate_world2019,Q25asMin +APE_comm_Agg,APE_comm,nest_level,source_primary,source_item,sink_item,output_specific_extraction_rate,sink_FBS_item,extraction_rate_Q25,extraction_rate_world2019,Q25asMin Wheat,Wheat,1,TRUE,Wheat,"Food preparations, flour, malt extract",,Wheat and products,0.942971612,0.985277283,FALSE Wheat,Wheat,1,TRUE,Wheat,Bulgur,,Wheat and products,0.942971612,0.985277283,FALSE Wheat,Wheat,1,TRUE,Wheat,"Cereals, breakfast",,Wheat and products,0.942971612,0.985277283,FALSE diff --git a/man/module_yextension_L100_FoodBalanceSheet.Rd b/man/module_xfaostat_L107_FoodBalanceSheet.Rd similarity index 59% rename from man/module_yextension_L100_FoodBalanceSheet.Rd rename to man/module_xfaostat_L107_FoodBalanceSheet.Rd index 349a41bf..bfbf903e 100644 --- a/man/module_yextension_L100_FoodBalanceSheet.Rd +++ b/man/module_xfaostat_L107_FoodBalanceSheet.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/yextension_L100_FoodBalanceSheet.R -\name{module_yextension_L100_FoodBalanceSheet} -\alias{module_yextension_L100_FoodBalanceSheet} -\title{module_yextension_L100_FoodBalanceSheet} +% Please edit documentation in R/xfaostat_L107_FoodBalanceSheet.R +\name{module_xfaostat_L107_FoodBalanceSheet} +\alias{module_xfaostat_L107_FoodBalanceSheet} +\title{module_xfaostat_L107_FoodBalanceSheet} \usage{ -module_yextension_L100_FoodBalanceSheet(command, ...) +module_xfaostat_L107_FoodBalanceSheet(command, ...) } \arguments{ \item{command}{API command to execute} @@ -13,11 +13,11 @@ module_yextension_L100_FoodBalanceSheet(command, ...) } \value{ Depends on \code{command}: either a vector of required inputs, a vector of output names, or (if - \code{command} is "MAKE") all the generated outputs: \code{FAO_Food_Macronutrient_All}, + \code{command} is "MAKE") all the generated outputs: \code{L107.FAO_Food_Macronutrient_All}, \code{FAO_Food_MacronutrientRate_2010_2019_MaxValue} } \description{ -Generate supply utilization balance in primary equivalent +Generate FBS supply utilization balance in primary equivalent } \details{ This chunk aggregates supply utilization accounts into food balance sheet items diff --git a/man/module_yfaostat_SUA_FBS_CSVExport.Rd b/man/module_xfaostat_L999_DataExport.Rd similarity index 69% rename from man/module_yfaostat_SUA_FBS_CSVExport.Rd rename to man/module_xfaostat_L999_DataExport.Rd index 45d39597..4085da7a 100644 --- a/man/module_yfaostat_SUA_FBS_CSVExport.Rd +++ b/man/module_xfaostat_L999_DataExport.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/yfaostat_SUA_FBS_CSVExport.R -\name{module_yfaostat_SUA_FBS_CSVExport} -\alias{module_yfaostat_SUA_FBS_CSVExport} -\title{module_yfaostat_SUA_FBS_CSVExport} +% Please edit documentation in R/xfaostat_L999_DataExport.R +\name{module_xfaostat_L999_DataExport} +\alias{module_xfaostat_L999_DataExport} +\title{module_xfaostat_L999_DataExport} \usage{ -module_yfaostat_SUA_FBS_CSVExport(command, ...) +module_xfaostat_L999_DataExport(command, ...) } \arguments{ \item{command}{API command to execute}