Public full release (#19)

* Fixes to user-defined energy data
* Fix_metadata_collate
* Rename processing scripts
* Modify input and output directories for user-defined energy scripts
* Add option to clean user-defined-energy to Makefile
* Modify Makefile to clean additional diagnostic directories
* Move log directory to top-level
* Remove duplicate and/or unused files
* Correct shipping fuel time series.
* Correct UN energy data negative values
* Update waste emissions code and assumptions
* Update korea bcoc
* Update README for release

Author: ssmithClimate

.gitignore

@@ -30,4 +30,4 @@ input/energy/OECD_Conversion_Factors.csv
 .Rproj.user
 
 #Other
-.DS_Store
+.DS_Store

Makefile

@@ -19,13 +19,13 @@ SC_MAPPINGS = input/mappings/scaling
 ACTIV = input/activity
 INV_DATA = input/emissions-inventories
 MED_OUT = intermediate-output
-ENERGY_IN = input/energy
 DIAG_OUT = diagnostic-output
 FINAL_OUT = final-emissions
 EXT_IN = input/extension
-USER_EN_IN = input/extension/user-defined-energy
+USER_EN_PROCESS	= input/energy/user-defined-energy
+USER_EN_IN = input/energy/user-defined-energy/user_energy_input
 EXT_DATA = input/extension/extension-data
-LOGS = code/logs
+LOGS = logs
 DOCS = documentation
 
 # --------------------------------------------------------------
@@ -109,7 +109,7 @@ endif
 
 # Note that this is an inefficient method of creating data for multiple species
 # If a multi-processor machine is available, CEDS should be instead run in parellel for multiple species
-all: SO2-emissions BC-emissions OC-emissions NOx-emissions CO-emissions NMVOC-emissions CO2-emissions NH3-emissions
+all: SO2-emissions BC-emissions OC-emissions NOx-emissions CO-emissions NMVOC-emissions CO2-emissions NH3-emissions CH4-emissions
 part1: SO2-emissions NOx-emissions NH3-emissions
 part2: BC-emissions OC-emissions CO2-emissions
 part3: CO-emissions NMVOC-emissions CH4-emissions
@@ -119,7 +119,19 @@ part3: CO-emissions NMVOC-emissions CH4-emissions
 # Targets used to remove output files for a fresh run
 clean-all: \
 	clean-intermediate clean-diagnostic clean-final clean-logs clean-io clean-modA clean-modB clean-modC \
-	clean-modD clean-modE clean-modF clean-modH clean-gridding
+	clean-modD clean-modE clean-modF clean-modH clean-gridding clean-user_defined_energy
+
+clean-user_defined_energy:
+# Deletes all CSVs in the directory except for: 
+# 1) CEDS user-defined energy inputs, metadata, and instructions (U.*.csv)
+# 2) Relevant mapping and instructions files for user-defined energy input which require pre-processing for use in CEDS (A.*-instructions.csv, A.*_sector_map.csv, A.*-mapping.xlsx)
+	find $(USER_EN_IN) -name "*.csv" ! -name "U.*.csv" ! -name "A.*-instructions.csv" ! -name "A.*_sector_map.csv" -delete
+
+# Deletes all xlsx files that don't end in -mapping.xlsx and begin with "A." or "U."
+	find $(USER_EN_IN) -name "*.xlsx" ! -name "A.*-mapping.xlsx" ! -name "U.*-mapping.xlsx" -delete
+
+# Deletes all txt files that aren't the README file
+	find $(USER_EN_IN) -name "*.txt" ! -name "README.txt" -delete
 
 clean-intermediate:
 	rm -fv $(MED_OUT)/*.csv
@@ -129,7 +141,11 @@ clean-diagnostic:
 	rm -fv $(DIAG_OUT)/summary-plots/*.csv \
 	rm -fv $(DIAG_OUT)/summary-plots/*.pdf \
 	rm -fv $(DIAG_OUT)/ceds-comparisons/*.pdf \
-	rm -fv $(DIAG_OUT)/ceds-comparisons/*.csv
+	rm -fv $(DIAG_OUT)/ceds-comparisons/*.csv \
+	rm -fv $(DIAG_OUT)/ceds-comparisons/sector-level/*.csv \
+	rm -fv $(DIAG_OUT)/ceds-comparisons/sector-level/*.pdf \
+	rm -fv $(DIAG_OUT)/user-data/*.csv \
+	rm -fv $(DIAG_OUT)/user-data/*.png
 
 clean-final:
 	rm -fv $(FINAL_OUT)/*.csv
@@ -392,16 +408,10 @@ $(MED_OUT)/A.comb_othertrans_activity.csv: \
 # Extends IEA data with BP data
 $(MED_OUT)/A.IEA_BP_energy_ext.csv: \
 	$(MOD_A)/A3.1.IEA_BP_data_extension.R \
-	$(MOD_A)/A3.2.Adjust_Shipping_Fuel_Cons.R \
 	$(MED_OUT)/A.comb_othertrans_activity.csv \
 	$(MAPPINGS)/Master_Fuel_Sector_List.xlsx \
-	$(ENERGY_DATA)/BP_energy_data.xlsx \
-	$(ENERGY_DATA)/Shipping_Fuel_Consumption.xlsx
+	$(ENERGY_DATA)/BP_energy_data.xlsx
 	Rscript $< $(EM) --nosave --no-restore
-	Rscript $(word 2,$^) $(EM) --nosave --no-restore
-
-$(MED_OUT)/A.intl_shipping_en.csv: \
-	$(MED_OUT)/A.IEA_BP_energy_ext.csv
 
 # aa3-2
 # Write out difference between IEA and CEDS coal
@@ -419,7 +429,7 @@ $(MED_OUT)/A.IEA_CEDS_coal_difference.csv: \
 # aa3-3
 # Process pig iron production
 $(EXT_DATA)/A.Pig_Iron_Production.csv: \
-	$(MOD_A)/A3.4.proc_pig_iron.R \
+	$(MOD_A)/A3.3.proc_pig_iron.R \
 	$(MED_OUT)/A.UN_pop_master.csv \
 	#(ACTIV)/metals/Blast_furnace_iron_production_1850-2014.xlsx \
 	#(ACTIV)/metals/Pig_Iron_Production_US.csv \
@@ -535,7 +545,7 @@ $(MED_OUT)/A.other_biomass_extended.csv: \
 	$(MED_OUT)/A.default_comb_activity_with_other.csv
 	Rscript $< $(EM) --nosave --no-restore
 
-# combine all combustion extension data
+# Combine all combustion extension data
 $(MED_OUT)/A.comb_default_activity_extended.csv: \
 	$(MOD_A)/A6.4.extended_default_comb_activity.R \
 	$(MED_OUT)/A.comb_activity_extended_coal.csv \
@@ -544,12 +554,11 @@ $(MED_OUT)/A.comb_default_activity_extended.csv: \
 	$(MED_OUT)/A.other_biomass_extended.csv \
 	$(MED_OUT)/A.industrial_biomass_extended.csv \
 	$(MED_OUT)/A.residential_biomass_full.csv \
-	#$(MED_OUT)/A.intl_shipping_en.csv \
 	$(MED_OUT)/A.default_comb_activity_with_other.csv \
-	$(ENERGY_IN)/IEA_iso_start_data.csv
+	$(ENERGY_DATA)/IEA_iso_start_data.csv
 	Rscript $< $(EM) --nosave --no-restore
 
-# Non Combustion activity data
+# Non-Combustion activity data
 $(MED_OUT)/A.NC_activity_extended_db.csv: \
 	$(MOD_A)/A7.1.base_activity.R \
 	$(MOD_A)/A7.2.add_activity.R \
@@ -570,21 +579,32 @@ $(MED_OUT)/A.NC_default_activity_extended.csv: \
 	$(MED_OUT)/A.NC_activity_extended_db.csv
 	Rscript $< $(EM) --nosave --no-restore
 
+# User-added combustion activity incorporated
 $(MED_OUT)/A.comb_user_added.csv: \
 	$(MOD_A)/A8.1.add_user-defined_data.R \
 	$(MED_OUT)/A.comb_default_activity_extended.csv
 	Rscript $< $(EM) --nosave --no-restore
 
+# International shipping fix
+$(MED_OUT)/A.comb_int_shipping_adjusted.csv: \
+	$(MOD_A)/A8.2.Adjust_Shipping_Fuel_Cons.R \
+	$(MED_OUT)/A.comb_user_added.csv \
+	$(MED_OUT)/A.IEA_en_stat_ctry_hist.csv \
+	$(EN_MAPPINGS)/IEA_product_fuel.csv \
+	$(ENERGY_DATA)/Shipping_Fuel_Consumption.xlsx
+	Rscript $< $(EM) --nosave --no-restore
+
+# Final activity data
 $(MED_OUT)/A.total_activity_extended.csv: \
-	$(MOD_A)/A8.2.combine_extended_activity.R \
+	$(MOD_A)/A8.3.combine_extended_activity.R \
 	$(MED_OUT)/A.NC_default_activity_extended.csv \
-	$(MED_OUT)/A.comb_user_added.csv
+	$(MED_OUT)/A.comb_int_shipping_adjusted.csv
 	Rscript $< $(EM) --nosave --no-restore
 
 $(MED_OUT)/A.final_comb_activity_modern.csv: \
-	$(MOD_A)/A8.2.combine_extended_activity.R \
+	$(MOD_A)/A8.3.combine_extended_activity.R \
 	$(MED_OUT)/A.NC_default_activity_extended.csv \
-	$(MED_OUT)/A.comb_user_added.csv
+	$(MED_OUT)/A.comb_int_shipping_adjusted.csv
 	Rscript $< $(EM) --nosave --no-restore
 
 $(MED_OUT)/A.total_activity_extended_coal.csv: \

README.md

@@ -2,26 +2,29 @@
 The Community Emissions Data System (CEDS) produces consistent estimates of global air emissions species over the industrial era (1750 - present). The system is written in R and uses open-source data (with the exception of the IEA energy statistics which must be purchased from IEA). CEDS is publicly available through an [Open Source License](#license-section).
 
 ***
-**Pre-release Version:** August 25, 2019. The current code and data in the repository is a pre-release version. Please feel free to explore, comment on this version and send us any questions. Note that we will be pushing several updates to the system leading up to the first public release. 
+**Public Release:** December 23, 2019. The current code and data in the repository is a full public release of the CEDS system.
 
-* Additional fixes to improve consistency of historical calcuations and user options
-* Resolution of some current [open Issues](https://github.com/JGCRI/CEDS/issues).
-* User guide additions.
-* Description and graphs of emission differences with the CMIP6 data release
+This release was focused on fixing existing issues and adding new system capabilities as compared to the system that produced the CMIP6 data. In particular, the ability of users to add historical energy data for any country, allowing the system to more accurately reflect historical energy consumption trends. This capability is currently used for the USA, UK, and Germany.
 
-See [release notes](https://github.com/JGCRI/CEDS/wiki/Release-Notes) for a summary of changes. 
+For details on this release see:
+
+* The [release notes](https://github.com/JGCRI/CEDS/wiki/Release-Notes) for a summary of changes since the CMIP6 data release.
+* Graphs and explanations for emission differences between this version and the CMIP6 data release documented in Hoesly et al (2018a) [*to be added shortly*]. These differences can be interpreted as corrections to the CMIP6-data release that result from the updates included in this release. 
+* Emissions by country and sector.
+
+We are working on a major CEDS update which will extend the time series to 2018, update historical assumptions where necessary, and will include gridded emissions.
 
 ***
 
 Documentation of CEDS assumptions and system operation, including a user guide, are available at the [CEDS project wiki](https://github.com/JGCRI/CEDS/wiki) and in the journal paper listed below. 
 
-Current issues with the data or system are documented in the [CEDS Issues](https://github.com/JGCRI/CEDS/issues) system in this GitHub repository (current issues as of September 2017 are also listed in the "Known Issues" section of the Data and Assumptions wiki. Newer issues are on GitHub.). Users can submit issues using this system. These can include anomalies found in either the aggregate or gridded emissions data. Please use an appropriate tag for any submitted issues. Note that by default only unresolved issues are shown. All issues, including resolved issues, can be viewed by removing the "is:open" filter. Issues relevant for CMIP6 data releases are tagged with a “CMIP6” label (note that some of these will be closed if resolved in subsequent CEDS data releases.)
+Current issues with the data or system are documented in the [CEDS Issues](https://github.com/JGCRI/CEDS/issues) system in this GitHub repository (current issues as of September 2017 are also listed in the "Known Issues" section of the Data and Assumptions wiki. Newer issues are on GitHub.). Users can submit issues using this system. These can include anomalies found in either the aggregate or gridded emissions data. Please use an appropriate tag for any submitted issues. Note that by default only unresolved issues are shown. All issues, including resolved issues, can be viewed by removing the "is:open" filter. *Issues relevant for CMIP6 data releases are tagged with a “CMIP6” label (note that issues will be closed when resolved in subsequent CEDS data releases.)*
 
-Further information can also be found at the [project web site](http://www.globalchange.umd.edu/ceds/), including a [CMIP6 page](http://www.globalchange.umd.edu/ceds/ceds-cmip6-data/) that provides details for obtaining gridded emission datasets produced by this project. 
+Further information can also be found at the [project web site](http://www.globalchange.umd.edu/ceds/), including a [CMIP6 page](http://www.globalchange.umd.edu/ceds/ceds-cmip6-data/) that provides details for obtaining gridded emission datasets produced by this project for use in CMIP6.
 
-Over the summer and fall of 2019 we will be working on a major update which will extending the time series to 2018 and update historical assumptions.
+If you plan to use the CEDS data system for a research project you are encouraged to contact [Steve Smith](mailto:[email protected]) so that we can coordinate with any on-going work on the CEDS system and make sure we are not duplicating effort. CEDS is research software, and we will be happy to help and make sure that you are able to make the best possible use of this system.
 
-If you plan to use the CEDS data system for a research project you are encouraged to contact [Steve Smith](mailto:[email protected]) so that we can coordinate with any on-going work on the CEDS system. CEDS is research software, and we will be happy to help you make sure and make the best possible use of this system. 
+CEDS has only been possible through the participation of many collaborators. Our **collaboration policy** is that collaborators who contribute data used in CEDS updates will be included as authors on the journal paper that describes the next CEDS major release. We particularly encourage contributions of updated emission information from countries or sectors not well represented in the datasets currently used in CEDS.
 
 # Journal Papers
 [Hoesly et al, Historical (1750–2014) anthropogenic emissions of reactive gases and aerosols from the Community Emissions Data System (CEDS). Geosci. Model Dev. 11, 369-408, 2018a.](https://www.geosci-model-dev.net/11/369/2018/gmd-11-369-2018.html)

code/module-A/A1.1.UN_pop_WB_HYDE_extension.R

@@ -48,8 +48,6 @@
 # TODO: Update extendProxy() function.
 # ------------------------------------------------------------------------------
 
-# ------------------------------------------------------------------------------
-# ------------------------------------------------------------------------------
 # 0. Read in global settings and headers
 # Define PARAM_DIR as the location of the CEDS "parameters" directory, relative
 # to the "input" directory.

code/module-A/A1.3.IEA_downscale_ctry.R

@@ -340,13 +340,15 @@
         "1971", "1990", paste0( "Natural gas liquids (kt) in final consumption go uncounted ",
                           "before 1990 since shares cannot be calculated from 1990 values" ) )
 
-    addMetaData( meta_note, meta_names )
+    source_info <- "A1.3.IEA_downscale_ctry.R"
+
+    addMetaData( meta_note, meta_names, source_info )
 
     meta_note <- c( "Energy Consumption", "NA", "Former Soviet Union", "Consumption",
         "1971", "1990", paste0( "Natural gas liquids (kt) in final consumption go uncounted ",
                           "before 1990 since shares cannot be calculated from 1990 values" ) )
 
-    addMetaData( meta_note, meta_names )
+    addMetaData( meta_note, meta_names, source_info )
 
 # Take the combination of PRODUCT and FLOW columns, and keep only the rows that
 #   contain the combination of PRODUCT and FLOW in IEA_product_downscaling

code/module-A/A1.4.IEA_heat_content.R

@@ -1,6 +1,6 @@
 # Program Name: A1.4.IEA_heat_content.R
 # Author: Linh Vu, Rachel Hoesly
-# Date Last Updated: 16, August 2019
+# Date Last Updated: September 5, 2019
 # Program Purpose: Computes weighted average heat content from IEA Conversion Factors
 #                  by country, year and fuel type. Currently doing this for coal.
 # Input Files: OECD_Conversion_Factors_Full.csv, NonOECD_Conversion_Factors_Full.csv,
@@ -35,7 +35,7 @@
   conversion_NonOECD <- readData( "ENERGY_IN", "NonOECD_Conversion_Factors_Full" )
   activity_data <- readData( "MED_OUT", "A.IEA_en_stat_ctry_hist" )
 
-  IEA_product_fuel <- readData( "MAPPINGS", "IEA_product_fuel", domain_extension = "energy/" )
+  IEA_product_fuel <- readData( "EN_MAPPINGS", "IEA_product_fuel" )
   MCL <- readData( "MAPPINGS", "Master_Country_List" )
 
 # ---------------------------------------------------------------------------

code/module-A/A2.3.write_IEA_diff.R

@@ -1,7 +1,7 @@
 #------------------------------------------------------------------------------
 # Program Name: A2.3.write_IEA_diff.R
 # Authors Names: Linh Vu, Rachel Hoesly, Patrick O'Rourke
-# Date Last Modified: July 10, 2019
+# Date Last Modified: September 5, 2019
 # Program Purpose:    Write out difference between IEA DOMSUP and CEDS
 #                     consumption for coal, natural gas, oil
 # Input Files: A.IEA_en_stat_ctry_hist.csv, en_biomass_fsu_fix.csv
@@ -33,8 +33,8 @@
 
     MSL <- readData( "MAPPINGS", "Master_Fuel_Sector_List", ".xlsx", sheet_selection = "Sectors" )
     IEA_en_stat_ctry_hist <- readData( "MED_OUT", "A.IEA_en_stat_ctry_hist" )
-    IEA_product_fuel <- readData( "MAPPINGS", "IEA_product_fuel", domain_extension = "energy/" )
-    IEA_flow_sector <- readData( "MAPPINGS", "IEA_flow_sector", domain_extension = "energy/" )
+    IEA_product_fuel <- readData( "EN_MAPPINGS", "IEA_product_fuel" )
+    IEA_flow_sector <- readData( "EN_MAPPINGS", "IEA_flow_sector" )
 
     A.en_biomass_fsu_fix <- readData( "MED_OUT", "A.en_biomass_fsu_fix" )