diff --git a/.github/workflows/run_netcdf_test.yml b/.github/workflows/run_netcdf_test.yml new file mode 100644 index 0000000..432c077 --- /dev/null +++ b/.github/workflows/run_netcdf_test.yml @@ -0,0 +1,23 @@ +name: run-netcdf-test + +on: + push: + branches: + - main + pull_request: + +jobs: + netcdf_test: + name: netcdf test + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + sudo apt update + sudo apt -yq install $( +#include +#include +#include +#include +#include +#include "quantize.h" +#include "quantize_params.c" + +double clock_to_second(clock_t t) { + // Return double precision time in seconds + double time_taken; + time_taken = ((double) t) / CLOCKS_PER_SEC; + return time_taken; +} + +void populateData(float* data_ptr, int dim1_size, int dim2_size) { + /* populate the data at address data_ptr with a 2d analytic function, + a sine wave along the j dimension, that scales along the i direction */ + int ii, jj; + float dbl_dim2_size; + dbl_dim2_size = (float) dim2_size; + for (ii = 0; ii < dim1_size; ii++) { + for (jj = 0; jj < dim2_size; jj++) { + *(data_ptr + (ii * dim1_size) + jj) = \ + sin(((2.0*PI)/dbl_dim2_size)*jj) *ii; + } + } +} + + +int main() { + int i; + + // Define the number of dimensions and size of data + int ndims = 2; + int dim1_size = 10; + int dim2_size = 10; + + float* data_ptr; + struct PackingParams* packing_params; + struct PackingParams my_param; + + /* NetCDF related variables */ + int ncid; + int x_dimid, y_dimid, varid; + int dimids[ndims]; + + // Time variable + clock_t t; + double write_time, close_time; + + // Define our packing params from quantize_params.h + packing_params = define_params(); + + // Create data only once + data_ptr = (float*) calloc(dim1_size*dim2_size, sizeof(float)); + populateData(data_ptr, dim1_size, dim2_size); + + + /* Loop over the array of packing_params defined in quantize_params.h, + setting up and writing the variable in data_ptr depending on the + individual parameter setting */ + for (i = 0; i 0) { + nc_def_var_quantize(ncid, varid, my_param.netcdf_quantize_mode, \ + my_param.netcdf_nsd); + } + + // Set up compression if appropriate + if (my_param.compress > 0) { + nc_def_var_deflate(ncid, varid, 0, 1, 1); + } + + // End definitition + nc_enddef(ncid); + + // write the data into the file + t = clock(); + nc_put_var_float(ncid, varid, data_ptr); + write_time = clock_to_second(clock() - t); + + printf("File %s nc_put_var_float() takes %.4f s\n", + my_param.filename, write_time); + + //close the file + t = clock(); + nc_close(ncid); + close_time = clock_to_second(clock() - t); + printf("File %s nc_close() takes %.4f s \n", + my_param.filename, close_time); + + printf("File %s total time: %.4f\n\n", my_param.filename, + write_time + close_time); + } + free(data_ptr); + free(packing_params); + return 0; +} diff --git a/netcdf_examples/quantize_test/quantize.h b/netcdf_examples/quantize_test/quantize.h new file mode 100644 index 0000000..b156674 --- /dev/null +++ b/netcdf_examples/quantize_test/quantize.h @@ -0,0 +1,10 @@ +/* Function prototype for quantize_params.c */ +struct PackingParams *define_params(); + +/* function prototypes for quantize.c */ +double clock_to_second(clock_t); +void populateData(float*, int, int); +int main(); + +/* Definitions */ +#define PI 3.14159265 diff --git a/netcdf_examples/quantize_test/quantize_params.c b/netcdf_examples/quantize_test/quantize_params.c new file mode 100644 index 0000000..39ce2da --- /dev/null +++ b/netcdf_examples/quantize_test/quantize_params.c @@ -0,0 +1,69 @@ +//How many experiments are we running? +#define NUM_PACKING_PARAMS 5 + +/* Define our struct to hold the packing parameters. This are populated + in the define_params function below, allowing a neat way of iterating + over experiments */ +struct PackingParams { + int compress; // perform compression, 1 for compression, 0 for not + int do_quantize; // perform quantization, 1 for quantization, 0 for not + int netcdf_quantize_mode; /* Chose netcdf quantization mode, choices: + 1) NC_QUANTIZE_BITGROOM + 2) NC_QUANTIZE_GRANULARBR + 3) NC_QUANTIZE_BITROUND */ + int netcdf_nsd; /* Number of significant digits to preserve. For + NC_QUANTIZE_BITGROOM and NC_QUANTIZE_GRANULARBR these are + decimal significant figures, for NC_QUANTIZE_BITROUND + these are binary signficant figures. + Note: 1 decimal sf requires ~3.32 bits. */ + char filename[200]; // Name of file to write to + char fieldname[200]; // Name of field to write to file +}; + +struct PackingParams *define_params() { + + //Allocate our array of parameters + struct PackingParams* param_array; + param_array = (struct PackingParams*) \ + malloc(NUM_PACKING_PARAMS*sizeof(struct PackingParams)); + + // define our parameters for this experiment + // Reference + param_array[0].compress = 0; + param_array[0].do_quantize = 0; + strcpy(param_array[0].filename, "reference.nc"); + strcpy(param_array[0].fieldname, "field"); + + // Reference compress + param_array[1].compress = 1; + param_array[1].do_quantize = 0; + strcpy(param_array[1].filename, "reference_comp.nc"); + strcpy(param_array[1].fieldname, "field"); + + // Bitgroom pack and compress, nsd = 3 + param_array[2].compress = 1; + param_array[2].do_quantize = 1; + param_array[2].netcdf_quantize_mode = NC_QUANTIZE_BITGROOM; + param_array[2].netcdf_nsd = 3; + strcpy(param_array[2].filename, "quant_bg_3_comp.nc"); + strcpy(param_array[2].fieldname, "field"); + + // Granular pack and compress, nsd = 3 + param_array[3].compress = 1; + param_array[3].do_quantize = 1; + param_array[3].netcdf_quantize_mode = NC_QUANTIZE_GRANULARBR; + param_array[3].netcdf_nsd = 3; + strcpy(param_array[3].filename, "quant_gran_3_comp.nc"); + strcpy(param_array[3].fieldname, "field"); + + // Bitround pack and compress, nsd = 10bit (3dec) + param_array[4].compress = 1; + param_array[4].do_quantize = 1; + param_array[4].netcdf_quantize_mode = NC_QUANTIZE_BITROUND; + param_array[4].netcdf_nsd = 10; + strcpy(param_array[4].filename, "quant_br_10b_comp.nc"); + strcpy(param_array[4].fieldname, "field"); + + + return param_array; +} diff --git a/netcdf_examples/quantize_test/reference.cdl b/netcdf_examples/quantize_test/reference.cdl new file mode 100644 index 0000000..5836c90 --- /dev/null +++ b/netcdf_examples/quantize_test/reference.cdl @@ -0,0 +1,29 @@ +netcdf reference { +dimensions: + x = 10 ; + y = 10 ; +variables: + float field(x, y) ; +data: + + field = + 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, + 0, 0.5877852, 0.9510565, 0.9510565, 0.5877852, 3.589793e-09, -0.5877852, + -0.9510565, -0.9510565, -0.5877852, + 0, 1.17557, 1.902113, 1.902113, 1.17557, 7.179586e-09, -1.17557, -1.902113, + -1.902113, -1.17557, + 0, 1.763356, 2.853169, 2.853169, 1.763356, 1.076938e-08, -1.763356, + -2.853169, -2.853169, -1.763356, + 0, 2.351141, 3.804226, 3.804226, 2.351141, 1.435917e-08, -2.351141, + -3.804226, -3.804226, -2.351141, + 0, 2.938926, 4.755282, 4.755282, 2.938926, 1.794896e-08, -2.938926, + -4.755282, -4.755282, -2.938926, + 0, 3.526711, 5.706339, 5.706339, 3.526711, 2.153876e-08, -3.526711, + -5.706339, -5.706339, -3.526711, + 0, 4.114497, 6.657396, 6.657396, 4.114497, 2.512855e-08, -4.114497, + -6.657396, -6.657396, -4.114497, + 0, 4.702282, 7.608452, 7.608452, 4.702282, 2.871835e-08, -4.702282, + -7.608452, -7.608452, -4.702282, + 0, 5.290067, 8.559508, 8.559508, 5.290067, 3.230814e-08, -5.290067, + -8.559508, -8.559508, -5.290067 ; +} diff --git a/netcdf_examples/quantize_test/reference_comp.cdl b/netcdf_examples/quantize_test/reference_comp.cdl new file mode 100644 index 0000000..cdd61eb --- /dev/null +++ b/netcdf_examples/quantize_test/reference_comp.cdl @@ -0,0 +1,29 @@ +netcdf reference_comp { +dimensions: + x = 10 ; + y = 10 ; +variables: + float field(x, y) ; +data: + + field = + 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, + 0, 0.5877852, 0.9510565, 0.9510565, 0.5877852, 3.589793e-09, -0.5877852, + -0.9510565, -0.9510565, -0.5877852, + 0, 1.17557, 1.902113, 1.902113, 1.17557, 7.179586e-09, -1.17557, -1.902113, + -1.902113, -1.17557, + 0, 1.763356, 2.853169, 2.853169, 1.763356, 1.076938e-08, -1.763356, + -2.853169, -2.853169, -1.763356, + 0, 2.351141, 3.804226, 3.804226, 2.351141, 1.435917e-08, -2.351141, + -3.804226, -3.804226, -2.351141, + 0, 2.938926, 4.755282, 4.755282, 2.938926, 1.794896e-08, -2.938926, + -4.755282, -4.755282, -2.938926, + 0, 3.526711, 5.706339, 5.706339, 3.526711, 2.153876e-08, -3.526711, + -5.706339, -5.706339, -3.526711, + 0, 4.114497, 6.657396, 6.657396, 4.114497, 2.512855e-08, -4.114497, + -6.657396, -6.657396, -4.114497, + 0, 4.702282, 7.608452, 7.608452, 4.702282, 2.871835e-08, -4.702282, + -7.608452, -7.608452, -4.702282, + 0, 5.290067, 8.559508, 8.559508, 5.290067, 3.230814e-08, -5.290067, + -8.559508, -8.559508, -5.290067 ; +} diff --git a/netcdf_examples/quantize_test/test_quantize.py b/netcdf_examples/quantize_test/test_quantize.py new file mode 100644 index 0000000..88fe4c4 --- /dev/null +++ b/netcdf_examples/quantize_test/test_quantize.py @@ -0,0 +1,74 @@ +import copy +import glob +import netCDF4 +import numpy +import os +import sys +import subprocess +import unittest + +this_path = os.path.realpath(__file__) +this_dir = os.path.dirname(this_path) + +class TestQuantize(unittest.TestCase): + + test_dir = this_dir + tolerance = 1.0e-04 + expected_netcdf_files = ['quant_bg_3_comp.nc', + 'quant_br_10b_comp.nc', + 'quant_gran_3_comp.nc', + 'reference.nc', + 'reference_comp.nc'] + + def setUp(self): + ''' + In the setup, we build the quantization test, and run it + ''' + subprocess.run(['make', 'clean'], cwd=self.test_dir) + subprocess.run(['make', 'quantize_github'], cwd=self.test_dir) + subprocess.run('./quantize.exe', cwd=self.test_dir) + + def test_cdl_files(self): + ''' + Test we have the expected .cdl files + ''' + expected_cdl_files = ['quant_bg_3_comp.cdl', + 'quant_br_10b_comp.cdl', + 'quant_gran_3_comp.cdl', + 'reference.cdl', + 'reference_comp.cdl'] + cdl_files = [f for f in os.listdir(self.test_dir) if f[-3:] == 'cdl'] + self.assertCountEqual(cdl_files, expected_cdl_files) + + def test_netcdf_files(self): + ''' + Check the expected netcdf files are generated + ''' + nc_files = [f for f in os.listdir(self.test_dir) if f[-2:] == 'nc'] + self.assertCountEqual(nc_files, self.expected_netcdf_files) + + def test_check_output(self): + ''' + Generate our reference netcdf files from the cdl files, and ensure + they are the same (within self.tolerance) as those produced from + the test + ''' + for netcdf_file in self.expected_netcdf_files: + netcdf_fileroot = ''.join(netcdf_file.split('.')[:-1]) + # create our reference netcdf file + reference_ncfile = '{}_ref.nc'.format(netcdf_fileroot) + subprocess.run(['ncgen', '-k', 'nc4', '-o', reference_ncfile, + '{}.cdl'.format(netcdf_fileroot)], + cwd=self.test_dir) + test_results = netCDF4.Dataset( + os.path.join(self.test_dir, netcdf_file))['field'][:] + expected = netCDF4.Dataset( + os.path.join(self.test_dir, reference_ncfile))['field'][:] + diff = test_results - expected + result = numpy.allclose(test_results, expected, rtol=self.tolerance) + if not result: + sys.stdout.write('The produced data array in file {0}.nc ' + 'differes from \nthat in' + ' the reference cdl file {0}.cdl\n'. + format(netcdf_file)) + self.assertTrue(result)