Merge pull request #15 from mo-marqh/scaleOffsetPack

Scale offset packing test added

xios_examples/packing_scale_offset/Makefile

@@ -0,0 +1,58 @@
+# Make file for the pack demonstartion XIOS programme
+# Targets provided our detailed below...
+#
+# all: (default) Build the pack programme
+# clean: Delete all final products and working files
+# run: run the programme
+#
+# Environment Variables expected by this MakeFile:
+#
+# NETCDF_LIBDIR: the directories for the netCDF lib files
+#                encoded as a -L string, e.g.
+#                "-L/dir1 -L/dir2"
+# NETCDF_INCDIR: the directories for the netCDF include files
+#                encoded as a -I string, e.g.
+#                "-I/dir3 -I/dir4"
+#     (note, this is for consistency with the XIOS build process
+#      required for the CI build machine.
+#      this is not required for other directories)
+#
+# XIOS_INCDIR: The directory for XIOS include files
+# XIOS_LIBDIR: The directory for XIOS lib files
+# XIOS_BINDIR: The directory for XIOS binary files
+
+FCFLAGS = -g
+
+FC = mpif90       # compiler driver for MPI programs
+
+# compiler flag, includes
+FCFLAGS += -I$(XIOS_INCDIR) $(NETCDF_INCDIR)
+
+# loader flags
+LDFLAGS = \
+	-L$(XIOS_LIBDIR) \
+	$(NETCDF_LIBDIR) \
+	-lxios \
+	-lnetcdf \
+	-lnetcdff \
+	-lstdc++ 
+
+.PHONY: all, clean, run
+
+all: pack
+
+# fortran compilation
+%.o: %.F90
+	$(FC) $(FCFLAGS) -c $<
+
+# fortran linking
+pack: pack.o
+	$(FC) -o pack.exe pack.o $(LDFLAGS) \
+		&& ln -fs $(XIOS_BINDIR)/xios_server.exe .
+
+run:
+	mpiexec -n 1 ./pack.exe : -n 1 ./xios_server.exe
+
+# cleanup
+clean:
+	rm -f *.exe *.o *.mod *.MOD *.out *.err *.nc

xios_examples/packing_scale_offset/README

@@ -0,0 +1,5 @@
+Offset & Scal Factor packing
+----------------------------
+
+These examples read in arbitrary data from a netCDF file and pack the data to a hard coded precision to produce integer results data with metadata defining how to recover the reduced precision floats.
+The original and packed data are written to one output NetCDF file.

xios_examples/packing_scale_offset/axis_check.xml

@@ -0,0 +1,25 @@
+<context>
+
+  <calendar type="Gregorian"/>
+
+  <axis_definition>
+    <axis id="x" unit="1" long_name="original x coordinate" />
+    <axis id="y" unit="1" long_name="original y coordinate" />
+
+  </axis_definition>
+
+  <grid_definition>
+    <grid id="oax_grid">
+      <axis axis_ref="x" />
+      <axis axis_ref="y" />
+    </grid>
+
+  </grid_definition>
+
+  <file_definition type="one_file">
+    <file id="din" name="domain_input" output_freq="1ts" mode="read" enabled=".true.">
+      <field id="odatax" name="original_data" grid_ref="oax_grid" operation="instant" />
+    </file>
+  </file_definition>
+
+</context>

xios_examples/packing_scale_offset/domain_input_simple_linear.cdl

@@ -0,0 +1,31 @@
+netcdf domain_input {
+dimensions:
+        x = 5 ;
+        y = 5 ;
+variables:
+        float x(x) ;
+                x:long_name = "original x coordinate" ;
+                x:units = "1";
+        float y(y) ;
+                y:long_name = "original y coordinate" ;
+                y:units = "1";
+        double original_data(y,x) ;
+                original_data:long_name = "input data values" ;
+                original_data:units = "1";
+
+// global attributes:
+                :title = "Input data for XIOS Domain resampling; data is a sum of the x & y coordinates each multiplied by 1/3; x/3 + y/3 ." ;
+
+data:
+
+ x = 0, 2, 4, 6, 8 ;
+
+ y = 0, 2, 4, 6, 8 ;
+
+ original_data =  0,  0.6666666666666666,  1.3333333333333333,  2,  2.6666666666666665,
+                  0.6666666666666666,  1.3333333333333333,  2,  2.6666666666666665, 3.3333333333333335,
+                  1.3333333333333333,  2,  2.6666666666666665, 3.3333333333333335, 4,
+                  2,  2.6666666666666665, 3.3333333333333335, 4, 4.666666666666667,
+                  2.6666666666666665, 3.3333333333333335, 4, 4.666666666666667, 5.333333333333333 ;
+
+}

xios_examples/packing_scale_offset/iodef.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<simulation>
+
+<context id="main" src="main.xml"/>
+<context id="axis_check" src="axis_check.xml"/>
+
+<context id="xios" src="xios.xml"/>
+
+</simulation>

xios_examples/packing_scale_offset/main.xml

@@ -0,0 +1,37 @@
+<context>
+
+  <calendar type="Gregorian"/>
+
+  <domain_definition>
+
+    <domain id="original_domain" type="rectilinear" />
+
+  </domain_definition>
+
+  <grid_definition>
+
+    <grid id="original_grid">
+      <domain domain_ref="original_domain" />
+    </grid>
+
+  </grid_definition>
+
+  <field_definition>
+    <field id="odata" name="original_data" grid_ref="original_grid" long_name="input data values" unit="1" prec="8" />
+    <field id="pdata" name="packed_data" grid_ref="original_grid" long_name="packed data values" unit="1" scale_factor="0.01" add_offset="0" compression_level="9" prec="2" />
+  </field_definition>
+
+
+  <file_definition type="one_file">
+    <file id="domain_input" output_freq="1ts" mode="read" enabled=".true.">
+      <field id="odatain" name="original_data" grid_ref="original_grid" operation="instant" />
+    </file>
+    <file id="domain_output" output_freq="1ts">
+      <field_group operation="once">
+        <field field_ref="odata" />
+        <field field_ref="pdata" />
+      </field_group>
+    </file>
+  </file_definition>
+
+</context>

xios_examples/packing_scale_offset/pack.F90

@@ -0,0 +1,120 @@
+!-----------------------------------------------------------------------------
+! (C) Crown copyright 2020 Met Office. All rights reserved.
+! The file LICENCE, distributed with this code, contains details of the terms
+! under which the code may be used.
+!-----------------------------------------------------------------------------
+!> Read 2D data on a domain and resample using the axis_input.nc file
+!>
+program resample
+  use xios
+  use mpi
+
+  implicit none
+
+  integer :: comm = -1
+  integer :: rank = -1
+  integer :: npar = 0
+
+  call initialise()
+  call simulate()
+  call finalise()
+contains
+
+  subroutine initialise()
+
+    type(xios_date) :: origin
+    type(xios_date) :: start
+    type(xios_duration) :: tstep
+    integer :: mpi_error
+    integer :: lenx
+    integer :: leny
+
+    ! Arbitrary datetime setup, required for XIOS but unused
+    origin = xios_date(2022, 2, 2, 12, 0, 0)
+    start = xios_date(2022, 12, 13, 12, 0, 0)
+    tstep = xios_hour
+
+    ! Initialise MPI and XIOS
+    call MPI_INIT(mpi_error)
+
+    call xios_initialize('client', return_comm=comm)
+
+    call MPI_Comm_rank(comm, rank, mpi_error)
+    call MPI_Comm_size(comm, npar, mpi_error)
+
+    ! use the axis_check context to obtain sizing information on all arrays
+    ! for use in defining the main context interpretation
+    call xios_context_initialize('axis_check', comm)
+    call xios_set_time_origin(origin)
+    call xios_set_start_date(start)
+    call xios_set_timestep(tstep)
+
+    call xios_close_context_definition()
+
+    call xios_get_axis_attr('x', n_glo=lenx)
+    call xios_get_axis_attr('y', n_glo=leny)
+
+    ! initialize the main context for interacting with the data.
+    call xios_context_initialize('main', comm)
+
+    call xios_set_time_origin(origin)
+    call xios_set_start_date(start)
+    call xios_set_timestep(tstep)
+
+    call xios_set_domain_attr("original_domain", ni=lenx, nj=leny, ibegin=0, jbegin=0)
+
+    call xios_close_context_definition()
+
+  end subroutine initialise
+
+  subroutine finalise()
+
+    integer :: mpi_error
+
+    ! Finalise all XIOS contexts and MPI
+    call xios_set_current_context('axis_check')
+    call xios_context_finalize()
+    call xios_set_current_context('main')
+    call xios_context_finalize()
+    call MPI_Comm_free(comm, mpi_error)
+    call xios_finalize()
+    call MPI_Finalize(mpi_error)
+
+  end subroutine finalise
+
+  subroutine simulate()
+
+    type(xios_date) :: current
+    integer :: ts
+    integer :: lenx
+    integer :: leny
+    integer :: i, j
+
+    ! Allocatable arrays, size is taken from input file
+    double precision, dimension (:,:), allocatable :: inodata
+
+    call xios_get_domain_attr('original_domain', ni_glo=lenx)
+    call xios_get_domain_attr('original_domain', nj_glo=leny)
+
+    allocate ( inodata(leny, lenx) )
+
+    ! Load data from the input file
+    call xios_recv_field('odatain', inodata)
+
+    do ts=1, 1
+      call xios_update_calendar(ts)
+      call xios_get_current_date(current)
+      ! Send (copy) the original data to the output file.
+      call xios_send_field('odata', inodata)
+      ! Send the original data to the output file pack target.
+      ! The `pdata` field is defined as integer with scale_factor
+      ! and `add_offset` defined, so XIOS will pack this data
+      ! on write.
+      call xios_send_field('pdata', inodata)
+    enddo
+
+    deallocate (inodata)
+
+  end subroutine simulate
+
+end program resample

xios_examples/packing_scale_offset/test_packing_cases.py

@@ -0,0 +1,89 @@
+import copy
+import glob
+import netCDF4
+import numpy as np
+import os
+import subprocess
+import unittest
+
+import xios_examples.shared_testing as xshared
+
+this_path = os.path.realpath(__file__)
+this_dir = os.path.dirname(this_path)
+
+class TestPackDomain(xshared._TestCase):
+    test_dir = this_dir
+    transient_inputs = ['domain_input.nc']
+    transient_outputs = ['domain_output.nc']
+    executable = './pack.exe'
+
+    @classmethod
+    def make_a_pack_test(cls, inf):
+        """
+        this function makes a test case and returns it as a test function,
+        suitable to be dynamically added to a TestCase for running.
+
+        """
+        # always copy for value, don't pass by reference.
+        infile = copy.copy(inf)
+        # expected by the fortran XIOS pack program's main.xml
+        inputfile = cls.transient_inputs[0]
+        outputfile = cls.transient_outputs[0]
+        def test_pack(self):
+            # create a netCDF file from the `.cdl` input
+            subprocess.run(['ncgen', '-k', 'nc4', '-o', inputfile,
+                            infile], cwd=cls.test_dir, check=True)
+            cls.run_mpi_xios()
+
+            # load the result netCDF file
+            runfile = '{}/{}'.format(cls.test_dir, outputfile)
+            assert(os.path.exists(runfile))
+            rootgrp = netCDF4.Dataset(runfile, 'r')
+            # read data from the packed, expected & diff variables
+            expected = rootgrp['original_data'][:]
+            result = rootgrp['packed_data'][:]
+            rtol = rootgrp['packed_data'].scale_factor
+            # prepare message for failure
+            msg = ('\n the packed data array\n {res}\n '
+                   'differs from the original data array\n {exp} \n '
+                   'with diff outside expected tolerance {rtol}\n {diff}\n')
+            msg = msg.format(exp=expected, res=result, rtol=rtol,
+                             diff=result-expected)
+            if not np.allclose(result, expected, rtol=rtol):
+                # print message for fail case,
+                # as expected failures do not report msg.
+                print(msg)
+            # assert that all of the `diff` varaible values are zero
+            # self.assertTrue(not np.any(diff), msg=msg)
+            self.assertTrue(np.allclose(result, expected, rtol=rtol) and
+                            not np.allclose(result, expected, rtol=0.1*rtol),
+                            msg=msg)
+        return test_pack
+
+
+# A list of input `.cdl` files where XIOS is known to produce different
+# output from the expected output data
+# for future investigation / ToDo
+# this is a dict, where the name of the key is the name of the test
+# to register as a known_failure (tname)
+# and the value is a string explaining the failure
+# this handles FAIL conditions but NOT ERROR conditions
+known_failures = {}
+
+# iterate through `.cdl` files in this test case folder
+for f in glob.glob('{}/*.cdl'.format(this_dir)):
+    # unique name for the test
+    tname = 'test_{}'.format(os.path.splitext(os.path.basename(f))[0])
+    # add the test as an attribute (function) to the test class
+    if os.environ.get('MVER', '').startswith('XIOS3/trunk'):
+        # these tests are hitting exceptions with XIOS3
+        # but not XIOS2, so skip for XIOS3 runner
+        setattr(TestPackDomain, tname,
+                unittest.skip(TestPackDomain.make_a_pack_test(f)))
+    elif tname in known_failures:
+        # set decorator @unittest.expectedFailure
+        setattr(TestPackDomain, tname,
+                unittest.expectedFailure(TestPackDomain.make_a_pack_test(f)))
+    else:
+        setattr(TestPackDomain, tname,
+                TestPackDomain.make_a_pack_test(f))