Auto-generate zero data for missing data payload (#362)

* DataProxy auto-generate zero data

* remove expectedFailure decorator for pickle tests

* review actions

.cirrus.yml

@@ -28,8 +28,8 @@ env:
   PIP_CACHE_PACKAGES: "pip setuptools wheel nox pyyaml"
   # Base directory for the iris-test-data.
   IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data
-  # Git commit hash for iris test data.
-  IRIS_TEST_DATA_REF: "2.0.0"
+  # Tagged release version of the iris-test-data.
+  IRIS_TEST_DATA_REF: "2.23"
 
 #
 # Linting

iris_grib/message.py

@@ -237,14 +237,29 @@ def _bitmap(self, bitmap_section):
         return bitmap
 
     def __getitem__(self, keys):
-        # NB. Currently assumes that the validity of this interpretation
+        # N.B. Assumes that the validity of this interpretation
         # is checked before this proxy is created.
 
         message = self.recreate_raw()
         sections = message.sections
+        data = None
+
+        if 5 in sections:
+            # Data Representation Section.
+            if sections[5]["bitsPerValue"] == 0:
+                # Auto-generate zero data of the expected shape and dtype, as
+                # there is no data stored within the Data Section of this GRIB
+                # message. Also flatten the result to 1-D for potential bitmap
+                # post-processing.
+                data = np.ravel(np.zeros(self.shape, dtype=self.dtype))
+
+        if data is None:
+            # Data Section.
+            data = sections[7]["codedValues"]
+
+        # Bit-map Section.
         bitmap_section = sections[6]
         bitmap = self._bitmap(bitmap_section)
-        data = sections[7]['codedValues']
 
         if bitmap is not None:
             # Note that bitmap and data are both 1D arrays at this point.

iris_grib/tests/integration/iris_integration/test_pickle_message.py

@@ -9,7 +9,6 @@
 import iris_grib.tests as tests
 
 import pickle
-import unittest
 
 from iris.tests.integration.test_pickle import Common as PickleCommon
 
@@ -39,15 +38,12 @@ def pickle_obj(self, obj):
 
     # These probably "ought" to work, but currently fail.
     # see https://github.com/SciTools/iris-grib/issues/202
-    @unittest.expectedFailure
     def test_protocol_0(self):
         super().test_protocol_0()
 
-    @unittest.expectedFailure
     def test_protocol_1(self):
         super().test_protocol_1()
 
-    @unittest.expectedFailure
     def test_protocol_2(self):
         super().test_protocol_2()
 

iris_grib/tests/integration/load_convert/test_data_section.py

@@ -11,6 +11,7 @@
 # before importing anything else.
 import iris_grib.tests as tests
 
+import numpy as np
 import numpy.ma as ma
 
 from iris import load_cube
@@ -68,5 +69,20 @@ def test_grid_complex_spatial_differencing(self):
         self.assertCMLApproxData(cube)
 
 
+class TestDataProxy(tests.IrisGribTest):
+    def test_data_representation__no_bitsPerValue(self):
+        """Ensures that zero data is auto-generated."""
+        # This test file contains one GRIB2 message with no data payload
+        # in Data Section [7], but has "bitsPerValue=0" in the Data
+        # Representation Section [5] to trigger auto-generation of zero
+        # data payload by the DataProxy.
+        path = tests.get_data_path(
+            ("GRIB", "missing_values", "ice_severity__no_bitsPerValue.grib2")
+        )
+        cube = load_cube(path)
+        self.assertCML(cube)
+        self.assertEqual(0, np.sum(cube.data))
+
+
 if __name__ == '__main__':
     tests.main()

iris_grib/tests/results/integration/load_convert/data_section/TestDataProxy/data_representation__no_bitsPerValue.cml

@@ -0,0 +1,35 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" units="unknown">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c019n037"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[18]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[472014.]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="77a50eb5" points="[90.  , 89.75, 89.5 , ..., 60.5 , 60.25, 60.  ]" shape="(121,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="f913a8b3" points="[ 60.  ,  60.25,  60.5 , ..., 209.5 , 209.75,
+ 210.  ]" shape="(601,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="302bf438" long_name="pressure" points="[30090.]" shape="(1,)" units="Unit('Pa')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[472032.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0x13d8eac8" dtype="float64" shape="(121, 601)"/>
+  </cube>
+</cubes>