Changes to SST1MEventSource (#729)

* Add test using SST1MEventSource in the calibration pipeline

* Cleanup of SST1MEventSource

Removed functions inside containers
Corrected tels_with_data into a set
Added the channel dimension to r0.tel[i].waveform

* Corrected telid for subarray

* Added container to __all__

* Filled event_id and tels_with_data for other levels

* Moved TelescopeDescription so that it is not re-created for each event

* Add obs_id for the other data levels

* Some minor corrections

Restored relative import
Corrected comment

ctapipe/io/containers.py

@@ -21,6 +21,8 @@
            'DL1CameraContainer',
            'TargetIOContainer',
            'TargetIOCameraContainer',
+           'SST1MContainer',
+           'SST1MCameraContainer',
            'MCEventContainer',
            'MCHeaderContainer',
            'MCCameraEventContainer',
@@ -32,6 +34,7 @@
            'ParticleClassificationContainer',
            'DataContainer',
            'TargetIODataContainer',
+           'SST1MDataContainer',
            'HillasParametersContainer']
 
 
@@ -50,35 +53,13 @@ class SST1MCameraContainer(Container):
         None,
         "trigger 19 patch cluster trace (n_clusters)")
 
-    def fill_from_zfile_event(self, event, pixel_sort_ids):
-        self.pixel_flags = event.pixels_flags[pixel_sort_ids]
-        self.digicam_baseline = event.hiGain.waveforms.baselines[
-            pixel_sort_ids]
-        self.local_camera_clock = (
-            event.local_time_sec * 1E9 + event.local_time_nanosec)
-        self.gps_time = (
-            event.trig.timeSec * 1E9 + event.trig.timeNanoSec)
-        self.camera_event_type = event.event_type
-        self.array_event_type = event.eventType
-        self.trigger_input_traces = event.trigger_input_traces
-        self.trigger_output_patch7 = event.trigger_output_patch7
-        self.trigger_output_patch19 = event.trigger_output_patch19
-
 
 class SST1MContainer(Container):
     tels_with_data = Field([], "list of telescopes with data")
     tel = Field(
         Map(SST1MCameraContainer),
         "map of tel_id to SST1MCameraContainer")
 
-    def fill_from_zfile_event(self, event, pixel_sort_ids):
-        self.tels_with_data = [event.telescopeID, ]
-        sst1m_cam_container = self.tel[event.telescopeID]
-        sst1m_cam_container.fill_from_zfile_event(
-            event,
-            pixel_sort_ids,
-        )
-
 
 # todo: change some of these Maps to be just 3D NDarrays?
 

ctapipe/io/sst1meventsource.py

@@ -18,38 +18,76 @@ def __init__(self, config=None, tool=None, **kwargs):
         super().__init__(config=config, tool=tool, **kwargs)
         from protozfits import SimpleFile
         self.file = SimpleFile(self.input_url)
-
+        # TODO: Correct pixel ordering
+        self._tel_desc = TelescopeDescription.from_name(
+            optics_name='SST-1M',
+            camera_name='DigiCam'
+        )
 
     def _generator(self):
-        self._pixel_sort_ids = None
+        pixel_sort_ids = None
 
         for count, event in enumerate(self.file.Events):
-            if self._pixel_sort_ids is None:
-                self._pixel_sort_ids = np.argsort(
-                    event.hiGain.waveforms.pixelsIndices)
-                self.n_pixels = len(self._pixel_sort_ids)
+            if pixel_sort_ids is None:
+                pixel_indices = event.hiGain.waveforms.pixelsIndices
+                pixel_sort_ids = np.argsort(pixel_indices)
+                self.n_pixels = len(pixel_sort_ids)
+            telid = event.telescopeID
             data = SST1MDataContainer()
             data.count = count
-            data.sst1m.fill_from_zfile_event(event, self._pixel_sort_ids)
-            self.fill_R0Container_from_zfile_event(data.r0, event)
-            data.inst.subarray.tels[0] = TelescopeDescription.from_name(
-                optics_name='SST-1M', 
-                camera_name='DigiCam'
-            )
-            yield data
 
+            data.inst.subarray.tels[telid] = self._tel_desc
+
+            # Data level Containers
+            data.r0.obs_id = -1
+            data.r0.event_id = event.eventNumber
+            data.r0.tels_with_data = {telid}
+            data.r1.obs_id = -1
+            data.r1.event_id = event.eventNumber
+            data.r1.tels_with_data = {telid}
+            data.dl0.obs_id = -1
+            data.dl0.event_id = event.eventNumber
+            data.dl0.tels_with_data = {telid}
+
+            # R0CameraContainer
+            camera_time = event.local_time_sec * 1E9 + event.local_time_nanosec
+            samples = event.hiGain.waveforms.samples.reshape(self.n_pixels, -1)
+            data.r0.tel[telid].trigger_time = camera_time
+            data.r0.tel[telid].trigger_type = event.event_type
+            data.r0.tel[telid].waveform = samples[pixel_sort_ids][None, :]
+            data.r0.tel[telid].num_samples = samples.shape[-1]
+
+            # SST1MContainer
+            data.sst1m.tels_with_data = {telid}
+
+            # SST1MCameraContainer
+            digicam_baseline = event.hiGain.waveforms.baselines[pixel_sort_ids]
+            gps_time = (event.trig.timeSec * 1E9 + event.trig.timeNanoSec)
+            container = data.sst1m.tel[telid]
+            container.pixel_flags = event.pixels_flags[pixel_sort_ids]
+            container.digicam_baseline = digicam_baseline
+            container.local_camera_clock = camera_time
+            container.gps_time = gps_time
+            container.camera_event_type = event.event_type
+            container.array_event_type = event.eventType
+            container.trigger_input_traces = event.trigger_input_traces
+            container.trigger_output_patch7 = event.trigger_output_patch7
+            container.trigger_output_patch19 = event.trigger_output_patch19
+
+            yield data
 
     @staticmethod
     def is_compatible(file_path):
         from astropy.io import fits
         try:
             h = fits.open(file_path)[1].header
-            ttypes = [
-                h[x] for x in h.keys() if 'TTYPE' in x
-            ]
+            ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
         except OSError:
             # not even a fits file
             return False
+        except IndexError:
+            # A fits file of a different format
+            return False
 
         is_protobuf_zfits_file = (
             (h['XTENSION'] == 'BINTABLE') and
@@ -61,30 +99,3 @@ def is_compatible(file_path):
         is_sst1m_file = 'trigger_input_traces' in ttypes
 
         return is_protobuf_zfits_file & is_sst1m_file
-
-    def fill_R0CameraContainer_from_zfile_event(self, container, event):
-        container.trigger_time = (
-            event.local_time_sec * 1E9 + event.local_time_nanosec)
-        container.trigger_type = event.event_type
-
-        _samples = (
-            event.hiGain.waveforms.samples
-        ).reshape(self.n_pixels, -1)
-        container.waveform = _samples[self._pixel_sort_ids]
-
-        # Why is this needed ???
-        # This is exactly the definition of waveforms.
-        container.num_samples = container.waveform.shape[1]
-
-    def fill_R0Container_from_zfile_event(self, container, event):
-        container.obs_id = -1  # I do not know what this is.
-        container.event_id = event.eventNumber
-        container.tels_with_data = [event.telescopeID, ]
-        r0_cam_container = container.tel[event.telescopeID]
-
-        self.fill_R0CameraContainer_from_zfile_event(
-            r0_cam_container,
-            event
-        )
-
-

ctapipe/io/tests/test_sst1meventsource.py

@@ -1,8 +1,9 @@
 from pkg_resources import resource_filename
 import os
 import copy
-
 import pytest
+from ctapipe.calib.camera.calibrator import CameraCalibrator
+
 pytest.importorskip("protozfits", minversion="0.44.3")
 
 example_file_path = resource_filename(
@@ -15,7 +16,7 @@
 )
 
 FIRST_EVENT_NUMBER_IN_FILE = 97750287
-ADC_SAMPLES_SHAPE = (1296, 50)
+ADC_SAMPLES_SHAPE = (1, 1296, 50)
 
 
 def test_loop_over_events():
@@ -28,7 +29,7 @@ def test_loop_over_events():
     )
 
     for i, event in enumerate(inputfile_reader):
-        assert event.r0.tels_with_data == [1]
+        assert event.r0.tels_with_data == {1}
         for telid in event.r0.tels_with_data:
             assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i
             assert event.r0.tel[telid].waveform.shape == ADC_SAMPLES_SHAPE
@@ -70,3 +71,12 @@ def test_factory_for_protozfits_file():
     reader = EventSourceFactory.produce(input_url=example_file_path)
     assert isinstance(reader, SST1MEventSource)
     assert reader.input_url == example_file_path
+
+
+def test_pipeline():
+    from ctapipe.io.sst1meventsource import SST1MEventSource
+    reader = SST1MEventSource(input_url=example_file_path, max_events=10)
+    calibrator = CameraCalibrator(eventsource=reader)
+    for event in reader:
+        calibrator.calibrate(event)
+        assert event.r0.tel.keys() == event.dl1.tel.keys()