From 319e3494b10a3ff1d014e8f4c7d0e779b9f75051 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Thu, 9 Jan 2025 14:00:49 -0500 Subject: [PATCH] JP-3682: split outlier detection into multiple steps --- jwst/outlier_detection/__init__.py | 12 +- jwst/outlier_detection/_fileio.py | 2 +- jwst/outlier_detection/coron.py | 67 ----- jwst/outlier_detection/imaging.py | 101 -------- .../outlier_detection_coron_step.py | 81 ++++++ .../{ifu.py => outlier_detection_ifu_step.py} | 180 +++++++------- .../outlier_detection_imaging_step.py | 121 +++++++++ .../outlier_detection_spec_step.py | 124 ++++++++++ .../outlier_detection_step.py | 230 ------------------ .../{tso.py => outlier_detection_tso_step.py} | 112 +++++---- jwst/outlier_detection/spec.py | 108 -------- .../tests/test_algorithms.py | 2 +- jwst/outlier_detection/tests/test_fileio.py | 6 +- .../tests/test_outlier_detection.py | 27 +- jwst/outlier_detection/utils.py | 56 +++++ jwst/pipeline/calwebb_coron3.py | 4 +- jwst/pipeline/calwebb_image3.py | 4 +- jwst/pipeline/calwebb_spec3.py | 21 +- jwst/pipeline/calwebb_tso3.py | 6 +- jwst/pipeline/outlier_detection.cfg | 2 - jwst/pipeline/outlier_detection_tso.cfg | 2 +- jwst/step.py | 12 +- jwst/stpipe/integration.py | 6 +- 23 files changed, 602 insertions(+), 684 deletions(-) delete mode 100644 jwst/outlier_detection/coron.py delete mode 100644 jwst/outlier_detection/imaging.py create mode 100644 jwst/outlier_detection/outlier_detection_coron_step.py rename jwst/outlier_detection/{ifu.py => outlier_detection_ifu_step.py} (68%) create mode 100644 jwst/outlier_detection/outlier_detection_imaging_step.py create mode 100644 jwst/outlier_detection/outlier_detection_spec_step.py delete mode 100644 jwst/outlier_detection/outlier_detection_step.py rename jwst/outlier_detection/{tso.py => outlier_detection_tso_step.py} (58%) delete mode 100644 jwst/outlier_detection/spec.py delete mode 100644 jwst/pipeline/outlier_detection.cfg diff --git a/jwst/outlier_detection/__init__.py b/jwst/outlier_detection/__init__.py index c012c72cbf..42c6635247 100644 --- a/jwst/outlier_detection/__init__.py +++ b/jwst/outlier_detection/__init__.py @@ -1,3 +1,11 @@ -from .outlier_detection_step import OutlierDetectionStep +from .outlier_detection_coron_step import OutlierDetectionCoronStep +from .outlier_detection_ifu_step import OutlierDetectionIFUStep +from .outlier_detection_spec_step import OutlierDetectionSpecStep +from .outlier_detection_imaging_step import OutlierDetectionImagingStep +from .outlier_detection_tso_step import OutlierDetectionTSOStep -__all__ = ['OutlierDetectionStep'] +__all__ = ['OutlierDetectionCoronStep', + 'OutlierDetectionIFUStep', + 'OutlierDetectionSpecStep', + 'OutlierDetectionImagingStep', + 'OutlierDetectionTSOStep'] diff --git a/jwst/outlier_detection/_fileio.py b/jwst/outlier_detection/_fileio.py index 3cc2e960c5..d72c492425 100644 --- a/jwst/outlier_detection/_fileio.py +++ b/jwst/outlier_detection/_fileio.py @@ -100,7 +100,7 @@ def _make_blot_model(input_model, blot, blot_err): def _save_intermediate_output(model, suffix, make_output_path): """Save an intermediate output from outlier detection. - Ensure all intermediate outputs from OutlierDetectionStep have + Ensure all intermediate outputs from the steps have consistent file naming conventions. Parameters diff --git a/jwst/outlier_detection/coron.py b/jwst/outlier_detection/coron.py deleted file mode 100644 index 9a3bf9e984..0000000000 --- a/jwst/outlier_detection/coron.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Submodule for performing outlier detection on coronagraphy data. -""" - -import logging - -import numpy as np - -from stdatamodels.jwst import datamodels - -from jwst.resample.resample_utils import build_mask - -from .utils import create_cube_median, flag_model_crs -from ._fileio import save_median - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_model, - save_intermediate_results, - good_bits, - maskpt, - snr, - make_output_path, -): - """ - Flag outliers in coronography data. - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_model, datamodels.JwstDataModel): - input_model = datamodels.open(input_model) - - if not isinstance(input_model, datamodels.CubeModel): - raise TypeError(f"Input must be a CubeModel: {input_model}") - - # FIXME weight_type could now be used here. Similar to tso data coron - # data was previously losing var_rnoise due to the conversion from a cube - # to a ModelContainer (which makes the default ivm weight ignore var_rnoise). - # Now that it's handled as a cube we could use the var_rnoise. - input_model.wht = build_mask(input_model.dq, good_bits).astype(np.float32) - - # Perform median combination on set of drizzled mosaics - median_data = create_cube_median(input_model, maskpt) - - if save_intermediate_results: - # make a median model - median_model = datamodels.ImageModel(median_data) - median_model.update(input_model) - median_model.meta.wcs = input_model.meta.wcs - - save_median(median_model, make_output_path) - del median_model - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - flag_model_crs( - input_model, - median_data, - snr, - ) - return input_model diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py deleted file mode 100644 index 0c5d7d163a..0000000000 --- a/jwst/outlier_detection/imaging.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Submodule for performing outlier detection on imaging data. -""" - -import logging - -from jwst.datamodels import ModelLibrary -from jwst.resample import resample -from jwst.stpipe.utilities import record_step_status - -from .utils import (flag_model_crs, - flag_resampled_model_crs, - median_without_resampling, - median_with_resampling) - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_models, - save_intermediate_results, - good_bits, - maskpt, - snr1, - snr2, - scale1, - scale2, - backg, - resample_data, - weight_type, - pixfrac, - kernel, - fillval, - in_memory, - make_output_path, -): - """ - Flag outliers in imaging data. - - input_models is expected to be a ModelLibrary - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_models, ModelLibrary): - input_models = ModelLibrary(input_models, on_disk=not in_memory) - - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) - return input_models - - if resample_data: - resamp = resample.ResampleData( - input_models, - single=True, - blendheaders=False, - wht_type=weight_type, - pixfrac=pixfrac, - kernel=kernel, - fillval=fillval, - good_bits=good_bits, - ) - median_data, median_wcs = median_with_resampling(input_models, - resamp, - maskpt, - save_intermediate_results=save_intermediate_results, - make_output_path=make_output_path,) - else: - median_data, median_wcs = median_without_resampling(input_models, - maskpt, - weight_type, - good_bits, - save_intermediate_results=save_intermediate_results, - make_output_path=make_output_path,) - - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - with input_models: - for image in input_models: - if resample_data: - flag_resampled_model_crs(image, - median_data, - median_wcs, - snr1, - snr2, - scale1, - scale2, - backg, - save_blot=save_intermediate_results, - make_output_path=make_output_path) - else: - flag_model_crs(image, median_data, snr1) - input_models.shelve(image, modify=True) - - return input_models diff --git a/jwst/outlier_detection/outlier_detection_coron_step.py b/jwst/outlier_detection/outlier_detection_coron_step.py new file mode 100644 index 0000000000..1520709096 --- /dev/null +++ b/jwst/outlier_detection/outlier_detection_coron_step.py @@ -0,0 +1,81 @@ +""" +Submodule for performing outlier detection on coronagraphy data. +""" + +import logging + +import numpy as np + +from stdatamodels.jwst import datamodels +from jwst.stpipe import Step + +from jwst.resample.resample_utils import build_mask + +from .utils import create_cube_median, flag_model_crs, OutlierDetectionStepBase +from ._fileio import save_median + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +__all__ = ["OutlierDetectionCoronStep"] + + +class OutlierDetectionCoronStep(Step, OutlierDetectionStepBase): + """Flag outlier bad pixels and cosmic rays in DQ array of each input image. + Input images can be listed in an input association file or already opened + with a ModelContainer. DQ arrays are modified in place. + Parameters + ----------- + input_model : ~jwst.datamodels.CubeModel + CubeModel or filename pointing to a CubeModel + """ + + class_alias = "outlier_detection_coron" + + spec = """ + maskpt = float(default=0.7) + snr = float(default=5.0) + save_intermediate_results = boolean(default=False) + good_bits = string(default="~DO_NOT_USE") # DQ flags to allow + """ + + def process(self, input_model): + """Perform outlier detection processing on input data.""" + + # determine the asn_id (if not set by the pipeline) + asn_id = self._get_asn_id(input_model) + self.log.info(f"Outlier Detection asn_id: {asn_id}") + + if not isinstance(input_model, datamodels.JwstDataModel): + input_model = datamodels.open(input_model) + + if not isinstance(input_model, datamodels.CubeModel): + raise TypeError(f"Input must be a CubeModel: {input_model}") + + # FIXME weight_type could now be used here. Similar to tso data coron + # data was previously losing var_rnoise due to the conversion from a cube + # to a ModelContainer (which makes the default ivm weight ignore var_rnoise). + # Now that it's handled as a cube we could use the var_rnoise. + input_model.wht = build_mask(input_model.dq, self.good_bits).astype(np.float32) + + # Perform median combination on set of drizzled mosaics + median_data = create_cube_median(input_model, self.maskpt) + + if self.save_intermediate_results: + # make a median model + median_model = datamodels.ImageModel(median_data) + median_model.update(input_model) + median_model.meta.wcs = input_model.meta.wcs + + save_median(median_model, self.make_output_path) + del median_model + + # Perform outlier detection using statistical comparisons between + # each original input image and its blotted version of the median image + flag_model_crs( + input_model, + median_data, + self.snr, + ) + return self._set_status(input_model, True) diff --git a/jwst/outlier_detection/ifu.py b/jwst/outlier_detection/outlier_detection_ifu_step.py similarity index 68% rename from jwst/outlier_detection/ifu.py rename to jwst/outlier_detection/outlier_detection_ifu_step.py index 2c903ba16a..0a2d02fb43 100644 --- a/jwst/outlier_detection/ifu.py +++ b/jwst/outlier_detection/outlier_detection_ifu_step.py @@ -1,32 +1,3 @@ -""" -Submodule defined for performing outlier detection on IFU data. - -This is the controlling routine for the outlier detection process. -It loads and sets the various input data and parameters needed to flag -outliers. Pixel are flagged as outliers based on the MINIMUM difference -a pixel has with its neighbor across all the input cal files. - -Notes ------ -This routine performs the following operations:: - - 1. Extracts parameter settings from input ModelContainer and merges - them with any user-provided values - 2. Loop over cal files - a. read in science data - b. Store computed neighbor differences for all the pixels. - The neighbor pixel differences are defined by the dispersion axis. - For MIRI, with the dispersion axis along the y axis, the neighbors that are used to - to find the differences are to the left and right of each pixel being examined. - For NIRSpec, with the dispersion along the x axis, the neighbors that are used to - find the differences are above and below the pixel being examined. - 3. For each input file store the minimum of the pixel neighbor differences - 4. Comparing all the differences from all the input data find the minimum neighbor difference - 5. Normalize minimum difference to local median of difference array - 6. Select outliers by flagging those normalized minimum values > threshold_percent - 7. Updates input ImageModel DQ arrays with mask of detected outliers. -""" - import logging import numpy as np @@ -34,77 +5,110 @@ from jwst.datamodels import ModelContainer from jwst.lib.pipe_utils import match_nans_and_flags from jwst.stpipe.utilities import record_step_status +from jwst.stpipe import Step from stdatamodels.jwst import datamodels from stdatamodels.jwst.datamodels import dqflags from stcal.outlier_detection.utils import medfilt +from .utils import OutlierDetectionStepBase log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) -__all__ = ["detect_outliers"] +__all__ = ["OutlierDetectionIFUStep"] -def detect_outliers( - input_models, - save_intermediate_results, - kernel_size, - ifu_second_check, - threshold_percent, - make_output_path, -): +class OutlierDetectionIFUStep(Step, OutlierDetectionStepBase): + """Flag outlier bad pixels and cosmic rays in DQ array of each input image. + Input images can be listed in an input association file or already opened + with a ModelContainer. DQ arrays are modified in place. + Parameters + ----------- + input_models : asn file or ~jwst.datamodels.ModelContainer + Single filename association table, or a datamodels.ModelContainer. + Notes + ----- + This routine performs the following operations. + 1. Extracts parameter settings from input ModelContainer and merges + them with any user-provided values + 2. Loop over cal files + a. read in science data + b. Store computed neighbor differences for all the pixels. + The neighbor pixel differences are defined by the dispersion axis. + For MIRI, with the dispersion axis along the y axis, the neighbors that are used to + to find the differences are to the left and right of each pixel being examined. + For NIRSpec, with the dispersion along the x axis, the neighbors that are used to + find the differences are above and below the pixel being examined. + 3. For each input file store the minimum of the pixel neighbor differences + 4. Comparing all the differences from all the input data find the minimum neighbor difference + 5. Normalize minimum difference to local median of difference array + 6. select outliers by flagging those normailzed minimum values > threshold_percent + 7. Updates input ImageModel DQ arrays with mask of detected outliers. """ - Flag outliers in ifu data. - See `OutlierDetectionStep.spec` for documentation of these arguments. + class_alias = "outlier_detection_ifu" + + spec = """ + kernel_size = string(default='7 7') + threshold_percent = float(default=99.8) + ifu_second_check = boolean(default=False) + save_intermediate_results = boolean(default=False) """ - if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models) - - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) - return input_models - - sizex, sizey = [int(val) for val in kernel_size.split()] - kern_size = np.zeros(2, dtype=int) - kern_size[0] = sizex - kern_size[1] = sizey - - # check if kernel size is an odd value - if kern_size[0] % 2 == 0: - log.info("X kernel size is given as an even number. This value must be an odd number. Increasing number by 1") - kern_size[0] = kern_size[0] + 1 - log.info("New x kernel size is {}: ".format(kern_size[0])) - if kern_size[1] % 2 == 0: - log.info("Y kernel size is given as an even number. This value must be an odd number. Increasing number by 1") - kern_size[1] = kern_size[1] + 1 - log.info("New y kernel size is {}: ".format(kern_size[1])) - - (diffaxis, ny, nx) = _find_detector_parameters(input_models) - - nfiles = len(input_models) - detector = np.empty(nfiles, dtype=' 1) and (rolling_window_width < weighted_cube.shape[0]): - medians = compute_rolling_median(weighted_cube, weight_threshold, w=rolling_window_width) - - else: - medians = nanmedian3D(weighted_cube.data, overwrite_input=False) - # this is a 2-D array, need to repeat it into the time axis - # for consistent shape with rolling median case - medians = np.broadcast_to(medians, weighted_cube.shape) - - # Save median model if pars['save_intermediate_results'] is True - # this will be a CubeModel with rolling median values. - if save_intermediate_results: - median_model = dm.CubeModel(data=medians) # type: ignore[name-defined] - with dm.open(weighted_cube) as dm0: - median_model.update(dm0) - save_median(median_model, make_output_path) - del median_model - - # no need for blotting, resample is turned off for TSO - # go straight to outlier detection - log.info("Flagging outliers") - flag_model_crs( - input_model, - medians, - snr, - ) - return input_model + class_alias = "outlier_detection_tso" + + spec = """ + maskpt = float(default=0.7) + snr = float(default=5.0) + rolling_window_width = integer(default=25) + save_intermediate_results = boolean(default=False) + good_bits = string(default="~DO_NOT_USE") # DQ flags to allow + """ + + def process(self, input_model): + """Perform outlier detection processing on input data.""" + + # determine the asn_id (if not set by the pipeline) + asn_id = self._get_asn_id(input_model) + self.log.info(f"Outlier Detection asn_id: {asn_id}") + + if not isinstance(input_model, dm.JwstDataModel): + input_model = dm.open(input_model) + if isinstance(input_model, dm.ModelContainer): + raise TypeError("OutlierDetectionTSO does not support ModelContainer input.") + weighted_cube = weight_no_resample(input_model, self.good_bits) + + weight_threshold = compute_weight_threshold(weighted_cube.wht, self.maskpt) + + if (self.rolling_window_width > 1) and (self.rolling_window_width < weighted_cube.shape[0]): + medians = compute_rolling_median(weighted_cube, weight_threshold, w=self.rolling_window_width) + + else: + medians = nanmedian3D(weighted_cube.data, overwrite_input=False) + # this is a 2-D array, need to repeat it into the time axis + # for consistent shape with rolling median case + medians = np.broadcast_to(medians, weighted_cube.shape) + + # Save median model if pars['save_intermediate_results'] is True + # this will be a CubeModel with rolling median values. + if self.save_intermediate_results: + median_model = dm.CubeModel(data=medians) # type: ignore[name-defined] + with dm.open(weighted_cube) as dm0: + median_model.update(dm0) + save_median(median_model, self.make_output_path) + del median_model + + # no need for blotting, resample is turned off for TSO + # go straight to outlier detection + log.info("Flagging outliers") + flag_model_crs( + input_model, + medians, + self.snr, + ) + return self._set_status(input_model, True) def weight_no_resample(input_model, good_bits): diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py deleted file mode 100644 index 6467daadd4..0000000000 --- a/jwst/outlier_detection/spec.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Perform outlier detection on spectra.""" - -from jwst.datamodels import ModelContainer, ModelLibrary -from jwst.stpipe.utilities import record_step_status - -from ..resample import resample_spec -from .utils import (flag_crs_in_models, - flag_crs_in_models_with_resampling, - median_with_resampling, - median_without_resampling) - -import logging -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_models, - save_intermediate_results, - good_bits, - maskpt, - snr1, - snr2, - scale1, - scale2, - backg, - resample_data, - weight_type, - pixfrac, - kernel, - fillval, - in_memory, - make_output_path, -): - """Flag outliers in spec data. - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models) - - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) - return input_models - - # convert to library for resample - # for compatibility with image3 pipeline - library = ModelLibrary(input_models, on_disk=False) - - if resample_data is True: - # Start by creating resampled/mosaic images for - # each group of exposures - resamp = resample_spec.ResampleSpecData( - input_models, - single=True, - blendheaders=False, - wht_type=weight_type, - pixfrac=pixfrac, - kernel=kernel, - fillval=fillval, - good_bits=good_bits, - ) - - median_data, median_wcs, median_err = median_with_resampling( - library, - resamp, - maskpt, - save_intermediate_results=save_intermediate_results, - make_output_path=make_output_path, - return_error=True) - else: - median_data, median_wcs, median_err = median_without_resampling( - library, - maskpt, - weight_type, - good_bits, - save_intermediate_results=save_intermediate_results, - make_output_path=make_output_path, - return_error=True - ) - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - if resample_data: - flag_crs_in_models_with_resampling( - input_models, - median_data, - median_wcs, - snr1, - snr2, - scale1, - scale2, - backg, - median_err=median_err, - save_blot=save_intermediate_results, - make_output_path=make_output_path - ) - else: - flag_crs_in_models(input_models, - median_data, - snr1, - median_err=median_err) - return input_models diff --git a/jwst/outlier_detection/tests/test_algorithms.py b/jwst/outlier_detection/tests/test_algorithms.py index b3b9255bb4..05686acea0 100644 --- a/jwst/outlier_detection/tests/test_algorithms.py +++ b/jwst/outlier_detection/tests/test_algorithms.py @@ -1,6 +1,6 @@ import numpy as np -from jwst.outlier_detection.tso import moving_median_over_zeroth_axis +from jwst.outlier_detection.outlier_detection_tso_step import moving_median_over_zeroth_axis def test_rolling_median(): diff --git a/jwst/outlier_detection/tests/test_fileio.py b/jwst/outlier_detection/tests/test_fileio.py index 41ef2c1b2f..adb5df0227 100644 --- a/jwst/outlier_detection/tests/test_fileio.py +++ b/jwst/outlier_detection/tests/test_fileio.py @@ -1,7 +1,7 @@ import pytest from jwst.outlier_detection._fileio import _save_intermediate_output from jwst.datamodels import ImageModel # type: ignore[attr-defined] -from jwst.step import OutlierDetectionStep +from jwst.step import OutlierDetectionImagingStep import os import numpy as np from functools import partial @@ -21,7 +21,7 @@ def model(): @pytest.fixture(scope="module") def make_output_path(): - return OutlierDetectionStep().make_output_path + return OutlierDetectionImagingStep().make_output_path @pytest.mark.parametrize("asn_id", [None, ASN_ID]) @@ -37,4 +37,4 @@ def test_save(tmp_cwd, model, make_output_path, asn_id, slit_id): stem = model.meta.filename.split("_")[0] inputs = [val for val in [stem, asn_id, slit_id, SUFFIX] if val is not None] expected_filename = "_".join(inputs)+".fits" - assert os.path.isfile(expected_filename) \ No newline at end of file + assert os.path.isfile(expected_filename) diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 1699e6638e..ccabf134a2 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -9,8 +9,13 @@ from jwst.datamodels import ModelContainer, ModelLibrary from jwst.assign_wcs import AssignWcsStep -from jwst.outlier_detection import OutlierDetectionStep from jwst.outlier_detection.utils import _flag_resampled_model_crs +from jwst.outlier_detection import ( + OutlierDetectionCoronStep, + OutlierDetectionImagingStep, + OutlierDetectionTSOStep, + OutlierDetectionSpecStep, +) from jwst.resample.tests.test_resample_step import miri_rate_model from jwst.outlier_detection.utils import median_with_resampling, median_without_resampling from jwst.resample.resample import ResampleData @@ -267,7 +272,7 @@ def test_outlier_step_no_outliers(mirimage_three_sci, do_resample, tmp_cwd): container = ModelContainer(list(mirimage_three_sci)) container[0].var_rnoise[10, 10] = 1E9 pristine = ModelContainer([m.copy() for m in container]) - OutlierDetectionStep.call(container, in_memory=True, resample_data=do_resample) + OutlierDetectionImagingStep.call(container, in_memory=True, resample_data=do_resample) # Make sure nothing changed in SCI and DQ arrays for image, uncorrected in zip(pristine, container): @@ -286,7 +291,7 @@ def test_outlier_step_weak_cr_imaging(mirimage_three_sci, tmp_cwd): container.shelve(zeroth) # Verify that intermediate files are removed - OutlierDetectionStep.call(container) + OutlierDetectionImagingStep.call(container) i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits')) median_files = glob(os.path.join(tmp_cwd, '*median.fits')) assert len(i2d_files) == 0 @@ -296,7 +301,7 @@ def test_outlier_step_weak_cr_imaging(mirimage_three_sci, tmp_cwd): data_as_cube = list(container.map_function( lambda model, index: model.data.copy(), modify=False)) - result = OutlierDetectionStep.call( + result = OutlierDetectionImagingStep.call( container, save_results=True, save_intermediate_results=True ) @@ -348,7 +353,7 @@ def test_outlier_step_spec(tmp_cwd, tmp_path, resample, save_intermediate): container[0].data[209, 37] += 1 # Call outlier detection - result = OutlierDetectionStep.call( + result = OutlierDetectionSpecStep.call( container, resample_data=resample, output_dir=output_dir, save_results=True, save_intermediate_results=save_intermediate) @@ -369,7 +374,7 @@ def test_outlier_step_spec(tmp_cwd, tmp_path, resample, save_intermediate): expected_intermediate = 0 for dirname in [output_dir, tmp_cwd]: all_files = glob(os.path.join(dirname, '*.fits')) - result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + result_files = glob(os.path.join(dirname, '*outlierdetectionspecstep.fits')) i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) @@ -454,7 +459,7 @@ def test_outlier_step_on_disk(three_sci_as_asn, tmp_cwd): data_as_cube = list(container.map_function( lambda model, index: model.data.copy(), modify=False)) - result = OutlierDetectionStep.call( + result = OutlierDetectionImagingStep.call( container, save_results=True, save_intermediate_results=True, in_memory=False ) @@ -479,7 +484,7 @@ def test_outlier_step_on_disk(three_sci_as_asn, tmp_cwd): dirname = tmp_cwd all_files = glob(os.path.join(dirname, '*.fits')) input_files = glob(os.path.join(dirname, '*_cal.fits')) - result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + result_files = glob(os.path.join(dirname, '*outlierdetectionimagingstep.fits')) i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) @@ -518,7 +523,7 @@ def test_outlier_step_square_source_no_outliers(mirimage_three_sci, tmp_cwd): dq_as_cube.append(model.dq.copy()) container.shelve(model, modify=False) - result = OutlierDetectionStep.call(container, in_memory=True) + result = OutlierDetectionImagingStep.call(container, in_memory=True) # Make sure nothing changed in SCI and DQ arrays with container: @@ -549,7 +554,7 @@ def test_outlier_step_weak_cr_coron(we_three_sci, tmp_cwd): # coron3 will provide a CubeModel so convert the container to a cube cube = container_to_cube(container) - result = OutlierDetectionStep.call(cube) + result = OutlierDetectionCoronStep.call(cube) # Make sure nothing changed in SCI array except that # outliers are NaN @@ -588,7 +593,7 @@ def test_outlier_step_weak_cr_tso(mirimage_50_sci, rolling_window_width): cube = container_to_cube(im) - result = OutlierDetectionStep.call(cube, rolling_window_width=rolling_window_width) + result = OutlierDetectionTSOStep.call(cube, rolling_window_width=rolling_window_width) # Make sure nothing changed in SCI array except # that outliers are NaN diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index ac0d2b6192..86563a5d06 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -1,6 +1,7 @@ """Utilities for outlier detection methods.""" import copy +from abc import ABC, abstractmethod from functools import partial import numpy as np @@ -10,6 +11,8 @@ from stcal.outlier_detection.utils import compute_weight_threshold, gwcs_blot, flag_crs, flag_resampled_crs from stcal.outlier_detection.median import MedianComputer, nanmedian3D from stdatamodels.jwst import datamodels +from jwst.datamodels import ModelContainer, ModelLibrary +from jwst.stpipe.utilities import record_step_status from . import _fileio import logging @@ -21,6 +24,59 @@ OUTLIER = datamodels.dqflags.pixel['OUTLIER'] +class OutlierDetectionStepBase(ABC): + """Minimal base class holding common methods for outlier detection steps.""" + + @abstractmethod + def search_attr(self, attr, **kwargs): + pass + + @abstractmethod + def _make_output_path(self): + pass + + def _get_asn_id(self, input_models): + """Find association ID for any allowed input model type, + and update make_output_path such that the association ID + is included in intermediate and output file names.""" + # handle if input_models isn't open + if isinstance(input_models, (str, dict)): + input_models = datamodels.open(input_models, asn_n_members=1) + + # Setup output path naming if associations are involved. + try: + if isinstance(input_models, ModelLibrary): + asn_id = input_models.asn["asn_id"] + elif isinstance(input_models, ModelContainer): + asn_id = input_models.asn_table["asn_id"] + else: + asn_id = input_models.meta.asn_table.asn_id + except (AttributeError, KeyError): + asn_id = None + + if asn_id is None: + asn_id = self.search_attr('asn_id') + if asn_id is not None: + _make_output_path = self.search_attr( + '_make_output_path', parent_first=True + ) + + self._make_output_path = partial( + _make_output_path, + asn_id=asn_id + ) + self.log.info(f"Outlier Detection asn_id: {asn_id}") + return + + def _set_status(self, input_models, status): + # this might be called with the input which might be a filename or path + if not isinstance(input_models, (datamodels.JwstDataModel, ModelLibrary, ModelContainer)): + input_models = datamodels.open(input_models) + + record_step_status(input_models, "outlier_detection", status) + return input_models + + def create_cube_median(cube_model, maskpt): """Compute the median over a cube of data. diff --git a/jwst/pipeline/calwebb_coron3.py b/jwst/pipeline/calwebb_coron3.py index 54b2b093ec..341a1456ad 100644 --- a/jwst/pipeline/calwebb_coron3.py +++ b/jwst/pipeline/calwebb_coron3.py @@ -13,7 +13,7 @@ from ..coron import stack_refs_step from ..coron import align_refs_step from ..coron import klip_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_coron_step from ..resample import resample_step __all__ = ['Coron3Pipeline'] @@ -74,7 +74,7 @@ class Coron3Pipeline(Pipeline): 'stack_refs': stack_refs_step.StackRefsStep, 'align_refs': align_refs_step.AlignRefsStep, 'klip': klip_step.KlipStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection': outlier_detection_coron_step.OutlierDetectionCoronStep, 'resample': resample_step.ResampleStep } diff --git a/jwst/pipeline/calwebb_image3.py b/jwst/pipeline/calwebb_image3.py index 36d1a85680..a4ae68a387 100644 --- a/jwst/pipeline/calwebb_image3.py +++ b/jwst/pipeline/calwebb_image3.py @@ -10,7 +10,7 @@ from ..tweakreg import tweakreg_step from ..skymatch import skymatch_step from ..resample import resample_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_imaging_step from ..source_catalog import source_catalog_step __all__ = ['Image3Pipeline'] @@ -41,7 +41,7 @@ class Image3Pipeline(Pipeline): 'assign_mtwcs': assign_mtwcs_step.AssignMTWcsStep, 'tweakreg': tweakreg_step.TweakRegStep, 'skymatch': skymatch_step.SkyMatchStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection': outlier_detection_imaging_step.OutlierDetectionImagingStep, 'resample': resample_step.ResampleStep, 'source_catalog': source_catalog_step.SourceCatalogStep } diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 39b3b75f95..e0fca436ce 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -19,7 +19,7 @@ from ..extract_1d import extract_1d_step from ..master_background import master_background_step from ..mrs_imatch import mrs_imatch_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_spec_step, outlier_detection_ifu_step from ..resample import resample_spec_step from ..combine_1d import combine_1d_step from ..photom import photom_step @@ -42,7 +42,7 @@ class Spec3Pipeline(Pipeline): assign moving target wcs (assign_mtwcs) master background subtraction (master_background) MIRI MRS background matching (mrs_imatch) - outlier detection (outlier_detection) + outlier detection (outlier_detection_spec or outlier_detection_ifu) 2-D spectroscopic resampling (resample_spec) 3-D spectroscopic resampling (cube_build) 1-D spectral extraction (extract_1d) @@ -60,7 +60,8 @@ class Spec3Pipeline(Pipeline): 'assign_mtwcs': assign_mtwcs_step.AssignMTWcsStep, 'master_background': master_background_step.MasterBackgroundStep, 'mrs_imatch': mrs_imatch_step.MRSIMatchStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection_spec': outlier_detection_spec_step.OutlierDetectionSpecStep, + 'outlier_detection_ifu': outlier_detection_ifu_step.OutlierDetectionIFUStep, 'pixel_replace': pixel_replace_step.PixelReplaceStep, 'resample_spec': resample_spec_step.ResampleSpecStep, 'cube_build': cube_build_step.CubeBuildStep, @@ -85,8 +86,10 @@ def process(self, input): # Setup sub-step defaults self.master_background.suffix = 'mbsub' self.mrs_imatch.suffix = 'mrs_imatch' - self.outlier_detection.suffix = 'crf' - self.outlier_detection.save_results = self.save_results + self.outlier_detection_spec.suffix = 'crf' + self.outlier_detection_spec.save_results = self.save_results + self.outlier_detection_ifu.suffix = 'crf' + self.outlier_detection_ifu.save_results = self.save_results self.resample_spec.suffix = 's2d' self.resample_spec.save_results = self.save_results self.cube_build.suffix = 's3d' @@ -104,7 +107,8 @@ def process(self, input): # These steps save intermediate files, resulting in meta.filename # being modified. This can affect the filenames of subsequent # steps. - self.outlier_detection.save_model = invariant_filename(self.outlier_detection.save_model) + self.outlier_detection_spec.save_model = invariant_filename(self.outlier_detection_spec.save_model) + self.outlier_detection_ifu.save_model = invariant_filename(self.outlier_detection_ifu.save_model) self.pixel_replace.save_model = invariant_filename(self.pixel_replace.save_model) # Retrieve the inputs: @@ -227,10 +231,9 @@ def process(self, input): for cal_array in result: cal_array.meta.asn.table_name = op.basename(input_models.asn_table_name) if exptype in IFU_EXPTYPES: - self.outlier_detection.mode = 'ifu' + result = self.outlier_detection_ifu.run(result) else: - self.outlier_detection.mode = 'spec' - result = self.outlier_detection.run(result) + result = self.outlier_detection_spec.run(result) # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. diff --git a/jwst/pipeline/calwebb_tso3.py b/jwst/pipeline/calwebb_tso3.py index 58c87d753b..8e8116ffaf 100644 --- a/jwst/pipeline/calwebb_tso3.py +++ b/jwst/pipeline/calwebb_tso3.py @@ -8,7 +8,7 @@ from ..stpipe import Pipeline -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_tso_step from ..tso_photometry import tso_photometry_step from ..extract_1d import extract_1d_step from ..white_light import white_light_step @@ -40,8 +40,7 @@ class Tso3Pipeline(Pipeline): spec = "" # Define alias to steps - step_defs = {'outlier_detection': - outlier_detection_step.OutlierDetectionStep, + step_defs = {'outlier_detection': outlier_detection_tso_step.OutlierDetectionTSOStep, 'tso_photometry': tso_photometry_step.TSOPhotometryStep, 'pixel_replace': pixel_replace_step.PixelReplaceStep, 'extract_1d': extract_1d_step.Extract1dStep, @@ -77,7 +76,6 @@ def process(self, input): # This asn_id assignment is important as it allows outlier detection # to know the asn_id since that step receives the cube as input. self.asn_id = input_models.asn_table["asn_id"] - self.outlier_detection.mode = 'tso' # Input may consist of multiple exposures, so loop over each of them input_exptype = None diff --git a/jwst/pipeline/outlier_detection.cfg b/jwst/pipeline/outlier_detection.cfg deleted file mode 100644 index b7fd5ab069..0000000000 --- a/jwst/pipeline/outlier_detection.cfg +++ /dev/null @@ -1,2 +0,0 @@ -name = "outlier_detection" -class = "jwst.outlier_detection.OutlierDetectionStep" diff --git a/jwst/pipeline/outlier_detection_tso.cfg b/jwst/pipeline/outlier_detection_tso.cfg index b7fd5ab069..ca8e651335 100644 --- a/jwst/pipeline/outlier_detection_tso.cfg +++ b/jwst/pipeline/outlier_detection_tso.cfg @@ -1,2 +1,2 @@ name = "outlier_detection" -class = "jwst.outlier_detection.OutlierDetectionStep" +class = "jwst.outlier_detection.OutlierDetectionTSOStep" diff --git a/jwst/step.py b/jwst/step.py index fe051a7b62..c2d0e6b8c2 100644 --- a/jwst/step.py +++ b/jwst/step.py @@ -36,7 +36,11 @@ from .mrs_imatch.mrs_imatch_step import MRSIMatchStep from .msaflagopen.msaflagopen_step import MSAFlagOpenStep from .nsclean.nsclean_step import NSCleanStep -from .outlier_detection.outlier_detection_step import OutlierDetectionStep +from .outlier_detection.outlier_detection_coron_step import OutlierDetectionCoronStep +from .outlier_detection.outlier_detection_ifu_step import OutlierDetectionIFUStep +from .outlier_detection.outlier_detection_imaging_step import OutlierDetectionImagingStep +from .outlier_detection.outlier_detection_spec_step import OutlierDetectionSpecStep +from .outlier_detection.outlier_detection_tso_step import OutlierDetectionTSOStep from .pathloss.pathloss_step import PathLossStep from .persistence.persistence_step import PersistenceStep from .photom.photom_step import PhotomStep @@ -101,7 +105,11 @@ "MRSIMatchStep", "MSAFlagOpenStep", "NSCleanStep", - "OutlierDetectionStep", + "OutlierDetectionCoronStep", + "OutlierDetectionIFUStep", + "OutlierDetectionImagingStep", + "OutlierDetectionSpecStep", + "OutlierDetectionTSOStep", "PathLossStep", "PersistenceStep", "PhotomStep", diff --git a/jwst/stpipe/integration.py b/jwst/stpipe/integration.py index 7b787b47a7..1f9a288e24 100644 --- a/jwst/stpipe/integration.py +++ b/jwst/stpipe/integration.py @@ -68,7 +68,11 @@ def get_steps(): ("jwst.step.MRSIMatchStep", 'mrs_imatch', False), ("jwst.step.MSAFlagOpenStep", 'msa_flagging', False), ("jwst.step.NSCleanStep", 'nsclean', False), - ("jwst.step.OutlierDetectionStep", 'outlier_detection', False), + ("jwst.step.OutlierDetectionCoronStep", 'outlier_detection_coron', False), + ("jwst.step.OutlierDetectionIFUStep", 'outlier_detection_ifu', False), + ("jwst.step.OutlierDetectionImagingStep", 'outlier_detection_imaging', False), + ("jwst.step.OutlierDetectionSpecStep", 'outlier_detection_spec', False), + ("jwst.step.OutlierDetectionTSOStep", 'outlier_detection_tso', False), ("jwst.step.PathLossStep", 'pathloss', False), ("jwst.step.PersistenceStep", 'persistence', False), ("jwst.step.PhotomStep", 'photom', False),