Merge branch 'master' into calc-photom-pixarea

.github/workflows/ci.yml

@@ -71,6 +71,7 @@ jobs:
         - linux: py311-xdist
         - macos: py311-xdist
           pytest-results-summary: true
+        - linux: py311-stdevdeps-xdist
         - linux: py311-devdeps-xdist
         - linux: py311-xdist-cov
           coverage: codecov

CHANGES.rst

@@ -1,8 +1,69 @@
-1.12.2 (unreleased)
+1.12.4 (unreleased)
 ===================
 
 - 
 
+1.12.3 (2023-10-03)
+===================
+
+documentation
+-------------
+
+- Updated the ``calwebb_spec2`` pipeline docs to indicate that the ``pixel_replace`` step is
+  applied to NIRSpec IFU data, and made minor wording updates to the ``pixel_replace`` step
+  docs.  [#7990]
+
+- Updated the ``FilterOffset`` reference file docs to indicate that this reference file
+  type is also used for NIRISS and increase clarity on how the offsets are applied. [#7991]
+
+
+1.12.2 (2023-10-02)
+===================
+
+assign_wcs
+----------
+
+- Increase margin at edges of NIRSpec MOS slits to reduce edge effects in resampling. [#7976]
+
+associations
+------------
+
+- Prevent group candidates from generating level 3 TSO associations. [#7982]
+
+charge_migration
+----------------
+
+- Added tests to see if the data array is changed after runnung the step and
+  set the default signal_threshold to 25000. [#7895]
+
+extract_1d
+----------
+
+- Move MIRI MRS residual fringe correction later in the code after normalization by
+  pixel area of each wavelength plane, and apply residual fringe correction to surface
+  brightness and background vectors as well as flux. [#7980]
+
+flat_field
+----------
+
+- Update the ``combine_fast_slow`` function for NIRSpec spectroscopic flats 
+  to use 1D error values provided by F-flat reference files. [#7978]
+
+- For NIRSpec modes, set all DO_NOT_USE pixels to NaN after flat
+  correction. [#7979]
+
+set_telescope_pointing
+----------------------
+
+- Ensure that CRPIX1/2 are set to default values for Guiding modes that fail pointing determination [#7983]
+
+straylight
+----------
+
+- Adjust MIRI MRS straylight routine to ensure cross-artifact correction does not
+  get applied to pedestal dark signal. [#7980]
+
+
 1.12.1 (2023-09-26)
 ===================
 
@@ -119,7 +180,7 @@ datamodels
 - updated ``stdatamodels`` pin to ``>=1.8.0`` [#7854]
 
 documentation
-------------
+-------------
 
 - Fixed a reference to the ``ramp_fitting` module in the user documentation. [#7898]
 

CITATION.cff

@@ -75,7 +75,7 @@ authors:
   given-names: "Maria"
   orcid: "https://orcid.org/0000-0003-2314-3453"
 title: "JWST Calibration Pipeline"
-version: 1.12.1
+version: 1.12.3
 doi: 10.5281/zenodo.7038885
-date-released: 2023-09-26
+date-released: 2023-10-03
 url: "https://github.com/spacetelescope/jwst"

JenkinsfileRT

@@ -74,15 +74,11 @@ bc0.conda_ver = '22.11.1'
 bc0.conda_packages = [
     "python=${python_version}",
 ]
-bc0.pip_reqs_files = ['requirements-sdp.txt', 'requirements-max.txt']
+bc0.pip_reqs_files = ['requirements-dev-st.txt', 'requirements-max.txt', 'requirements-sdp.txt']
 bc0.build_cmds = [
     "pip install certifi -U --force-reinstall",
     "pip install -e .[test,sdp] --no-cache-dir",
     "pip install pytest-xdist",
-    // these latest-commit installs are overridden by `requirements-sdp.txt` if it is populated
-    "pip install git+https://github.com/spacetelescope/stdatamodels.git@master git+https://github.com/spacetelescope/stcal.git@main",
-    "pip install -r requirements-max.txt",
-    "pip install -r requirements-sdp.txt",
 ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"]
 bc0.test_cmds = [
     "pytest --cov-report=xml --cov=./ -r sxf --bigdata --slow \
@@ -107,5 +103,8 @@ bc1.name = 'macos-stable-deps'
 bc1.test_cmds = []
 bc1.test_configs = []
 
-utils.run([jobconfig, bc0, bc1])
+bc2 = utils.copy(bc0)
+bc2.pip_reqs_files = ['requirements-dev-st.txt']
+
+utils.run([jobconfig, bc0, bc1, bc2])
 }  // withCredentials

JenkinsfileRT_dev

@@ -74,12 +74,11 @@ bc0.conda_ver = '22.11.1'
 bc0.conda_packages = [
     "python=${python_version}",
 ]
-bc0.pip_reqs_files = ['requirements-dev.txt']
+bc0.pip_reqs_files = ['requirements-dev-st.txt', 'requirements-dev-thirdparty.txt']
 bc0.build_cmds = [
     "pip install certifi -U --force-reinstall",
     "pip install -e .[test,sdp] --no-cache-dir",
     "pip install pytest-xdist",
-    "pip install -r requirements-dev.txt",
 ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"]
 bc0.test_cmds = [
     "pytest -r sxf --bigdata --slow \

README.md

@@ -210,6 +210,8 @@ the specified context and less than the context for the next release.
 
 | jwst tag            | DMS build | SDP_VER  | CRDS_CONTEXT | Released   | Ops Install | Notes                                         |
 |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------|
+| 1.12.3              | B10.0rc4  | 2023.3.x | 1135         | 2023-10-03 |             | Fourth release candidate for B10.0            |
+| 1.12.2              | B10.0rc3  | 2023.3.x | 1135         | 2023-10-02 |             | Third release candidate for B10.0             |
 | 1.12.1              | B10.0rc2  | 2023.3.x | 1132         | 2023-09-26 |             | Second release candidate for B10.0            |
 | 1.12.0              | B10.0rc1  | 2023.3.x | 1130         | 2023-09-18 |             | First release candidate for B10.0             |
 | 1.11.4              | B9.3.1    | 2023.2.1 | 1107         | 2023-08-14 |             | Final release for B9.3.1 patch                |

docs/jwst/pipeline/calwebb_spec2.rst

@@ -77,7 +77,7 @@ TSO exposures. The instrument mode abbreviations used in the table are as follow
 +----------------------------------------------------------+-----+-----+-----+-----+-----+-----+-----------------+------+--------+-----+
 | :ref:`residual_fringe <residual_fringe_step>` \ :sup:`2` |     |     |     |     |     | |c| |                 |      |        |     |
 +----------------------------------------------------------+-----+-----+-----+-----+-----+-----+-----------------+------+--------+-----+
-| :ref:`pixel_replace <pixel_replace_step>` \ :sup:`2`     | |c| | |c| |     | |c| | |c| |     |                 | |c|  |  |c|   |     |
+| :ref:`pixel_replace <pixel_replace_step>` \ :sup:`2`     | |c| | |c| | |c| | |c| | |c| |     |                 | |c|  |  |c|   |     |
 +----------------------------------------------------------+-----+-----+-----+-----+-----+-----+-----------------+------+--------+-----+
 | :ref:`resample_spec <resample_step>`                     | |c| | |c| |     | |c| |     |     |                 |      |        |     |
 +----------------------------------------------------------+-----+-----+-----+-----+-----+-----+-----------------+------+--------+-----+
@@ -92,7 +92,8 @@ For NIRISS and NIRCam WFSS, as well as NIRCam TSO grism exposures, the order is
 flat_field, extract_2d, and srctype (no wavecorr).
 For all other modes the order is extract_2d, srctype, wavecorr, and flat_field.
 
-:sup:`2`\ By default this step is skipped in the ``calwebb_spec2`` pipeline.
+:sup:`2`\ By default this step is skipped in the ``calwebb_spec2`` pipeline, but
+is enabled for some modes via overrides provided in parameter reference files.
 
 :sup:`3`\ NIRISS SOSS can have multiple spectral orders contribute flux to one pixel; because
 photometric correction values depend on the spectral order assigned to a pixel, the order of

docs/jwst/pixel_replace/main.rst

@@ -4,15 +4,19 @@ Description
 :Classes: `jwst.pixel_replace.PixelReplaceStep`
 :Alias: pixel_replace
 
-During spectral extraction, pixels flagged as bad are ignored in the summation process.
+During 1-D spectral extraction (:ref:`extract_1d <extract_1d_step>` step),
+pixels flagged as bad are ignored in the summation process.
 If a bad pixel is part of the point-spread function (PSF) at a given wavelength, the
-absence of the signal in the flagged pixel will lead to a divot at that wavelength in
+absence of the signal in the flagged pixel will lead to a hollow space at that wavelength in
 the extracted spectrum.
 
 To avoid this defect in the 1-D spectrum, this step estimates the flux values of pixels
-flagged as ``DO_NOT_USE`` in 2-D extracted spectra, prior to rectification in the
-``resample_spec`` step. ``pixel_replace`` inserts these estimates into the data array,
-unsets the ``DO_NOT_USE`` flag and sets the ``FLUX_ESTIMATED`` flag for each pixel affected.
+flagged as ``DO_NOT_USE`` in 2-D extracted spectra using interpolation methods,
+prior to rectification in the :ref:`resample_spec <resample_step>` step.
+``pixel_replace`` inserts these estimates into the 2-D data array,
+unsets the ``DO_NOT_USE`` flag, and sets the ``FLUX_ESTIMATED`` flag for each affected pixel.
+
+This step is provided as a cosmetic feature and, for that reason, should be used with caution.
 
 Algorithms
 ==========
@@ -22,11 +26,11 @@ Adjacent Profile Approximation
 
 This is the default (and most extensively tested) algorithm for most spectroscopic modes.
 
-First, the input 2-d spectral cutout is scanned across the dispersion axis to determine
+First, the input 2-D spectral cutout is scanned across the dispersion axis to determine
 which cross-dispersion vectors (column or row, depending on dispersion direction) contain
 at least one flagged pixel. Next, for each affected vector, a median normalized profile is created.
 
-First, the adjacent arrays (the number of which is set by the step argument
+The adjacent arrays (the number of which is set by the step argument
 ``n_adjacent_cols``) are individually normalized. Next, each pixel in the profile is set to
 the median of the normalized values. This results in a median of normalized values filling the vector.
 
@@ -36,7 +40,7 @@ estimated from the scaled profile.
 Minimum Gradient Estimator
 --------------------------
 
-In the case of the MIRI MRS, NaN-valued pixels are partially compensated during the cube building process
+In the case of the MIRI MRS, NaN-valued pixels are partially compensated during the IFU cube building process
 using the overlap between detector pixels and output cube voxels.  The effects of NaN values are thus not
 as severe as for slit spectra, but can manifest as small dips in the extracted spectrum when a NaN value
 lands atop the peak of a spectral trace and cube building interpolates from lower-flux adjacent values.

docs/jwst/references_general/filteroffset_reffile.rst

@@ -20,6 +20,7 @@ Instrument Keywords
 ========== ================================================
 MIRI       INSTRUME, DETECTOR, EXP_TYPE, DATE-OBS, TIME-OBS
 NIRCam     INSTRUME, CHANNEL, MODULE, DATE-OBS, TIME-OBS
+NIRISS     INSTRUME, EXP_TYPE, DATE-OBS, TIME-OBS
 ========== ================================================
 
 .. include:: ../includes/standard_keywords.inc
@@ -30,8 +31,12 @@ The filteroffset reference file is an ASDF file that contains a list
 called ``filters``. Every item in the list contains one or more entries that
 are used as selectors, as well as the column and row offset values to be applied.
 For the MIRI instrument, there is one selector "filter", which is the name of
-the filter to which the offsets apply. For NIRCam, the selectors are "filter" and
-"pupil". The offsets, in pixels, are applied in the image science frame.
+the filter to which the offsets apply. For NIRCam and NIRISS, the selectors are "filter" and
+"pupil".
+
+The offsets, in units of pixels, are *added* to positions in images that use
+the reference filter/pupil, in order to align images to the reference filter/pupil
+frame.
 
 :filters:
     :filter: Filter name

jwst/assign_wcs/nirspec.py

@@ -635,7 +635,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position,
         slitlet_ids_unique.remove(-1)
 
     # add a margin to the slit y limits
-    margin = 0.05
+    margin = 0.5
 
     # Now lets look at each unique slitlet id
     for slitlet_id in slitlet_ids_unique:

jwst/assign_wcs/tests/test_nirspec.py

@@ -281,7 +281,7 @@ def test_msa_configuration_normal():
     dither_position = 1
     slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position,
                                               slit_y_range=[-.5, .5])
-    ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.15, 0.55, 4, 1, '1111x', '95065_1', '2122',
+    ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', '95065_1', '2122',
                              0.13, -0.31716078999999997, -0.18092266)
     _compare_slits(slitlet_info[0], ref_slit)
 
@@ -310,7 +310,7 @@ def test_msa_configuration_all_background():
     dither_position = 1
     slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position,
                                               slit_y_range=[-.5, .5])
-    ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -1.7, 1.7, 4, 0, '1x1', 'background_57', 'bkg_57',
+    ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 0, '1x1', 'background_57', 'bkg_57',
                              0, -0.5, -0.5)
     _compare_slits(slitlet_info[0], ref_slit)
 
@@ -326,7 +326,7 @@ def test_msa_configuration_row_skipped():
     dither_position = 1
     slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position,
                                               slit_y_range=[-.5, .5])
-    ref_slit = trmodels.Slit(58, 8646, 1, 251, 24, -2.85, 5.15, 4, 1, '11x1011', '95065_1', '2122',
+    ref_slit = trmodels.Slit(58, 8646, 1, 251, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122',
                              0.130, -0.31716078999999997, -0.18092266)
     _compare_slits(slitlet_info[0], ref_slit)
 
@@ -341,9 +341,9 @@ def test_msa_configuration_multiple_returns():
     dither_position = 1
     slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position,
                                               slit_y_range=[-.5, .5])
-    ref_slit1 = trmodels.Slit(59, 8651, 1, 256, 24, -2.85, 5.15, 4, 1, '11x1011', '95065_1', '2122',
+    ref_slit1 = trmodels.Slit(59, 8651, 1, 256, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122',
                               0.13000000000000003, -0.31716078999999997, -0.18092266)
-    ref_slit2 = trmodels.Slit(60, 11573, 1, 258, 32, -2.85, 4, 4, 2, '11x111', '95065_2', '172',
+    ref_slit2 = trmodels.Slit(60, 11573, 1, 258, 32, -3.3, 4.45, 4, 2, '11x111', '95065_2', '172',
                               0.70000000000000007, -0.31716078999999997, -0.18092266)
     _compare_slits(slitlet_info[0], ref_slit1)
     _compare_slits(slitlet_info[1], ref_slit2)

jwst/associations/lib/rules_level3.py

@@ -906,6 +906,14 @@ def __init__(self, *args, **kwargs):
             )
         ])
 
+        # Only valid if candidate type is 'observation'.
+        self.validity.update({
+            'is_type_observation': {
+                'validated': False,
+                'check': self._validate_candidates
+            }
+        })
+
         super(Asn_Lv3TSO, self).__init__(*args, **kwargs)
 
     def _init_hook(self, item):
@@ -914,6 +922,26 @@ def _init_hook(self, item):
         self.data['asn_type'] = 'tso3'
         super(Asn_Lv3TSO, self)._init_hook(item)
 
+    def _validate_candidates(self, member):
+        """Allow only observation-type candidates
+
+        Parameters
+        ----------
+        member : Member
+            Member being added. Ignored.
+
+        Returns
+        -------
+        True if candidate type is observation.
+        False otherwise.
+        """
+
+        # If a group candidate, reject.
+        if self.acid.type.lower() != 'observation':
+            return False
+
+        return True
+
 
 @RegistryMarker.rule
 class Asn_Lv3WFSCMB(AsnMixin_Science):

jwst/associations/tests/test_level3_basics.py

@@ -31,7 +31,6 @@ def test_meta():
 @pytest.mark.parametrize(
     'pool_file',
     [
-        'data/pool_005_spec_niriss.csv',
         'data/pool_006_spec_nirspec.csv',
         'data/pool_007_spec_miri.csv',
         'data/pool_010_spec_nirspec_lv2bkg.csv',

jwst/associations/tests/test_level3_spectrographic.py

@@ -18,7 +18,7 @@ class TestLevel3Spec(BasePoolRule):
     pools = [
         PoolParams(
             path=t_path('data/pool_005_spec_niriss.csv'),
-            n_asns=1,
+            n_asns=0,
             n_orphaned=0
         ),
         PoolParams(
@@ -28,7 +28,7 @@ class TestLevel3Spec(BasePoolRule):
         ),
         PoolParams(
             path=t_path('data/pool_007_spec_miri.csv'),
-            n_asns=2,
+            n_asns=1,
             n_orphaned=0
         ),
         PoolParams(

jwst/charge_migration/charge_migration_step.py

@@ -21,7 +21,7 @@ class ChargeMigrationStep(Step):
     class_alias = "charge_migration"
 
     spec = """
-        signal_threshold = float(default=30000)
+        signal_threshold = float(default=25000)
         skip = boolean(default=True)
     """