From c3e5c011dcea67fdecafaf29375fc366551f3184 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Tue, 11 Mar 2025 08:55:33 -0700 Subject: [PATCH 01/15] Add ampwise-averaged combined calibration frame CalibCombineTask output struct --- pipelines/LSSTCam/cpDark.yaml | 9 +++++++++ pipelines/_ingredients/cpPtcLSST.yaml | 1 + python/lsst/cp/pipe/cpCombine.py | 11 +++++++++++ 3 files changed, 21 insertions(+) diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/cpDark.yaml index eb23d22cb..23350fa2a 100644 --- a/pipelines/LSSTCam/cpDark.yaml +++ b/pipelines/LSSTCam/cpDark.yaml @@ -2,3 +2,12 @@ description: cp_pipe DARK calibration construction instrument: lsst.obs.lsst.LsstCam imports: - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkLSST.yaml +cpDarkCombine: + class: lsst.cp.pipe.CalibCombineTask + config: + # For initial LSSTCam dark calib, there are + # too few statistics, so we set all pixels + # of the dark the ampwise mean. Reset the + # output connection to this new calibration. + connections.outputData: "combinedDark" + connections.meanOutputData: "dark" diff --git a/pipelines/_ingredients/cpPtcLSST.yaml b/pipelines/_ingredients/cpPtcLSST.yaml index d85980829..2d8c9edc4 100644 --- a/pipelines/_ingredients/cpPtcLSST.yaml +++ b/pipelines/_ingredients/cpPtcLSST.yaml @@ -14,6 +14,7 @@ tasks: config.doCrosstalk = True config.crosstalk.doQuadraticCrosstalkCorrection = False config.doLinearize = True + config.doDeferredCharge = True config.doDefect = True config.doAmpOffset = True config.ampOffset.ampEdgeMaxOffset = 100000.0 diff --git a/python/lsst/cp/pipe/cpCombine.py b/python/lsst/cp/pipe/cpCombine.py index 47f63e0de..9f9b799f2 100644 --- a/python/lsst/cp/pipe/cpCombine.py +++ b/python/lsst/cp/pipe/cpCombine.py @@ -287,6 +287,9 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): ``outputData`` Final combined exposure generated from the inputs (`lsst.afw.image.Exposure`). + ``meanOutputData`` + Final combined exposure, with data set to ampwise mean. + (`lsst.afw.image.Exposure`) Raises ------ @@ -322,6 +325,7 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): # Create output exposure for combined data. combined = afwImage.MaskedImageF(width, height) combinedExp = afwImage.makeExposure(combined) + meanCombinedExp = combinedExp.clone() # Apply scaling: expScales = [] @@ -385,6 +389,12 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): self.interpolateNans(combined, maskPlane=self.config.noGoodPixelsMask) + + for amp in inputDetector: + ampDataBbox = amp.getBBox() + ampDataMean = np.mean(combinedExp.image[ampDataBbox].array) + meanCombinedExp.image[ampDataBbox].array = ampDataMean + if self.config.doVignette: polygon = inputExpHandles[0].get(component="validPolygon") maskVignettedRegion(combined, polygon=polygon, vignetteValue=0.0) @@ -406,6 +416,7 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): # Return return pipeBase.Struct( outputData=combinedExp, + meanOutputData=meanCombinedExp, ) def getDimensions(self, expHandleList): From b7d1204cc46c41ed72908a0a739513d27d3d6eda Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Sat, 15 Mar 2025 17:25:35 -0700 Subject: [PATCH 02/15] Change CTI solve errors to warnings if no exposures left after clip. --- python/lsst/cp/pipe/cpCombine.py | 1 - python/lsst/cp/pipe/cpCtiSolve.py | 27 ++++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/python/lsst/cp/pipe/cpCombine.py b/python/lsst/cp/pipe/cpCombine.py index 9f9b799f2..2a7c4241b 100644 --- a/python/lsst/cp/pipe/cpCombine.py +++ b/python/lsst/cp/pipe/cpCombine.py @@ -389,7 +389,6 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): self.interpolateNans(combined, maskPlane=self.config.noGoodPixelsMask) - for amp in inputDetector: ampDataBbox = amp.getBBox() ampDataMean = np.mean(combinedExp.image[ampDataBbox].array) diff --git a/python/lsst/cp/pipe/cpCtiSolve.py b/python/lsst/cp/pipe/cpCtiSolve.py index 9eeab4000..a62e84011 100644 --- a/python/lsst/cp/pipe/cpCtiSolve.py +++ b/python/lsst/cp/pipe/cpCtiSolve.py @@ -314,7 +314,17 @@ def solveLocalOffsets(self, inputMeasurements, calib, detector): Nskipped += 1 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") if len(signal) == 0 or len(data) == 0: - raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") + # All exposures excluded, set the calibration so that + # there is no correction + self.log.warning("All exposures brighter than config.maxImageMean are excluded. " + "Setting local offset drift scale to zero for amp {ampName}.") + # Arbitrary, will be overwritten by solveGlobalCti + calib.globalCti[ampName] = 10**(-6) + # Set to zero so that there is no correction + calib.driftScale[ampName] = 0.0 + # Arbitrary, unused if driftScale=0 + calib.decayTime[ampName] = 2.4 + continue signal = np.array(signal) data = np.array(data) @@ -433,7 +443,11 @@ def solveGlobalCti(self, inputMeasurements, calib, detector): Nskipped += 1 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") if len(signal) == 0 or len(data) == 0: - raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.") + # There are no exposures left, set globalCTI to 0 + self.log.warning("All exposures brighter than config.maxSignalForCti are excluded. " + f"Setting {ampName} global CTI to zero.") + calib.globalCti[ampName] = 0.0 + continue signal = np.array(signal) data = np.array(data) @@ -699,7 +713,14 @@ def findTraps(self, inputMeasurements, calib, detector): Nskipped += 1 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") if len(signal) == 0 or len(data) == 0: - raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") + # There are no exposures left, so set trap so that + # there is no correction + self.log.warning("All exposures brighter than config.maxImageMean are excluded. " + "Setting zero-sized serial trap for amp {ampName}.") + # Arbitrary trap with no size + trap = SerialTrap(0.0, 0.4, 1, 'linear', [1.0]) + calib.serialTraps[ampName] = trap + continue signal = np.array(signal) data = np.array(data) From 110e5d1bb0cccbbfd17948e5723ca7568c0762f7 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Sun, 16 Mar 2025 13:00:05 -0700 Subject: [PATCH 03/15] Change mean combined calib output to a config option --- pipelines/LSSTCam/cpDark.yaml | 3 +-- python/lsst/cp/pipe/cpCombine.py | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/cpDark.yaml index 23350fa2a..460ea0077 100644 --- a/pipelines/LSSTCam/cpDark.yaml +++ b/pipelines/LSSTCam/cpDark.yaml @@ -9,5 +9,4 @@ cpDarkCombine: # too few statistics, so we set all pixels # of the dark the ampwise mean. Reset the # output connection to this new calibration. - connections.outputData: "combinedDark" - connections.meanOutputData: "dark" + setAllPixelsToAmpMean: true diff --git a/python/lsst/cp/pipe/cpCombine.py b/python/lsst/cp/pipe/cpCombine.py index 2a7c4241b..07210da08 100644 --- a/python/lsst/cp/pipe/cpCombine.py +++ b/python/lsst/cp/pipe/cpCombine.py @@ -234,6 +234,11 @@ class CalibCombineConfig(pipeBase.PipelineTaskConfig, default=True, doc="Unset mask planes other than NO_DATA in output calibration?", ) + setAllPixelsToAmpMean = pexConfig.Field( + dtype=bool, + default=False, + doc="Return an image with every pixel equal to the amp-wise mean of the combined calib?", + ) stats = pexConfig.ConfigurableField( target=CalibStatsTask, doc="Background statistics configuration", @@ -287,9 +292,6 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): ``outputData`` Final combined exposure generated from the inputs (`lsst.afw.image.Exposure`). - ``meanOutputData`` - Final combined exposure, with data set to ampwise mean. - (`lsst.afw.image.Exposure`) Raises ------ @@ -389,11 +391,6 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): self.interpolateNans(combined, maskPlane=self.config.noGoodPixelsMask) - for amp in inputDetector: - ampDataBbox = amp.getBBox() - ampDataMean = np.mean(combinedExp.image[ampDataBbox].array) - meanCombinedExp.image[ampDataBbox].array = ampDataMean - if self.config.doVignette: polygon = inputExpHandles[0].get(component="validPolygon") maskVignettedRegion(combined, polygon=polygon, vignetteValue=0.0) @@ -412,10 +409,20 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): # Set QA headers self.calibStats(combinedExp, self.config.calibrationType) + # Optional: set every pixel in an amplifier to the amp-wise + # mean of the combined exposure. + if self.config.setAllPixelsToAmpMean: + # Possibly a good option if there are + # not enough input calibs, and the combined + # calib is noise-dominated + for amp in inputDetector: + ampDataBbox = amp.getBBox() + ampDataMean = np.mean(combinedExp.image[ampDataBbox].array) + combinedExp.image[ampDataBbox].array = ampDataMean + # Return return pipeBase.Struct( outputData=combinedExp, - meanOutputData=meanCombinedExp, ) def getDimensions(self, expHandleList): From e48d3b562d1214461431394ce7b1f9792fcb3357 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Sun, 16 Mar 2025 13:24:43 -0700 Subject: [PATCH 04/15] Move submit path to SCRATCH in bps templates --- bps/templates/bps_bfk.yaml | 1 + bps/templates/bps_bias.yaml | 1 + bps/templates/bps_biasBootstrap.yaml | 1 + bps/templates/bps_cti.yaml | 1 + bps/templates/bps_dark.yaml | 1 + bps/templates/bps_darkBootstrap.yaml | 1 + bps/templates/bps_defects.yaml | 1 + bps/templates/bps_flatBootstrap.yaml | 1 + bps/templates/bps_flat_g.yaml | 1 + bps/templates/bps_flat_i.yaml | 1 + bps/templates/bps_flat_r.yaml | 1 + bps/templates/bps_flat_u.yaml | 1 + bps/templates/bps_flat_y.yaml | 1 + bps/templates/bps_flat_z.yaml | 1 + bps/templates/bps_illuminationCorrection.yaml | 1 + bps/templates/bps_linearizer.yaml | 1 + bps/templates/bps_ptc.yaml | 1 + pipelines/LSSTCam/cpDark.yaml | 2 +- python/lsst/cp/pipe/ptc/cpPtcSolve.py | 2 +- 19 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index c1cd7940d..110a61054 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBfk.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_bfk" diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index 3f68021f5..30d9f082b 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBias.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_bias" diff --git a/bps/templates/bps_biasBootstrap.yaml b/bps/templates/bps_biasBootstrap.yaml index d84f2beba..4e2c862db 100644 --- a/bps/templates/bps_biasBootstrap.yaml +++ b/bps/templates/bps_biasBootstrap.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBiasBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} biasBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN" biasBootstrapRun: "{biasBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index ea34d7203..b9516fcdd 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpCti.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_cti" diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index e38e55fa0..950f70020 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDark.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_dark" diff --git a/bps/templates/bps_darkBootstrap.yaml b/bps/templates/bps_darkBootstrap.yaml index 8e0b5aba9..ffb0139cf 100644 --- a/bps/templates/bps_darkBootstrap.yaml +++ b/bps/templates/bps_darkBootstrap.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDarkBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} darkBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN" biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_defects.yaml b/bps/templates/bps_defects.yaml index c16572f78..10f103e03 100644 --- a/bps/templates/bps_defects.yaml +++ b/bps/templates/bps_defects.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDefects.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_flatBootstrap.yaml b/bps/templates/bps_flatBootstrap.yaml index 51260d9ef..5a73a51ac 100644 --- a/bps/templates/bps_flatBootstrap.yaml +++ b/bps/templates/bps_flatBootstrap.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlatBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} flatBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.$RERUN" biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index 609f870dd..c19a21ff2 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_g" diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index a30e6bf6c..1b83eb95f 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_i" diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index 01fa61506..1cd4b9a7e 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_r" diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index 45cbe81b6..b22bf2655 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_u" diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index 0cb96b2d1..ac44081b2 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_y" diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index da522174d..068cfa66e 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_z" diff --git a/bps/templates/bps_illuminationCorrection.yaml b/bps/templates/bps_illuminationCorrection.yaml index 404269462..92732b764 100644 --- a/bps/templates/bps_illuminationCorrection.yaml +++ b/bps/templates/bps_illuminationCorrection.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpIlluminationCorrection.y project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_illuminationCorrection" diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index d2dfc8989..c87c99909 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpLinearizer.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_linearizer" diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index e04ec51c0..df9466822 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -2,6 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml" project: "${TICKET}" campaign: "${TICKET}" +submitPath: $SCRATCH/submit/{outputRun} payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/cpDark.yaml index 460ea0077..61dd990f4 100644 --- a/pipelines/LSSTCam/cpDark.yaml +++ b/pipelines/LSSTCam/cpDark.yaml @@ -9,4 +9,4 @@ cpDarkCombine: # too few statistics, so we set all pixels # of the dark the ampwise mean. Reset the # output connection to this new calibration. - setAllPixelsToAmpMean: true + setAllPixelsToAmpMean: false diff --git a/python/lsst/cp/pipe/ptc/cpPtcSolve.py b/python/lsst/cp/pipe/ptc/cpPtcSolve.py index a253037b3..67c6465f7 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcSolve.py +++ b/python/lsst/cp/pipe/ptc/cpPtcSolve.py @@ -183,7 +183,7 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, "mean from the previous ``good`` point are allowed. If " "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " "otherwise adu.", - default=9_000., + default=1_000., ) scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field( dtype=bool, From 5b6b51de408f26c0554083e2725ca643a5a43a9c Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Tue, 18 Mar 2025 13:54:19 -0700 Subject: [PATCH 05/15] Add PTC outlier rejection based on maximum residual sigma vs signal gradient --- pipelines/LSSTCam/cpPtc.yaml | 3 +++ python/lsst/cp/pipe/cpCombine.py | 1 - python/lsst/cp/pipe/ptc/cpPtcSolve.py | 29 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/cpPtc.yaml index 613a8163f..05e1b50f6 100644 --- a/pipelines/LSSTCam/cpPtc.yaml +++ b/pipelines/LSSTCam/cpPtc.yaml @@ -10,3 +10,6 @@ tasks: doExtractPhotodiodeData: true auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] matchExposuresType: FLUX + cpPtcSolve: + config: + clipOnMaxAbsoluteResidualSigmaVsSignalGradient: false diff --git a/python/lsst/cp/pipe/cpCombine.py b/python/lsst/cp/pipe/cpCombine.py index 07210da08..7eb90bb9d 100644 --- a/python/lsst/cp/pipe/cpCombine.py +++ b/python/lsst/cp/pipe/cpCombine.py @@ -327,7 +327,6 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): # Create output exposure for combined data. combined = afwImage.MaskedImageF(width, height) combinedExp = afwImage.makeExposure(combined) - meanCombinedExp = combinedExp.clone() # Apply scaling: expScales = [] diff --git a/python/lsst/cp/pipe/ptc/cpPtcSolve.py b/python/lsst/cp/pipe/ptc/cpPtcSolve.py index 67c6465f7..4b7a5b57b 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcSolve.py +++ b/python/lsst/cp/pipe/ptc/cpPtcSolve.py @@ -193,6 +193,20 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, "to have units of electrons, otherwise adu.", default=True, ) + clipOnMaxAbsoluteResidualSigmaVsSignalGradient = pexConfig.Field( + dtype=bool, + doc="Calculate the gradient of the sigma residuals w.r.t. signal for each " + "point above 10k adu, and clip out points where the gradient of the sigma " + "residuals w.r.t. signal is less than maxAbsoluteResidualSigmaVsSignalSlope. " + "Only used if maxIterationsPtcOutliers > 0.", + default=False, + ) + maxAbsoluteResidualSigmaVsSignalGradient = pexConfig.Field( + dtype=float, + doc="The maximum absolute gradient of sigma residuals vs signal. Default based on " + "empirical testing from Run7 dense PTC for LSSTCam.", + default=0.0035, + ) minVarPivotSearch = pexConfig.Field( dtype=float, doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" @@ -1211,6 +1225,21 @@ def errFunc(p, x, y): & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) & mask ) + + sigmaResidGradient = np.gradient(sigResids, meanVecSorted) + + if self.config.clipOnMaxAbsoluteResidualSigmaVsSignalGradient: + # Caluclate the sigma residuals and the gradient w.r.t. + # signal. Clip out points beyond maximum value. Don't + # test this on points below 10000 adu. + aboveTenThousand = (meanVecSorted > 10000.0) + maxSigmaGradient = self.config.maxAbsoluteResidualSigmaVsSignalGradient + tmpMask = np.isfinite(sigResids[aboveTenThousand]) + tmpMask *= ( + np.abs(sigmaResidGradient[aboveTenThousand]) < maxSigmaGradient + ) + newMask[aboveTenThousand] *= tmpMask + # Demand at least 2 points to continue. if np.count_nonzero(newMask) < 2: msg = (f"SERIOUS: All points after outlier rejection are bad. " From b4f7ae31ab10205d272bbf3a96b43ebbe55d2ddf Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Tue, 18 Mar 2025 14:20:46 -0700 Subject: [PATCH 06/15] Lower sigmaCutPtcOutliers --- bps/templates/bps_ptc.yaml | 4 ++-- pipelines/LSSTCam/cpPtc.yaml | 3 --- pipelines/_ingredients/cpPtcLSST.yaml | 6 ++--- python/lsst/cp/pipe/ptc/cpPtcSolve.py | 32 ++------------------------- 4 files changed, 7 insertions(+), 38 deletions(-) diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index df9466822..c20c4716b 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -1,4 +1,4 @@ -pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml" +pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml#cpPtcSolve" project: "${TICKET}" campaign: "${TICKET}" @@ -8,5 +8,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ptcGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "u/abrought/LSSTCam/calib/DM-49175/test/ptcGen.20250309c,${CALIB_COLLECTIONS}" # ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC}" diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/cpPtc.yaml index 05e1b50f6..613a8163f 100644 --- a/pipelines/LSSTCam/cpPtc.yaml +++ b/pipelines/LSSTCam/cpPtc.yaml @@ -10,6 +10,3 @@ tasks: doExtractPhotodiodeData: true auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] matchExposuresType: FLUX - cpPtcSolve: - config: - clipOnMaxAbsoluteResidualSigmaVsSignalGradient: false diff --git a/pipelines/_ingredients/cpPtcLSST.yaml b/pipelines/_ingredients/cpPtcLSST.yaml index 2d8c9edc4..57a141179 100644 --- a/pipelines/_ingredients/cpPtcLSST.yaml +++ b/pipelines/_ingredients/cpPtcLSST.yaml @@ -37,9 +37,9 @@ tasks: config: connections.inputCovariances: "cpPtcPartial" connections.outputPtcDataset: "ptc" - ptcFitType: "FULLCOVARIANCE" - maximumRangeCovariancesAstier: 8 - maximumRangeCovariancesAstierFullCovFit: 8 + ptcFitType: "EXPAPPROXIMATION" + #maximumRangeCovariancesAstier: 8 + #maximumRangeCovariancesAstierFullCovFit: 8 doAmpOffsetGainRatioFixup: true subsets: # TODO DM-46448: This subset will no longer be necessary to run when the diff --git a/python/lsst/cp/pipe/ptc/cpPtcSolve.py b/python/lsst/cp/pipe/ptc/cpPtcSolve.py index 4b7a5b57b..b7768c1e8 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcSolve.py +++ b/python/lsst/cp/pipe/ptc/cpPtcSolve.py @@ -160,12 +160,12 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, sigmaCutPtcOutliers = pexConfig.Field( dtype=float, doc="Sigma cut for outlier rejection in PTC.", - default=5.0, + default=3.0, ) maxIterationsPtcOutliers = pexConfig.RangeField( dtype=int, doc="Maximum number of iterations for outlier rejection in PTC.", - default=2, + default=5, min=0 ) maxSignalInitialPtcOutlierFit = pexConfig.Field( @@ -191,22 +191,8 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, "approximate gain? If yes then " "maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit are assumed " "to have units of electrons, otherwise adu.", - default=True, - ) - clipOnMaxAbsoluteResidualSigmaVsSignalGradient = pexConfig.Field( - dtype=bool, - doc="Calculate the gradient of the sigma residuals w.r.t. signal for each " - "point above 10k adu, and clip out points where the gradient of the sigma " - "residuals w.r.t. signal is less than maxAbsoluteResidualSigmaVsSignalSlope. " - "Only used if maxIterationsPtcOutliers > 0.", default=False, ) - maxAbsoluteResidualSigmaVsSignalGradient = pexConfig.Field( - dtype=float, - doc="The maximum absolute gradient of sigma residuals vs signal. Default based on " - "empirical testing from Run7 dense PTC for LSSTCam.", - default=0.0035, - ) minVarPivotSearch = pexConfig.Field( dtype=float, doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" @@ -1226,20 +1212,6 @@ def errFunc(p, x, y): & mask ) - sigmaResidGradient = np.gradient(sigResids, meanVecSorted) - - if self.config.clipOnMaxAbsoluteResidualSigmaVsSignalGradient: - # Caluclate the sigma residuals and the gradient w.r.t. - # signal. Clip out points beyond maximum value. Don't - # test this on points below 10000 adu. - aboveTenThousand = (meanVecSorted > 10000.0) - maxSigmaGradient = self.config.maxAbsoluteResidualSigmaVsSignalGradient - tmpMask = np.isfinite(sigResids[aboveTenThousand]) - tmpMask *= ( - np.abs(sigmaResidGradient[aboveTenThousand]) < maxSigmaGradient - ) - newMask[aboveTenThousand] *= tmpMask - # Demand at least 2 points to continue. if np.count_nonzero(newMask) < 2: msg = (f"SERIOUS: All points after outlier rejection are bad. " From 85b7ff519dfc80034b0d001413dfbf4f2d918e80 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Wed, 19 Mar 2025 15:32:23 -0700 Subject: [PATCH 07/15] Add clustering by exposure for all isr task pipelines --- bps/templates/bps_bfk.yaml | 6 ++++++ bps/templates/bps_bias.yaml | 6 ++++++ bps/templates/bps_biasBootstrap.yaml | 6 ++++++ bps/templates/bps_cti.yaml | 6 ++++++ bps/templates/bps_dark.yaml | 6 ++++++ bps/templates/bps_darkBootstrap.yaml | 6 ++++++ bps/templates/bps_flatBootstrap.yaml | 6 ++++++ bps/templates/bps_flat_g.yaml | 6 ++++++ bps/templates/bps_flat_i.yaml | 6 ++++++ bps/templates/bps_flat_r.yaml | 6 ++++++ bps/templates/bps_flat_u.yaml | 6 ++++++ bps/templates/bps_flat_y.yaml | 6 ++++++ bps/templates/bps_flat_z.yaml | 6 ++++++ bps/templates/bps_illuminationCorrection.yaml | 6 ++++++ bps/templates/bps_linearizer.yaml | 6 ++++++ bps/templates/bps_ptc.yaml | 10 ++++++++-- pipelines/LSSTCam/cpDark.yaml | 8 -------- pipelines/_ingredients/cpPtcLSST.yaml | 7 +++---- python/lsst/cp/pipe/cpCombine.py | 16 ---------------- 19 files changed, 101 insertions(+), 30 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index 110a61054..0cdef0e88 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -1,5 +1,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBfk.yaml" +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpBfkIsr + dimensions: exposure + project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index 30d9f082b..6bc0cc67b 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -1,5 +1,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBias.yaml" +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpBiasIsr + dimensions: exposure + project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} diff --git a/bps/templates/bps_biasBootstrap.yaml b/bps/templates/bps_biasBootstrap.yaml index 4e2c862db..a6056bb47 100644 --- a/bps/templates/bps_biasBootstrap.yaml +++ b/bps/templates/bps_biasBootstrap.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpBiasBootstrapIsr + dimensions: exposure + biasBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN" biasBootstrapRun: "{biasBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index b9516fcdd..912cf9c0c 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpCtiIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_cti" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ctiGen.${RERUN}" diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index 950f70020..914e068bf 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpDarkIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_dark" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/darkGen.${RERUN}" diff --git a/bps/templates/bps_darkBootstrap.yaml b/bps/templates/bps_darkBootstrap.yaml index ffb0139cf..5014684eb 100644 --- a/bps/templates/bps_darkBootstrap.yaml +++ b/bps/templates/bps_darkBootstrap.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpDarkBootstrapIsr + dimensions: exposure + darkBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN" biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" darkBootstrapRun: "{darkBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_flatBootstrap.yaml b/bps/templates/bps_flatBootstrap.yaml index 5a73a51ac..5bb6e0ca6 100644 --- a/bps/templates/bps_flatBootstrap.yaml +++ b/bps/templates/bps_flatBootstrap.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatBootstrapIsr + dimensions: exposure + flatBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.$RERUN" biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index c19a21ff2..88cc1a0d3 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_g" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-g.${RERUN}" diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index 1b83eb95f..810364364 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_i" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-i.${RERUN}" diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index 1cd4b9a7e..7c29651d1 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_r" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-r.${RERUN}" diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index b22bf2655..037eb73c6 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_u" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-u.${RERUN}" diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index ac44081b2..c2ccd234c 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_y" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-y.${RERUN}" diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index 068cfa66e..6a761cc2b 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpFlatIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_z" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-z.${RERUN}" diff --git a/bps/templates/bps_illuminationCorrection.yaml b/bps/templates/bps_illuminationCorrection.yaml index 92732b764..20b59a6d5 100644 --- a/bps/templates/bps_illuminationCorrection.yaml +++ b/bps/templates/bps_illuminationCorrection.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpIlluminationCorrectionIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_illuminationCorrection" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/illuminationCorrectionGen.${RERUN}" diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index c87c99909..f5bca50a0 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -4,6 +4,12 @@ project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpLinearizerIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_linearizer" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/linearizerGen.${RERUN}" diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index c20c4716b..032ff68e2 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -1,12 +1,18 @@ -pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml#cpPtcSolve" +pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml" project: "${TICKET}" campaign: "${TICKET}" submitPath: $SCRATCH/submit/{outputRun} +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + isrExposure: + pipetasks: cpPtcIsr + dimensions: exposure + payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ptcGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "u/abrought/LSSTCam/calib/DM-49175/test/ptcGen.20250309c,${CALIB_COLLECTIONS}" # ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC}" diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/cpDark.yaml index 61dd990f4..eb23d22cb 100644 --- a/pipelines/LSSTCam/cpDark.yaml +++ b/pipelines/LSSTCam/cpDark.yaml @@ -2,11 +2,3 @@ description: cp_pipe DARK calibration construction instrument: lsst.obs.lsst.LsstCam imports: - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkLSST.yaml -cpDarkCombine: - class: lsst.cp.pipe.CalibCombineTask - config: - # For initial LSSTCam dark calib, there are - # too few statistics, so we set all pixels - # of the dark the ampwise mean. Reset the - # output connection to this new calibration. - setAllPixelsToAmpMean: false diff --git a/pipelines/_ingredients/cpPtcLSST.yaml b/pipelines/_ingredients/cpPtcLSST.yaml index 57a141179..d85980829 100644 --- a/pipelines/_ingredients/cpPtcLSST.yaml +++ b/pipelines/_ingredients/cpPtcLSST.yaml @@ -14,7 +14,6 @@ tasks: config.doCrosstalk = True config.crosstalk.doQuadraticCrosstalkCorrection = False config.doLinearize = True - config.doDeferredCharge = True config.doDefect = True config.doAmpOffset = True config.ampOffset.ampEdgeMaxOffset = 100000.0 @@ -37,9 +36,9 @@ tasks: config: connections.inputCovariances: "cpPtcPartial" connections.outputPtcDataset: "ptc" - ptcFitType: "EXPAPPROXIMATION" - #maximumRangeCovariancesAstier: 8 - #maximumRangeCovariancesAstierFullCovFit: 8 + ptcFitType: "FULLCOVARIANCE" + maximumRangeCovariancesAstier: 8 + maximumRangeCovariancesAstierFullCovFit: 8 doAmpOffsetGainRatioFixup: true subsets: # TODO DM-46448: This subset will no longer be necessary to run when the diff --git a/python/lsst/cp/pipe/cpCombine.py b/python/lsst/cp/pipe/cpCombine.py index 7eb90bb9d..47f63e0de 100644 --- a/python/lsst/cp/pipe/cpCombine.py +++ b/python/lsst/cp/pipe/cpCombine.py @@ -234,11 +234,6 @@ class CalibCombineConfig(pipeBase.PipelineTaskConfig, default=True, doc="Unset mask planes other than NO_DATA in output calibration?", ) - setAllPixelsToAmpMean = pexConfig.Field( - dtype=bool, - default=False, - doc="Return an image with every pixel equal to the amp-wise mean of the combined calib?", - ) stats = pexConfig.ConfigurableField( target=CalibStatsTask, doc="Background statistics configuration", @@ -408,17 +403,6 @@ def run(self, inputExpHandles, inputScales=None, inputDims=None): # Set QA headers self.calibStats(combinedExp, self.config.calibrationType) - # Optional: set every pixel in an amplifier to the amp-wise - # mean of the combined exposure. - if self.config.setAllPixelsToAmpMean: - # Possibly a good option if there are - # not enough input calibs, and the combined - # calib is noise-dominated - for amp in inputDetector: - ampDataBbox = amp.getBBox() - ampDataMean = np.mean(combinedExp.image[ampDataBbox].array) - combinedExp.image[ampDataBbox].array = ampDataMean - # Return return pipeBase.Struct( outputData=combinedExp, From b8a5d894f37d7b6be215a04bdabfbb658218061e Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Wed, 19 Mar 2025 17:38:20 -0700 Subject: [PATCH 08/15] Fix typo in cpBfkSolve pipeline step name --- pipelines/_ingredients/cpBfk.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/_ingredients/cpBfk.yaml b/pipelines/_ingredients/cpBfk.yaml index b3eda87fc..0cc45cca7 100644 --- a/pipelines/_ingredients/cpBfk.yaml +++ b/pipelines/_ingredients/cpBfk.yaml @@ -2,7 +2,7 @@ description: cp_pipe brighter-fatter kernel calibration construction. tasks: # TODO DM-46439: This can be renamed back to cpBfkSolve when repos # are cleaned up for the previous dimensionality error. - cpBfkSolveX: + cpBfkSolve: class: lsst.cp.pipe.BrighterFatterKernelSolveTask config: connections.inputPtc: ptc From 0466ad97a4aa67ec3635b0ce47de4275a1d5f81d Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Wed, 19 Mar 2025 19:20:30 -0700 Subject: [PATCH 09/15] Undo typo fix --- pipelines/LSSTCam/cpPtc.yaml | 5 +++++ pipelines/_ingredients/cpBfk.yaml | 2 +- python/lsst/cp/pipe/ptc/cpPtcSolve.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/cpPtc.yaml index 613a8163f..84a56b7f1 100644 --- a/pipelines/LSSTCam/cpPtc.yaml +++ b/pipelines/LSSTCam/cpPtc.yaml @@ -10,3 +10,8 @@ tasks: doExtractPhotodiodeData: true auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] matchExposuresType: FLUX + cpPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + sigmaCutPtcOutliers: 3.0 + maxDeltaInitialPtcOutlierFit: 1_000 diff --git a/pipelines/_ingredients/cpBfk.yaml b/pipelines/_ingredients/cpBfk.yaml index 0cc45cca7..b3eda87fc 100644 --- a/pipelines/_ingredients/cpBfk.yaml +++ b/pipelines/_ingredients/cpBfk.yaml @@ -2,7 +2,7 @@ description: cp_pipe brighter-fatter kernel calibration construction. tasks: # TODO DM-46439: This can be renamed back to cpBfkSolve when repos # are cleaned up for the previous dimensionality error. - cpBfkSolve: + cpBfkSolveX: class: lsst.cp.pipe.BrighterFatterKernelSolveTask config: connections.inputPtc: ptc diff --git a/python/lsst/cp/pipe/ptc/cpPtcSolve.py b/python/lsst/cp/pipe/ptc/cpPtcSolve.py index b7768c1e8..8048e3d6e 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcSolve.py +++ b/python/lsst/cp/pipe/ptc/cpPtcSolve.py @@ -160,12 +160,12 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, sigmaCutPtcOutliers = pexConfig.Field( dtype=float, doc="Sigma cut for outlier rejection in PTC.", - default=3.0, + default=5.0, ) maxIterationsPtcOutliers = pexConfig.RangeField( dtype=int, doc="Maximum number of iterations for outlier rejection in PTC.", - default=5, + default=2, min=0 ) maxSignalInitialPtcOutlierFit = pexConfig.Field( @@ -183,7 +183,7 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, "mean from the previous ``good`` point are allowed. If " "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " "otherwise adu.", - default=1_000., + default=9_000., ) scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field( dtype=bool, @@ -191,7 +191,7 @@ class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, "approximate gain? If yes then " "maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit are assumed " "to have units of electrons, otherwise adu.", - default=False, + default=True, ) minVarPivotSearch = pexConfig.Field( dtype=float, From 8025bb2e14c589db93feb44a9cfd25296cf4e2e8 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 20 Mar 2025 09:20:48 -0700 Subject: [PATCH 10/15] Add to user collections --- bps/templates/bps_bfk.yaml | 2 +- bps/templates/bps_bias.yaml | 2 +- bps/templates/bps_cti.yaml | 2 +- bps/templates/bps_dark.yaml | 2 +- bps/templates/bps_flat_g.yaml | 2 +- bps/templates/bps_flat_i.yaml | 2 +- bps/templates/bps_flat_r.yaml | 2 +- bps/templates/bps_flat_u.yaml | 2 +- bps/templates/bps_flat_y.yaml | 2 +- bps/templates/bps_flat_z.yaml | 2 +- bps/templates/bps_linearizer.yaml | 2 +- bps/templates/bps_ptc.yaml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index 0cdef0e88..25839f16d 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_bfk" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/bfkGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_BFK}" diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index 6bc0cc67b..f38395346 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_bias" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/biasGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_BIAS}" diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index 912cf9c0c..5048c9491 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_cti" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ctiGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_CTI}" diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index 914e068bf..7035ac3d6 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_dark" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/darkGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_DARK}" diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index 88cc1a0d3..175ea4911 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_g" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-g.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_g} and band='g'" diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index 810364364..946f4246f 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_i" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-i.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_i} and band='i'" diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index 7c29651d1..08e46a2e3 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_r" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-r.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_r} and band='r'" diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index 037eb73c6..ad3a7403e 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_u" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-u.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_u} and band='u'" diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index c2ccd234c..96dc20461 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_y" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-y.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_y} and band='y'" diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index 6a761cc2b..dc34b0fdd 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_z" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-z.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_z} and band='z'" diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index f5bca50a0..684e9ac21 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_linearizer" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/linearizerGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_LINEARIZER}" diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index 032ff68e2..b208b37cd 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ptcGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC}" From d05be021653d4a099150e876c96b43ce9c032f1c Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 20 Mar 2025 09:50:21 -0700 Subject: [PATCH 11/15] Adjust configs for LSSTCam dense PTC --- pipelines/LSSTCam/cpBfk.yaml | 7 +++++++ pipelines/LSSTCam/cpLinearizer.yaml | 7 +++++++ pipelines/LSSTCam/cpPtc.yaml | 2 ++ 3 files changed, 16 insertions(+) diff --git a/pipelines/LSSTCam/cpBfk.yaml b/pipelines/LSSTCam/cpBfk.yaml index 52045d218..60da2b4c8 100644 --- a/pipelines/LSSTCam/cpBfk.yaml +++ b/pipelines/LSSTCam/cpBfk.yaml @@ -10,3 +10,10 @@ tasks: doExtractPhotodiodeData: true auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] matchExposuresType: FLUX + cpBfkPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + sigmaCutPtcOutliers: 3.0 + maxDeltaInitialPtcOutlierFit: 1_000 + maxSignalInitialPtcOutlierFit: 30_000 + scaleMaxSignalInitialPtcOutlierFit: False diff --git a/pipelines/LSSTCam/cpLinearizer.yaml b/pipelines/LSSTCam/cpLinearizer.yaml index 488a9654d..48620174d 100644 --- a/pipelines/LSSTCam/cpLinearizer.yaml +++ b/pipelines/LSSTCam/cpLinearizer.yaml @@ -10,6 +10,13 @@ tasks: doExtractPhotodiodeData: true auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] matchExposuresType: FLUX + cpLinearizerPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + sigmaCutPtcOutliers: 3.0 + maxDeltaInitialPtcOutlierFit: 1_000 + maxSignalInitialPtcOutlierFit: 30_000 + scaleMaxSignalInitialPtcOutlierFit: False cpLinearizerSolve: class: lsst.cp.pipe.LinearitySolveTask config: diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/cpPtc.yaml index 84a56b7f1..ed83a62f7 100644 --- a/pipelines/LSSTCam/cpPtc.yaml +++ b/pipelines/LSSTCam/cpPtc.yaml @@ -15,3 +15,5 @@ tasks: config: sigmaCutPtcOutliers: 3.0 maxDeltaInitialPtcOutlierFit: 1_000 + maxSignalInitialPtcOutlierFit: 30_000 + scaleMaxSignalInitialPtcOutlierFit: False From c5ff93adb0991d8ab94832c9fd9a671f567b9224 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 20 Mar 2025 10:11:44 -0700 Subject: [PATCH 12/15] Change f-strings to %-style --- python/lsst/cp/pipe/cpCtiSolve.py | 34 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/python/lsst/cp/pipe/cpCtiSolve.py b/python/lsst/cp/pipe/cpCtiSolve.py index a62e84011..3bad9319b 100644 --- a/python/lsst/cp/pipe/cpCtiSolve.py +++ b/python/lsst/cp/pipe/cpCtiSolve.py @@ -312,7 +312,8 @@ def solveLocalOffsets(self, inputMeasurements, calib, detector): data.append(exposureDict[ampName]['SERIAL_OVERSCAN_VALUES'][start:stop+1]) else: Nskipped += 1 - self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") + self.log.info("Skipped %d exposures brighter than %f.", + Nskipped, self.config.maxImageMean) if len(signal) == 0 or len(data) == 0: # All exposures excluded, set the calibration so that # there is no correction @@ -444,8 +445,9 @@ def solveGlobalCti(self, inputMeasurements, calib, detector): self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") if len(signal) == 0 or len(data) == 0: # There are no exposures left, set globalCTI to 0 - self.log.warning("All exposures brighter than config.maxSignalForCti are excluded. " - f"Setting {ampName} global CTI to zero.") + self.log.warning("All exposures brighter than config.maxSignalForCti=%f " + "are excluded for %s. Setting global CTI to zero.", + self.config.maxSignalForCti, ampName) calib.globalCti[ampName] = 0.0 continue @@ -711,12 +713,13 @@ def findTraps(self, inputMeasurements, calib, detector): new_signal.append(exposureDict[ampName]['LAST_COLUMN_MEAN']) else: Nskipped += 1 - self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") + self.log.info("Skipped %d exposures brighter than %f.", + Nskipped, self.config.maxImageMean) if len(signal) == 0 or len(data) == 0: # There are no exposures left, so set trap so that # there is no correction self.log.warning("All exposures brighter than config.maxImageMean are excluded. " - "Setting zero-sized serial trap for amp {ampName}.") + "Setting zero-sized serial trap for amp %s.", ampName) # Arbitrary trap with no size trap = SerialTrap(0.0, 0.4, 1, 'linear', [1.0]) calib.serialTraps[ampName] = trap @@ -842,7 +845,8 @@ def calcEper(self, mode, inputMeasurements, amp): # Number of parallel shifts = nRows nShifts = amp.getRawDataBBox().getHeight() else: - raise RuntimeError(f"{mode} is not a known orientation for the EPER calculation.") + raise RuntimeError("%s is not a known orientation for the EPER " + "calculation.", mode) # The signal is the mean intensity of each input, and the # data are the overscan columns to fit. For detectors @@ -910,14 +914,14 @@ def calcCtiTurnoff(self, signalVec, dataVec, ctiRange, amp): # Check for remaining data points if dataVec.size == 0: self.log.warning("No data points after cti range cut to compute turnoff " - f"for amplifier {amp.getName()}. Setting turnoff point " - "to 0 electrons.") + "for amplifier %s. Setting turnoff point to 0 electrons.", + amp.getName()) return 0.0, 0.0 if dataVec.size < 2: self.log.warning("Insufficient data points after cti range cut to compute turnoff " - f"for amplifier {amp.getName()}. Setting turnoff point " - "to the maximum signal value.") + "for amplifier %s. Setting turnoff point to the maximum signal " + "value.", amp.getName()) return signalVec[-1], signalVec[-1] # Detrend the data @@ -946,8 +950,8 @@ def calcCtiTurnoff(self, signalVec, dataVec, ctiRange, amp): if cleanDataVec.size == 0: self.log.warning("No data points after sigma clipping to compute turnoff " - f" for amplifier {amp.getName()}. Setting turnoff point " - "to 0 electrons.") + "for amplifier %s. Setting turnoff point to 0 electrons.", + amp.getName()) return 0.0, 0.0 # Get the index of the highest true value in @@ -957,10 +961,10 @@ def calcCtiTurnoff(self, signalVec, dataVec, ctiRange, amp): if cleanDataVec[good][-1] in ctiRange or turnoffIdx in [0, len(signalVec)-1]: self.log.warning("Turnoff point is at the edge of the allowed range for " - f"amplifier {amp.getName()}.") + "amplifier %s.", amp.getName()) - self.log.info(f"Amp {amp.getName()}: There are {len(cleanDataVec[good])}/{len(dataVec)} data points " - f"left to determine turnoff point.") + self.log.info("Amp %s: There are %d/%d data points left to determine turnoff point.", + amp.getName(), len(cleanDataVec[good]), len(dataVec)) # Compute the sampling error as one half the # difference between the previous and next point. From 6206bb9a72c2efef1f5d4bb24388f9ff56bf014e Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 20 Mar 2025 10:41:36 -0700 Subject: [PATCH 13/15] Remove TAG collection from user calibs --- bps/templates/bps_bfk.yaml | 2 +- bps/templates/bps_bias.yaml | 2 +- bps/templates/bps_cti.yaml | 2 +- bps/templates/bps_dark.yaml | 2 +- bps/templates/bps_flat_g.yaml | 2 +- bps/templates/bps_flat_i.yaml | 2 +- bps/templates/bps_flat_r.yaml | 2 +- bps/templates/bps_flat_u.yaml | 2 +- bps/templates/bps_flat_y.yaml | 2 +- bps/templates/bps_flat_z.yaml | 2 +- bps/templates/bps_linearizer.yaml | 2 +- bps/templates/bps_ptc.yaml | 2 +- python/lsst/cp/pipe/ptc/cpPtcSolve.py | 1 - 13 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index 25839f16d..0cdef0e88 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_bfk" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/bfkGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_BFK}" diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index f38395346..6bc0cc67b 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_bias" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/biasGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_BIAS}" diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index 5048c9491..912cf9c0c 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_cti" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ctiGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_CTI}" diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index 7035ac3d6..914e068bf 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_dark" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/darkGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_DARK}" diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index 175ea4911..88cc1a0d3 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_g" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-g.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_g} and band='g'" diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index 946f4246f..810364364 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_i" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-i.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_i} and band='i'" diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index 08e46a2e3..7c29651d1 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_r" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-r.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_r} and band='r'" diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index ad3a7403e..037eb73c6 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_u" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-u.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_u} and band='u'" diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index 96dc20461..c2ccd234c 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_y" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-y.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_y} and band='y'" diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index dc34b0fdd..6a761cc2b 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_flat_z" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/flatGen-z.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_FLAT_z} and band='z'" diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index 684e9ac21..f5bca50a0 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_linearizer" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/linearizerGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC_LINEARIZER}" diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index b208b37cd..032ff68e2 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -14,5 +14,5 @@ payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ptcGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC}" diff --git a/python/lsst/cp/pipe/ptc/cpPtcSolve.py b/python/lsst/cp/pipe/ptc/cpPtcSolve.py index 8048e3d6e..a253037b3 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcSolve.py +++ b/python/lsst/cp/pipe/ptc/cpPtcSolve.py @@ -1211,7 +1211,6 @@ def errFunc(p, x, y): & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) & mask ) - # Demand at least 2 points to continue. if np.count_nonzero(newMask) < 2: msg = (f"SERIOUS: All points after outlier rejection are bad. " From 4c6cf560f4ae89921324e392cc02b2143c98b106 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 20 Mar 2025 11:56:05 -0700 Subject: [PATCH 14/15] Fix typos move old variable names to proper format --- bps/templates/bps_bfk.yaml | 7 +++-- bps/templates/bps_bias.yaml | 4 +-- bps/templates/bps_biasBootstrap.yaml | 4 +-- bps/templates/bps_cti.yaml | 4 +-- bps/templates/bps_dark.yaml | 4 +-- bps/templates/bps_darkBootstrap.yaml | 4 +-- bps/templates/bps_defects.yaml | 2 +- bps/templates/bps_flatBootstrap.yaml | 4 +-- bps/templates/bps_flat_g.yaml | 4 +-- bps/templates/bps_flat_i.yaml | 4 +-- bps/templates/bps_flat_r.yaml | 4 +-- bps/templates/bps_flat_u.yaml | 4 +-- bps/templates/bps_flat_y.yaml | 4 +-- bps/templates/bps_flat_z.yaml | 4 +-- bps/templates/bps_illuminationCorrection.yaml | 4 +-- bps/templates/bps_linearizer.yaml | 7 +++-- bps/templates/bps_ptc.yaml | 9 ++++-- python/lsst/cp/pipe/cpBfk.py | 28 ++++++++++++++----- python/lsst/cp/pipe/cpCtiSolve.py | 24 ++++++++-------- 19 files changed, 76 insertions(+), 53 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index 0cdef0e88..72cefecd8 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -2,13 +2,16 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBfk.yaml" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpBfkIsrByExposure: pipetasks: cpBfkIsr dimensions: exposure + cpBfkPtcExtractPairByExposure: + pipetasks: cpBfkPtcExtractPair + dimensions: exposure project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" payload: payloadName: "${INSTRUMENT}_${TICKET}_bfk" diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index 6bc0cc67b..cc1f21e5e 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -2,13 +2,13 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBias.yaml" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpBiasIsrByExposure: pipetasks: cpBiasIsr dimensions: exposure project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" payload: payloadName: "${INSTRUMENT}_${TICKET}_bias" diff --git a/bps/templates/bps_biasBootstrap.yaml b/bps/templates/bps_biasBootstrap.yaml index a6056bb47..1ec612af5 100644 --- a/bps/templates/bps_biasBootstrap.yaml +++ b/bps/templates/bps_biasBootstrap.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBiasBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpBiasBootstrapIsrByExposure: pipetasks: cpBiasBootstrapIsr dimensions: exposure diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index 912cf9c0c..ee3e3dfde 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpCti.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpCtiIsrByExposure: pipetasks: cpCtiIsr dimensions: exposure diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index 914e068bf..ff17b39d3 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDark.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpDarkIsrByExposure: pipetasks: cpDarkIsr dimensions: exposure diff --git a/bps/templates/bps_darkBootstrap.yaml b/bps/templates/bps_darkBootstrap.yaml index 5014684eb..b0e0c26f7 100644 --- a/bps/templates/bps_darkBootstrap.yaml +++ b/bps/templates/bps_darkBootstrap.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDarkBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpDarkBootstrapIsrByExposure: pipetasks: cpDarkBootstrapIsr dimensions: exposure diff --git a/bps/templates/bps_defects.yaml b/bps/templates/bps_defects.yaml index 10f103e03..ac1b8796c 100644 --- a/bps/templates/bps_defects.yaml +++ b/bps/templates/bps_defects.yaml @@ -2,7 +2,7 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDefects.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" diff --git a/bps/templates/bps_flatBootstrap.yaml b/bps/templates/bps_flatBootstrap.yaml index 5bb6e0ca6..f50cf4702 100644 --- a/bps/templates/bps_flatBootstrap.yaml +++ b/bps/templates/bps_flatBootstrap.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlatBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatBootstrapIsrByExposure: pipetasks: cpFlatBootstrapIsr dimensions: exposure diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index 88cc1a0d3..c00e714de 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index 810364364..7676997a2 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index 7c29651d1..a322900ef 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index 037eb73c6..7b03af9b8 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index c2ccd234c..c65c0bcab 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index 6a761cc2b..20a54b54a 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpFlatIsrByExposure: pipetasks: cpFlatIsr dimensions: exposure diff --git a/bps/templates/bps_illuminationCorrection.yaml b/bps/templates/bps_illuminationCorrection.yaml index 20b59a6d5..40bb743a3 100644 --- a/bps/templates/bps_illuminationCorrection.yaml +++ b/bps/templates/bps_illuminationCorrection.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpIlluminationCorrection.y project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpIlluminationCorrectionIsrByExposure: pipetasks: cpIlluminationCorrectionIsr dimensions: exposure diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index f5bca50a0..a9f2d5898 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -2,13 +2,16 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpLinearizer.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpLinearizerIsrByExposure: pipetasks: cpLinearizerIsr dimensions: exposure + cpLinearizerPtcExtractPairByExposure: + pipetasks: cpLinearizerPtcExtractPair + dimensions: exposure payload: payloadName: "${INSTRUMENT}_${TICKET}_linearizer" diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index 032ff68e2..d7ea6fbd2 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -2,17 +2,20 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: $SCRATCH/submit/{outputRun} +submitPath: "$SCRATCH/submit/{outputRun}" clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: - isrExposure: + cpPtcIsrByExposure: pipetasks: cpPtcIsr dimensions: exposure + cpPtcExtractPairByExposure: + pipetasks: cpPtcExtractPair + dimensions: exposure payload: payloadName: "${INSTRUMENT}_${TICKET}_ptc" output: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET}/${TAG}/ptcGen.${RERUN}" butlerConfig: "${REPO}" - inCollection: ${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" + inCollection: "${USER_CALIB_PREFIX}${INSTRUMENT}/calib/${TICKET},${RAW_COLLECTION},${CALIB_COLLECTIONS}" dataQuery: "${SELECTION_PTC}" diff --git a/python/lsst/cp/pipe/cpBfk.py b/python/lsst/cp/pipe/cpBfk.py index 41c9b022f..de7d70b0f 100644 --- a/python/lsst/cp/pipe/cpBfk.py +++ b/python/lsst/cp/pipe/cpBfk.py @@ -173,6 +173,11 @@ class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig, "be within +/- nSigmaTolForValidEdge of zero.", default=5.0, ) + fractionK00ForKernelMax = pexConfig.Field( + dtype=float, + doc="All kernel values must be less than value*abs(K_00) to be valid.", + default=0.025, + ) useBfkPtc = pexConfig.Field( dtype=bool, @@ -440,34 +445,43 @@ def run(self, inputPtc, dummy, camera, inputDims): # Get a mask for a 3px picture frame valid = True if kernelSize >= 7: + # Last 1px + edge = (xv == 0) + edge += (xv == kernelSize - 1) + edge += (yv == 0) + edge += (yv == kernelSize - 1) + + # Last 3 px edges = (xv < 3) edges += (xv > kernelSize - 3 - 1) edges += (yv < 3) edges += (yv > kernelSize - 3 - 1) + kernelEdge = bfk.ampKernels[ampName][edge].ravel() kernelEdges = bfk.ampKernels[ampName][edges].ravel() kernelEdgeStd = np.std(kernelEdges) threshold = self.config.nSigmaTolForValidEdge*kernelEdgeStd - # Edges should converge to zero + # Edge should converge to zero valid = np.all(np.isclose( - kernelEdges, 0, + kernelEdge, 0, rtol=threshold, )) if not valid: self.log.warning("%s: the kernel edges did not converge to 0 within " "+/- %s sigma.", ampName, threshold) - # Kernel should be negative - valid *= np.all(bfk.ampKernels[ampName][~edges] <= threshold) - if not np.all(bfk.ampKernels[ampName][~edges] <= threshold): - self.log.warning("%s: the kernel is not negative", ampName) - # The center should be the minimum valid *= np.all(bfk.ampKernels[ampName] >= kernelCenterValue) if not np.all(bfk.ampKernels[ampName] >= kernelCenterValue): self.log.warning("%s: the kernel center is not the absolute minimum.", ampName) + + # Kernel should be negative + negativityThreshold = self.config.fractionK00ForKernelMax*np.abs(kernelCenterValue) + valid *= np.all(bfk.ampKernels[ampName] <= negativityThreshold) + if not np.all(bfk.ampKernels[ampName] <= negativityThreshold): + self.log.warning("%s: the kernel is not negative", ampName) else: raise RuntimeError("%s: The kernel is too small for validity check.", ampName) diff --git a/python/lsst/cp/pipe/cpCtiSolve.py b/python/lsst/cp/pipe/cpCtiSolve.py index 3bad9319b..bc2ab1c2e 100644 --- a/python/lsst/cp/pipe/cpCtiSolve.py +++ b/python/lsst/cp/pipe/cpCtiSolve.py @@ -304,21 +304,22 @@ def solveLocalOffsets(self, inputMeasurements, calib, detector): # leaks into the overscan region. signal = [] data = [] - Nskipped = 0 + nSkipped = 0 for exposureEntry in inputMeasurements: exposureDict = exposureEntry['CTI'] if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: signal.append(exposureDict[ampName]['IMAGE_MEAN']) data.append(exposureDict[ampName]['SERIAL_OVERSCAN_VALUES'][start:stop+1]) else: - Nskipped += 1 + nSkipped += 1 self.log.info("Skipped %d exposures brighter than %f.", - Nskipped, self.config.maxImageMean) + nSkipped, self.config.maxImageMean) if len(signal) == 0 or len(data) == 0: # All exposures excluded, set the calibration so that # there is no correction self.log.warning("All exposures brighter than config.maxImageMean are excluded. " - "Setting local offset drift scale to zero for amp {ampName}.") + "Setting local offset drift scale to zero for amp %s.", + ampName) # Arbitrary, will be overwritten by solveGlobalCti calib.globalCti[ampName] = 10**(-6) # Set to zero so that there is no correction @@ -434,15 +435,15 @@ def solveGlobalCti(self, inputMeasurements, calib, detector): # leaks into the overscan region. signal = [] data = [] - Nskipped = 0 + nSkipped = 0 for exposureEntry in inputMeasurements: exposureDict = exposureEntry['CTI'] if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti: signal.append(exposureDict[ampName]['IMAGE_MEAN']) data.append(exposureDict[ampName]['SERIAL_OVERSCAN_VALUES'][start:stop+1]) else: - Nskipped += 1 - self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") + nSkipped += 1 + self.log.info(f"Skipped {nSkipped} exposures brighter than {self.config.maxSignalForCti}.") if len(signal) == 0 or len(data) == 0: # There are no exposures left, set globalCTI to 0 self.log.warning("All exposures brighter than config.maxSignalForCti=%f " @@ -704,7 +705,7 @@ def findTraps(self, inputMeasurements, calib, detector): signal = [] data = [] new_signal = [] - Nskipped = 0 + nSkipped = 0 for exposureEntry in inputMeasurements: exposureDict = exposureEntry['CTI'] if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: @@ -712,9 +713,9 @@ def findTraps(self, inputMeasurements, calib, detector): data.append(exposureDict[ampName]['SERIAL_OVERSCAN_VALUES'][start:stop+1]) new_signal.append(exposureDict[ampName]['LAST_COLUMN_MEAN']) else: - Nskipped += 1 + nSkipped += 1 self.log.info("Skipped %d exposures brighter than %f.", - Nskipped, self.config.maxImageMean) + nSkipped, self.config.maxImageMean) if len(signal) == 0 or len(data) == 0: # There are no exposures left, so set trap so that # there is no correction @@ -845,8 +846,7 @@ def calcEper(self, mode, inputMeasurements, amp): # Number of parallel shifts = nRows nShifts = amp.getRawDataBBox().getHeight() else: - raise RuntimeError("%s is not a known orientation for the EPER " - "calculation.", mode) + raise RuntimeError(f"{mode} is not a known orientation for the EPER calculation.") # The signal is the mean intensity of each input, and the # data are the overscan columns to fit. For detectors From 3ddcf84bdd1d6af31e7dbe370756363ac6a7c6e2 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Tue, 1 Apr 2025 16:09:51 -0700 Subject: [PATCH 15/15] Add doManualDefects = true to LSSTCam cpDefects pipelines --- bps/templates/bps_bfk.yaml | 3 ++- bps/templates/bps_bias.yaml | 3 ++- bps/templates/bps_biasBootstrap.yaml | 5 +++-- bps/templates/bps_cti.yaml | 3 ++- bps/templates/bps_dark.yaml | 3 ++- bps/templates/bps_darkBootstrap.yaml | 7 ++++--- bps/templates/bps_defects.yaml | 8 ++++---- bps/templates/bps_flatBootstrap.yaml | 9 +++++---- bps/templates/bps_flat_g.yaml | 3 ++- bps/templates/bps_flat_i.yaml | 3 ++- bps/templates/bps_flat_r.yaml | 3 ++- bps/templates/bps_flat_u.yaml | 3 ++- bps/templates/bps_flat_y.yaml | 3 ++- bps/templates/bps_flat_z.yaml | 3 ++- bps/templates/bps_illuminationCorrection.yaml | 3 ++- bps/templates/bps_linearizer.yaml | 3 ++- bps/templates/bps_ptc.yaml | 3 ++- pipelines/LSSTCam/cpDefects.yaml | 2 ++ python/lsst/cp/pipe/cpBfk.py | 9 +++++++++ 19 files changed, 53 insertions(+), 26 deletions(-) diff --git a/bps/templates/bps_bfk.yaml b/bps/templates/bps_bfk.yaml index 72cefecd8..4ff1941b0 100644 --- a/bps/templates/bps_bfk.yaml +++ b/bps/templates/bps_bfk.yaml @@ -1,5 +1,6 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBfk.yaml" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpBfkIsrByExposure: @@ -11,7 +12,7 @@ cluster: project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" payload: payloadName: "${INSTRUMENT}_${TICKET}_bfk" diff --git a/bps/templates/bps_bias.yaml b/bps/templates/bps_bias.yaml index cc1f21e5e..5decd9241 100644 --- a/bps/templates/bps_bias.yaml +++ b/bps/templates/bps_bias.yaml @@ -1,5 +1,6 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBias.yaml" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpBiasIsrByExposure: @@ -8,7 +9,7 @@ cluster: project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" payload: payloadName: "${INSTRUMENT}_${TICKET}_bias" diff --git a/bps/templates/bps_biasBootstrap.yaml b/bps/templates/bps_biasBootstrap.yaml index 1ec612af5..b3ab6bf1c 100644 --- a/bps/templates/bps_biasBootstrap.yaml +++ b/bps/templates/bps_biasBootstrap.yaml @@ -2,15 +2,16 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpBiasBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpBiasBootstrapIsrByExposure: pipetasks: cpBiasBootstrapIsr dimensions: exposure -biasBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN" +biasBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.${RERUN}" biasBootstrapRun: "{biasBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" payload: diff --git a/bps/templates/bps_cti.yaml b/bps/templates/bps_cti.yaml index ee3e3dfde..5208aa4af 100644 --- a/bps/templates/bps_cti.yaml +++ b/bps/templates/bps_cti.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpCti.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpCtiIsrByExposure: diff --git a/bps/templates/bps_dark.yaml b/bps/templates/bps_dark.yaml index ff17b39d3..5d629cb56 100644 --- a/bps/templates/bps_dark.yaml +++ b/bps/templates/bps_dark.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDark.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpDarkIsrByExposure: diff --git a/bps/templates/bps_darkBootstrap.yaml b/bps/templates/bps_darkBootstrap.yaml index b0e0c26f7..c9141dec4 100644 --- a/bps/templates/bps_darkBootstrap.yaml +++ b/bps/templates/bps_darkBootstrap.yaml @@ -2,16 +2,17 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDarkBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpDarkBootstrapIsrByExposure: pipetasks: cpDarkBootstrapIsr dimensions: exposure -darkBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN" -biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" +darkBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.${RERUN}" +biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" darkBootstrapRun: "{darkBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" payload: diff --git a/bps/templates/bps_defects.yaml b/bps/templates/bps_defects.yaml index ac1b8796c..68123ee41 100644 --- a/bps/templates/bps_defects.yaml +++ b/bps/templates/bps_defects.yaml @@ -2,11 +2,11 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpDefects.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" -biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" -darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" -flatBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" +biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" +darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" +flatBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" payload: payloadName: "${INSTRUMENT}_${TICKET}_defects" diff --git a/bps/templates/bps_flatBootstrap.yaml b/bps/templates/bps_flatBootstrap.yaml index f50cf4702..4f9ce2183 100644 --- a/bps/templates/bps_flatBootstrap.yaml +++ b/bps/templates/bps_flatBootstrap.yaml @@ -2,17 +2,18 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlatBootstrap.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatBootstrapIsrByExposure: pipetasks: cpFlatBootstrapIsr dimensions: exposure -flatBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.$RERUN" -biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" -darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.$RERUN/run${BOOTSTRAP_RUN_NUMBER}" +flatBootstrapCollection: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/flatBootstrapGen.${RERUN}" +biasBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/biasBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" +darkBootstrapRun: "${USER_CALIB_PREFIX}$INSTRUMENT/calib/$TICKET/$TAG/darkBootstrapGen.${RERUN}/run${BOOTSTRAP_RUN_NUMBER}" flatBootstrapRun: "{flatBootstrapCollection}/run${BOOTSTRAP_RUN_NUMBER}" payload: diff --git a/bps/templates/bps_flat_g.yaml b/bps/templates/bps_flat_g.yaml index c00e714de..ed6b10e80 100644 --- a/bps/templates/bps_flat_g.yaml +++ b/bps/templates/bps_flat_g.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_flat_i.yaml b/bps/templates/bps_flat_i.yaml index 7676997a2..9ddf92c48 100644 --- a/bps/templates/bps_flat_i.yaml +++ b/bps/templates/bps_flat_i.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_flat_r.yaml b/bps/templates/bps_flat_r.yaml index a322900ef..1b212a0cf 100644 --- a/bps/templates/bps_flat_r.yaml +++ b/bps/templates/bps_flat_r.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_flat_u.yaml b/bps/templates/bps_flat_u.yaml index 7b03af9b8..b09891ec8 100644 --- a/bps/templates/bps_flat_u.yaml +++ b/bps/templates/bps_flat_u.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_flat_y.yaml b/bps/templates/bps_flat_y.yaml index c65c0bcab..afc75416b 100644 --- a/bps/templates/bps_flat_y.yaml +++ b/bps/templates/bps_flat_y.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_flat_z.yaml b/bps/templates/bps_flat_z.yaml index 20a54b54a..8efe3f355 100644 --- a/bps/templates/bps_flat_z.yaml +++ b/bps/templates/bps_flat_z.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpFlat.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpFlatIsrByExposure: diff --git a/bps/templates/bps_illuminationCorrection.yaml b/bps/templates/bps_illuminationCorrection.yaml index 40bb743a3..cd90d7674 100644 --- a/bps/templates/bps_illuminationCorrection.yaml +++ b/bps/templates/bps_illuminationCorrection.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpIlluminationCorrection.y project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpIlluminationCorrectionIsrByExposure: diff --git a/bps/templates/bps_linearizer.yaml b/bps/templates/bps_linearizer.yaml index a9f2d5898..beaef248a 100644 --- a/bps/templates/bps_linearizer.yaml +++ b/bps/templates/bps_linearizer.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpLinearizer.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpLinearizerIsrByExposure: diff --git a/bps/templates/bps_ptc.yaml b/bps/templates/bps_ptc.yaml index d7ea6fbd2..5033b9343 100644 --- a/bps/templates/bps_ptc.yaml +++ b/bps/templates/bps_ptc.yaml @@ -2,8 +2,9 @@ pipelineYaml: "${CP_PIPE_DIR}/pipelines/${INSTRUMENT}/cpPtc.yaml" project: "${TICKET}" campaign: "${TICKET}" -submitPath: "$SCRATCH/submit/{outputRun}" +submitPath: "${SCRATCH}/submit/{outputRun}" +# Necessary to cluster by exposure if there are many input exposures clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering cluster: cpPtcIsrByExposure: diff --git a/pipelines/LSSTCam/cpDefects.yaml b/pipelines/LSSTCam/cpDefects.yaml index 015380d31..55536993c 100644 --- a/pipelines/LSSTCam/cpDefects.yaml +++ b/pipelines/LSSTCam/cpDefects.yaml @@ -26,4 +26,6 @@ tasks: cpMergeDefects: class: lsst.cp.pipe.MergeDefectsCombinedTask config: + connections.inputManualDefects: "manual_defects" edgesAsDefects: true + doManualDefects: true diff --git a/python/lsst/cp/pipe/cpBfk.py b/python/lsst/cp/pipe/cpBfk.py index de7d70b0f..b7132f1bf 100644 --- a/python/lsst/cp/pipe/cpBfk.py +++ b/python/lsst/cp/pipe/cpBfk.py @@ -438,6 +438,15 @@ def run(self, inputPtc, dummy, camera, inputDims): # Check for validity if self.config.doCheckValidity: + # We will check 3 boundary conditions to test the + # validitity of the kernel: + # (1) The edge of the kernel should be consistent with zero. + # (2) The kernel should be negative + # (3) The center of the kernal should be the absolute minimum. + + # Since there are typically too few pixels along the edge, we + # will use the stddev of the 3px picture frame of the kernel to + # set the tolerances of the validity conditions. kernelSize = bfk.ampKernels[ampName].shape[0] kernelCenterValue = bfk.ampKernels[ampName][kernelSize//2, kernelSize//2] xv, yv = np.meshgrid(range(kernelSize), range(kernelSize))