From 9c23429331164e7b2981032a9ba6be5931f4c352 Mon Sep 17 00:00:00 2001 From: David Staker Date: Fri, 7 Sep 2018 17:49:21 -0500 Subject: [PATCH 1/6] Initial work on numpydoc conversion --- doc/lsst.pipe.tasks/index.rst | 39 ++ python/lsst/pipe/tasks/calibrate.py | 246 ++++++----- python/lsst/pipe/tasks/characterizeImage.py | 230 +++++++---- python/lsst/pipe/tasks/coaddBase.py | 122 +++--- python/lsst/pipe/tasks/coaddHelpers.py | 57 ++- python/lsst/pipe/tasks/coaddInputRecorder.py | 68 +++- python/lsst/pipe/tasks/colorterms.py | 130 ++++-- python/lsst/pipe/tasks/dcrAssembleCoadd.py | 5 +- python/lsst/pipe/tasks/exampleCmdLineTask.py | 118 +++--- python/lsst/pipe/tasks/exampleStatsTasks.py | 109 ++--- python/lsst/pipe/tasks/fakes.py | 16 +- python/lsst/pipe/tasks/getRepositoryData.py | 70 +++- python/lsst/pipe/tasks/imageDifference.py | 63 +-- python/lsst/pipe/tasks/ingest.py | 177 +++++--- python/lsst/pipe/tasks/ingestCalibs.py | 42 +- python/lsst/pipe/tasks/ingestPgsql.py | 42 +- python/lsst/pipe/tasks/interpImage.py | 44 +- python/lsst/pipe/tasks/makeCoaddTempExp.py | 203 ++++++---- python/lsst/pipe/tasks/makeDiscreteSkyMap.py | 47 ++- python/lsst/pipe/tasks/makeSkyMap.py | 27 +- python/lsst/pipe/tasks/matchBackgrounds.py | 176 +++++--- python/lsst/pipe/tasks/measurePsf.py | 237 ++++------- python/lsst/pipe/tasks/multiBand.py | 383 ++++++++---------- python/lsst/pipe/tasks/objectMasks.py | 72 ++-- python/lsst/pipe/tasks/photoCal.py | 271 ++++++------- python/lsst/pipe/tasks/processCcd.py | 130 +++--- python/lsst/pipe/tasks/propagateVisitFlags.py | 63 +-- python/lsst/pipe/tasks/registerImage.py | 95 +++-- python/lsst/pipe/tasks/repair.py | 173 +++----- python/lsst/pipe/tasks/repositoryIterator.py | 121 ++++-- python/lsst/pipe/tasks/scaleVariance.py | 2 + python/lsst/pipe/tasks/scaleZeroPoint.py | 151 ++++--- python/lsst/pipe/tasks/selectImages.py | 108 +++-- python/lsst/pipe/tasks/setConfigFromEups.py | 21 +- python/lsst/pipe/tasks/setPrimaryFlags.py | 19 +- python/lsst/pipe/tasks/snapCombine.py | 102 +++-- .../lsst/pipe/tasks/transformMeasurement.py | 142 +++---- 37 files changed, 2335 insertions(+), 1786 deletions(-) diff --git a/doc/lsst.pipe.tasks/index.rst b/doc/lsst.pipe.tasks/index.rst index 433815628..369d6bf1f 100644 --- a/doc/lsst.pipe.tasks/index.rst +++ b/doc/lsst.pipe.tasks/index.rst @@ -72,3 +72,42 @@ Python API reference ==================== .. automodapi:: lsst.pipe.tasks.assembleCoadd +.. automodapi:: lsst.pipe.tasks.calibrate +.. automodapi:: lsst.pipe.tasks.characterizeImage +.. automodapi:: lsst.pipe.tasks.coaddBase +.. automodapi:: lsst.pipe.tasks.coaddHelpers +.. automodapi:: lsst.pipe.tasks.coaddInputRecorder +.. automodapi:: lsst.pipe.tasks.colorterms +.. automodapi:: lsst.pipe.tasks.dcrAssembleCoadd +.. automodapi:: lsst.pipe.tasks.dcrMultiBand +.. automodapi:: lsst.pipe.tasks.exampleCmdLineTask +.. automodapi:: lsst.pipe.tasks.exampleStatsTasks +.. automodapi:: lsst.pipe.tasks.fakes +.. automodapi:: lsst.pipe.tasks.getRepositoryData +.. automodapi:: lsst.pipe.tasks.imageDifference +.. automodapi:: lsst.pipe.tasks.ingestCalibs +.. automodapi:: lsst.pipe.tasks.ingestPgsql +.. automodapi:: lsst.pipe.tasks.ingest +.. automodapi:: lsst.pipe.tasks.interpImage +.. automodapi:: lsst.pipe.tasks.makeCoaddTempExp +.. automodapi:: lsst.pipe.tasks.makeDiscreteSkyMap +.. automodapi:: lsst.pipe.tasks.makeSkyMap +.. automodapi:: lsst.pipe.tasks.matchBackgrounds +.. automodapi:: lsst.pipe.tasks.measurePsf +.. automodapi:: lsst.pipe.tasks.multiBand +.. automodapi:: lsst.pipe.tasks.photoCal +.. automodapi:: lsst.pipe.tasks.processCcd +.. automodapi:: lsst.pipe.tasks.propagateVisitFlags +.. automodapi:: lsst.pipe.tasks.registerImage +.. automodapi:: lsst.pipe.tasks.repair +.. automodapi:: lsst.pipe.tasks.repositoryIterator +.. automodapi:: lsst.pipe.tasks.scaleVariance +.. automodapi:: lsst.pipe.tasks.scaleZeroPoint +.. automodapi:: lsst.pipe.tasks.selectImages +.. automodapi:: lsst.pipe.tasks.setConfigFromEups +.. automodapi:: lsst.pipe.tasks.setPrimaryFlags +.. automodapi:: lsst.pipe.tasks.snapCombine +.. automodapi:: lsst.pipe.tasks.transformMeasurement +.. automodapi:: lsst.pipe.tasks.version +.. automodapi:: lsst.pipe.tasks.warpAndPsfMatch +.. automodapi:: lsst.pipe.tasks.objectMasks \ No newline at end of file diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 8c4b5d913..4432f162e 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -180,35 +180,47 @@ def setDefaults(self): ## \} class CalibrateTask(pipeBase.CmdLineTask): - r"""!Calibrate an exposure: measure sources and perform astrometric and - photometric calibration - - @anchor CalibrateTask_ - - @section pipe_tasks_calibrate_Contents Contents - - - @ref pipe_tasks_calibrate_Purpose - - @ref pipe_tasks_calibrate_Initialize - - @ref pipe_tasks_calibrate_IO - - @ref pipe_tasks_calibrate_Config - - @ref pipe_tasks_calibrate_Metadata - - @ref pipe_tasks_calibrate_Debug - - - @section pipe_tasks_calibrate_Purpose Description - + """Calibrate an exposure: measure sources and perform astrometric and + photometric calibration + + Parameters + ---------- + butler : `lsst.daf.persistence.Butler` + The butler is passed to the refObjLoader constructor + in case it is needed. Ignored if the refObjLoader argument + provides a loader directly. + astromRefObjLoader : + An instance of LoadReferenceObjectsTasks + that supplies an external reference catalog for astrometric + calibration. May be None if the desired loader can be constructed + from the butler argument or all steps requiring a reference catalog + are disabled. + photoRefObjLoader : + An instance of LoadReferenceObjectsTasks + that supplies an external reference catalog for photometric + calibration. May be None if the desired loader can be constructed + from the butler argument or all steps requiring a reference catalog + are disabled. + icSourceSchema : + schema for icSource catalog, or None. + Schema values specified in config.icSourceFieldsToCopy will be + taken from this schema. If set to None, no values will be + propagated from the icSourceCatalog + kwargs : + other keyword arguments for + lsst.pipe.base.CmdLineTask + + Notes + ----- Given an exposure with a good PSF model and aperture correction map (e.g. as provided by @ref CharacterizeImageTask), perform the following - operations: + operations: + - Run detection and measurement - Run astrometry subtask to fit an improved WCS - Run photoCal subtask to fit the exposure's photometric zero-point - @section pipe_tasks_calibrate_Initialize Task initialisation - - @copydoc \_\_init\_\_ - - @section pipe_tasks_calibrate_IO Invoking the Task + Invoking the Task If you want this task to unpersist inputs or persist outputs, then call the `runDataRef` method (a wrapper around the `run` method). @@ -216,39 +228,42 @@ class CalibrateTask(pipeBase.CmdLineTask): If you already have the inputs unpersisted and do not want to persist the output then it is more direct to call the `run` method: - @section pipe_tasks_calibrate_Config Configuration parameters + Quantities set in exposure Metadata + + Exposure metadata - See @ref CalibrateConfig + .. code-block:: none - @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata + MAGZERO_RMS MAGZERO's RMS == sigma reported by photoCal task + ERO_NOBJ Number of stars used == ngood reported by photoCal + task + COLORTERM1 (always 0.0) + COLORTERM2 (always 0.0) + COLORTERM3 (always 0.0) - Exposure metadata -
-
MAGZERO_RMS
MAGZERO's RMS == sigma reported by photoCal task -
MAGZERO_NOBJ
Number of stars used == ngood reported by photoCal - task -
COLORTERM1
?? (always 0.0) -
COLORTERM2
?? (always 0.0) -
COLORTERM3
?? (always 0.0) -
- - @section pipe_tasks_calibrate_Debug Debug variables - - The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink + Debug variables + + The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task interface supports a flag `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`. CalibrateTask has a debug dictionary containing one key: -
-
calibrate -
frame (an int; <= 0 to not display) in which to display the exposure, + + .. code-block:: none + + calibrate + frame (an int; <= 0 to not display) in which to display the exposure, sources and matches. See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. -
+ + Examples + -------- For example, put something like: - @code{.py} + + .. code-block:: py + import lsstDebug def DebugInfo(name): di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would @@ -261,7 +276,7 @@ def DebugInfo(name): return di lsstDebug.Info = DebugInfo - @endcode + into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. @@ -278,26 +293,33 @@ def DebugInfo(name): def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, **kwargs): - """!Construct a CalibrateTask + """Construct a CalibrateTask - @param[in] butler The butler is passed to the refObjLoader constructor + Parameters + ---------- + butler : + The butler is passed to the refObjLoader constructor in case it is needed. Ignored if the refObjLoader argument provides a loader directly. - @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks + astromRefObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an external reference catalog for astrometric calibration. May be None if the desired loader can be constructed from the butler argument or all steps requiring a reference catalog are disabled. - @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks + photoRefObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an external reference catalog for photometric calibration. May be None if the desired loader can be constructed from the butler argument or all steps requiring a reference catalog are disabled. - @param[in] icSourceSchema schema for icSource catalog, or None. + icSourceSchema : + schema for icSource catalog, or None. Schema values specified in config.icSourceFieldsToCopy will be taken from this schema. If set to None, no values will be propagated from the icSourceCatalog - @param[in,out] kwargs other keyword arguments for + kwargs : + other keyword arguments for lsst.pipe.base.CmdLineTask """ pipeBase.CmdLineTask.__init__(self, **kwargs) @@ -382,33 +404,51 @@ def __init__(self, butler=None, astromRefObjLoader=None, @pipeBase.timeMethod def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True): - """!Calibrate an exposure, optionally unpersisting inputs and + """Calibrate an exposure, optionally unpersisting inputs and persisting outputs. This is a wrapper around the `run` method that unpersists inputs (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) - @param[in] dataRef butler data reference corresponding to a science + Parameters + ---------- + dataRef : + butler data reference corresponding to a science image - @param[in,out] exposure characterized exposure (an - lsst.afw.image.ExposureF or similar), or None to unpersist existing + exposure : `lsst.afw.image.ExposureF` + characterized exposure, or None to unpersist existing icExp and icBackground. See `run` method for details of what is read and written. - @param[in,out] background initial model of background already - subtracted from exposure (an lsst.afw.math.BackgroundList). May be + background : lsst.afw.math.BackgroundList + initial model of background already + subtracted from exposure. May be None if no background has been subtracted, though that is unusual for calibration. A refined background model is output. Ignored if exposure is None. - @param[in] icSourceCat catalog from which to copy the fields specified + icSourceCat : + catalog from which to copy the fields specified by icSourceKeys, or None; - @param[in] doUnpersist unpersist data: + doUnpersist : + unpersist data: - if True, exposure, background and icSourceCat are read from - dataRef and those three arguments must all be None; + dataRef and those three arguments must all be None; - if False the exposure must be provided; background and - icSourceCat are optional. True is intended for running as a - command-line task, False for running as a subtask - - @return same data as the calibrate method + icSourceCat are optional. True is intended for running as a + command-line task, False for running as a subtask + + Returns + ------- + calRes : + same data as the calibrate method + + Raises + ------ + RuntimeError + doUnpersist true; exposure, background + and icSourceCat must all be None + + RuntimeError + doUnpersist false; exposure must be provided """ self.log.info("Processing %s" % (dataRef.dataId)) @@ -446,34 +486,42 @@ def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None): - """!Calibrate an exposure (science image or coadd) + """Calibrate an exposure (science image or coadd) - @param[in,out] exposure exposure to calibrate (an - lsst.afw.image.ExposureF or similar); + Parameters + ---------- + exposure : `lsst.afw.image.ExposureF` + exposure to calibrate (an lsst.afw.image.ExposureF or similar); in: - MaskedImage - Psf + out: - MaskedImage has background subtracted - Wcs is replaced - Calib zero-point is set - @param[in] exposureIdInfo ID info for exposure (an - lsst.obs.base.ExposureIdInfo) If not provided, returned + + exposureIdInfo : `lsst.obs.base.ExposureIdInfo` + ID info for exposure. If not provided, returned SourceCatalog IDs will not be globally unique. - @param[in,out] background background model already subtracted from - exposure (an lsst.afw.math.BackgroundList). May be None if no + background : `lsst.afw.math.BackgroundList` + background model already subtracted from + exposure. May be None if no background has been subtracted, though that is unusual for calibration. A refined background model is output. - @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask + icSourceCat : + A SourceCatalog from CharacterizeImageTask from which we can copy some fields. - @return pipe_base Struct containing these fields: - - exposure calibrate science exposure with refined WCS and Calib - - background model of background subtracted from exposure (an - lsst.afw.math.BackgroundList) - - sourceCat catalog of measured sources - - astromMatches list of source/refObj matches from the astrometry - solver + Returns + ------- + result: `lsst.pipe.base.Struct` + + - ``exposure`` : calibrate science exposure with refined WCS and Calib + - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) + - ``sourceCat`` : catalog of measured sources + - ``astromMatches`` : list of source/refObj matches from the astrometry solver + """ # detect, deblend and measure sources if exposureIdInfo is None: @@ -604,12 +652,18 @@ def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta): """Write output data to the output repository - @param[in] dataRef butler data reference corresponding to a science - image - @param[in] exposure exposure to write - @param[in] background background model for exposure - @param[in] sourceCat catalog of measured sources - @param[in] astromMatches list of source/refObj matches from the + Parameters + ---------- + dataRef : + butler data reference corresponding to a science + exposure : `lsst.afw.image.ExposureF` + exposure to write + background :`lsst.afw.math.BackgroundList` + background model for exposure + sourceCat : + catalog of measured sources + astromMatches : + list of source/refObj matches from the astrometry solver """ dataRef.put(sourceCat, "src") @@ -632,12 +686,16 @@ def getSchemaCatalogs(self): return {"src": sourceCat} def setMetadata(self, exposure, photoRes=None): - """!Set task and exposure metadata + """Set task and exposure metadata Logs a warning and continues if needed data is missing. - @param[in,out] exposure exposure whose metadata is to be set - @param[in] photoRes results of running photoCal; if None then it was + Parameters + ---------- + exposure : `lsst.afw.image.ExposureF` + exposure whose metadata is to be set + photoRes : + results of running photoCal; if None then it was not run """ if photoRes is None: @@ -663,11 +721,17 @@ def setMetadata(self, exposure, photoRes=None): self.log.warn("Could not set exposure metadata: %s" % (e,)) def copyIcSourceFields(self, icSourceCat, sourceCat): - """!Match sources in icSourceCat and sourceCat and copy the specified fields + """Match sources in icSourceCat and sourceCat and copy the specified fields - @param[in] icSourceCat catalog from which to copy fields - @param[in,out] sourceCat catalog to which to copy fields + Parameters + ---------- + icSourceCat : + catalog from which to copy fields + sourceCat : + catalog to which to copy fields + Notes + ----- The fields copied are those specified by `config.icSourceFieldsToCopy` that actually exist in the schema. This was set up by the constructor using self.schemaMapper. diff --git a/python/lsst/pipe/tasks/characterizeImage.py b/python/lsst/pipe/tasks/characterizeImage.py index 6511e9d3b..8f09e3792 100644 --- a/python/lsst/pipe/tasks/characterizeImage.py +++ b/python/lsst/pipe/tasks/characterizeImage.py @@ -178,70 +178,80 @@ def validate(self): class CharacterizeImageTask(pipeBase.CmdLineTask): - r"""!Measure bright sources and use this to estimate background and PSF of an exposure + """Measure bright sources and use this to estimate background and PSF of an exposure + + Parameters + ---------- + butler : + A butler object is passed to the refObjLoader constructor in case + it is needed to load catalogs. May be None if a catalog-based star selector is + not used, if the reference object loader constructor does not require a butler, + or if a reference object loader is passed directly via the refObjLoader argument. + refObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an + external reference catalog to a catalog-based star selector. May be None if a + catalog star selector is not used or the loader can be constructed from the + butler argument. + schema : + initial schema (an lsst.afw.table.SourceTable), or None + kwargs : + other keyword arguments for lsst.pipe.base.CmdLineTask + + Notes + ----- + Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask): + - detect and measure bright sources + - repair cosmic rays + - measure and subtract background + - measure PSF - @anchor CharacterizeImageTask_ + If you want this task to unpersist inputs or persist outputs, then call + the `runDataRef` method (a thin wrapper around the `run` method). - @section pipe_tasks_characterizeImage_Contents Contents + If you already have the inputs unpersisted and do not want to persist the output + then it is more direct to call the `run` method: - - @ref pipe_tasks_characterizeImage_Purpose - - @ref pipe_tasks_characterizeImage_Initialize - - @ref pipe_tasks_characterizeImage_IO - - @ref pipe_tasks_characterizeImage_Config - - @ref pipe_tasks_characterizeImage_Debug + Configuration parameters + - See CharacterizeImageConfig + Debug variables - @section pipe_tasks_characterizeImage_Purpose Description + The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a flag + `--debug` to import `debug.py` from your `$PYTHONPATH`; see baseDebug for more about `debug.py`. - Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask): - - detect and measure bright sources - - repair cosmic rays - - measure and subtract background - - measure PSF + CharacterizeImageTask has a debug dictionary with the following keys - @section pipe_tasks_characterizeImage_Initialize Task initialisation + frame: + - `int`: if specified, the frame of first debug image displayed (defaults to 1) - @copydoc \_\_init\_\_ + repair_iter: + - `bool`; if True display image after each repair in the measure PSF loop - @section pipe_tasks_characterizeImage_IO Invoking the Task + background_iter: + - `bool`; if True display image after each background subtraction in the measure PSF loop - If you want this task to unpersist inputs or persist outputs, then call - the `runDataRef` method (a thin wrapper around the `run` method). + measure_iter: + - `bool`; if True display image and sources at the end of each iteration of the measure PSF loop - If you already have the inputs unpersisted and do not want to persist the output - then it is more direct to call the `run` method: + See lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. + + psf: + - `bool`; if True display image and sources after PSF is measured; + + this will be identical to the final image displayed by measure_iter if measure_iter is true - @section pipe_tasks_characterizeImage_Config Configuration parameters - - See @ref CharacterizeImageConfig - - @section pipe_tasks_characterizeImage_Debug Debug variables - - The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag - `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`. - - CharacterizeImageTask has a debug dictionary with the following keys: -
-
frame -
int: if specified, the frame of first debug image displayed (defaults to 1) -
repair_iter -
bool; if True display image after each repair in the measure PSF loop -
background_iter -
bool; if True display image after each background subtraction in the measure PSF loop -
measure_iter -
bool; if True display image and sources at the end of each iteration of the measure PSF loop - See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. -
psf -
bool; if True display image and sources after PSF is measured; - this will be identical to the final image displayed by measure_iter if measure_iter is true -
repair -
bool; if True display image and sources after final repair -
measure -
bool; if True display image and sources after final measurement -
+ repair: + - `bool`; if True display image and sources after final repair + measure: + - `bool`; if True display image and sources after final measurement + + Examples + -------- For example, put something like: - @code{.py} + + .. code-block:: none + import lsstDebug def DebugInfo(name): di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively @@ -253,9 +263,8 @@ def DebugInfo(name): return di lsstDebug.Info = DebugInfo - @endcode - into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. + into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. Some subtasks may have their own debug variables; see individual Task documentation. """ @@ -266,18 +275,24 @@ def DebugInfo(name): RunnerClass = pipeBase.ButlerInitializedTaskRunner def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): - """!Construct a CharacterizeImageTask + """Construct a CharacterizeImageTask - @param[in] butler A butler object is passed to the refObjLoader constructor in case + Parameters + ---------- + butler : + A butler object is passed to the refObjLoader constructor in case it is needed to load catalogs. May be None if a catalog-based star selector is not used, if the reference object loader constructor does not require a butler, or if a reference object loader is passed directly via the refObjLoader argument. - @param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an + refObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an external reference catalog to a catalog-based star selector. May be None if a catalog star selector is not used or the loader can be constructed from the butler argument. - @param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None - @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask + schema : + initial schema (an lsst.afw.table.SourceTable), or None + kwargs : + other keyword arguments for lsst.pipe.base.CmdLineTask """ pipeBase.CmdLineTask.__init__(self, **kwargs) if schema is None: @@ -307,29 +322,39 @@ def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): @pipeBase.timeMethod def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True): - """!Characterize a science image and, if wanted, persist the results + """Characterize a science image and, if wanted, persist the results This simply unpacks the exposure and passes it to the characterize method to do the work. - @param[in] dataRef: butler data reference for science exposure - @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar). + Parameters + ---------- + dataRef: butler data reference for science exposure + exposure : + exposure to characterize (an lsst.afw.image.ExposureF or similar). If None then unpersist from "postISRCCD". The following changes are made, depending on the config: + - set psf to the measured PSF - set apCorrMap to the measured aperture correction - subtract background - interpolate over cosmic rays - update detection and cosmic ray mask planes - @param[in,out] background initial model of background already subtracted from exposure + + background : `lsst.afw.math.BackgroundList` + initial model of background already subtracted from exposure (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, which is typical for image characterization. A refined background model is output. - @param[in] doUnpersist if True the exposure is read from the repository + doUnpersist : + if True the exposure is read from the repository and the exposure and background arguments must be None; if False the exposure must be provided. True is intended for running as a command-line task, False for running as a subtask - @return same data as the characterize method + Returns + ------- + charRes : + same data as the characterize method """ self._frame = self._initialFrame # reset debug display frame self.log.info("Processing %s" % (dataRef.dataId)) @@ -359,33 +384,43 @@ def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True): @pipeBase.timeMethod def run(self, exposure, exposureIdInfo=None, background=None): - """!Characterize a science image + """Characterize a science image Peforms the following operations: - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: - - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) + - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) - interpolate over cosmic rays - perform final measurement - @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar). + Parameters + ---------- + exposure : + exposure to characterize (an lsst.afw.image.ExposureF or similar). The following changes are made: - update or set psf - set apCorrMap - update detection and cosmic ray mask planes - subtract background and interpolate over cosmic rays - @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo). + + exposureIdInfo : + ID info for exposure (an lsst.obs.base.ExposureIdInfo). If not provided, returned SourceCatalog IDs will not be globally unique. - @param[in,out] background initial model of background already subtracted from exposure + background : `lsst.afw.math.BackgroundList` + initial model of background already subtracted from exposure (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, which is typical for image characterization. - @return pipe_base Struct containing these fields, all from the final iteration - of detectMeasureAndEstimatePsf: - - exposure: characterized exposure; image is repaired by interpolating over cosmic rays, + Returns + ------- + dmeRes : `struct` + pipe_base Struct containing these fields, all from the final iteration + of detectMeasureAndEstimatePsf: + - ``exposure`` : characterized exposure; image is repaired by interpolating over cosmic rays, mask is updated accordingly, and the PSF model is set - - sourceCat: detected sources (an lsst.afw.table.SourceCatalog) - - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) + - ``sourceCat`` : detected sources (an lsst.afw.table.SourceCatalog) + - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) + - ``psfCellSet`` : spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) + """ self._frame = self._initialFrame # reset debug display frame @@ -436,33 +471,46 @@ def run(self, exposure, exposureIdInfo=None, background=None): @pipeBase.timeMethod def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): - """!Perform one iteration of detect, measure and estimate PSF + """Perform one iteration of detect, measure and estimate PSF Performs the following operations: + - if config.doMeasurePsf or not exposure.hasPsf(): - - install a simple PSF model (replacing the existing one, if need be) + - install a simple PSF model (replacing the existing one, if need be) - interpolate over cosmic rays with keepCRs=True - estimate background and subtract it from the exposure - detect, deblend and measure sources, and subtract a refined background model; - - if config.doMeasurePsf: - - measure PSF - @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar) + if config.doMeasurePsf: + + - measure PSF + + exposure : + exposure to characterize (an lsst.afw.image.ExposureF or similar) The following changes are made: - update or set psf - update detection and cosmic ray mask planes - subtract background - @param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo) - @param[in,out] background initial model of background already subtracted from exposure + + exposureIdInfo : `lsst.afw.math.BackgroundList` + ID info for exposure (an lsst.obs_base.ExposureIdInfo) + background : `lsst.afw.math.BackgroundList` + initial model of background already subtracted from exposure (an lsst.afw.math.BackgroundList). - @return pipe_base Struct containing these fields, all from the final iteration + Returns + ------- + pipe_base_struct : `struct` + ` pipe_base Struct containing these fields, all from the final iteration of detect sources, measure sources and estimate PSF: - - exposure characterized exposure; image is repaired by interpolating over cosmic rays, - mask is updated accordingly, and the PSF model is set - - sourceCat detected sources (an lsst.afw.table.SourceCatalog) - - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - - psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) + + - ``exposure`` : characterized exposure; image is repaired by interpolating over cosmic rays, + + mask is updated accordingly, and the PSF model is set: + - ``sourceCat`` : detected sources (an lsst.afw.table.SourceCatalog) + - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) + - ``psfCellSet`` : spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) + """ # install a simple PSF model, if needed or wanted if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): @@ -518,9 +566,11 @@ def getSchemaCatalogs(self): def display(self, itemName, exposure, sourceCat=None): """Display exposure and sources on next frame, if display of itemName has been requested - @param[in] itemName name of item in debugInfo - @param[in] exposure exposure to display - @param[in] sourceCat source catalog to display + Parameters + ---------- + itemName : name of item in debugInfo + exposure : exposure to display + sourceCat : source catalog to display """ val = getDebugFrame(self._display, itemName) if not val: diff --git a/python/lsst/pipe/tasks/coaddBase.py b/python/lsst/pipe/tasks/coaddBase.py index cb802c9b5..bdb95f587 100644 --- a/python/lsst/pipe/tasks/coaddBase.py +++ b/python/lsst/pipe/tasks/coaddBase.py @@ -35,11 +35,9 @@ class CoaddBaseConfig(pexConfig.Config): - """!Configuration parameters for CoaddBaseTask + """Configuration parameters for CoaddBaseTask - @anchor CoaddBaseConfig_ - - @brief Configuration parameters shared between MakeCoaddTempExp and AssembleCoadd + Configuration parameters shared between MakeCoaddTempExp and AssembleCoadd """ coaddName = pexConfig.Field( doc="Coadd name: typically one of deep or goodSeeing.", @@ -93,7 +91,7 @@ def getTargetList(parsedCmd, **kwargs): class CoaddBaseTask(pipeBase.CmdLineTask): - """!Base class for coaddition. + """Base class for coaddition. Subclasses must specify _DefaultName """ @@ -106,18 +104,25 @@ def __init__(self, *args, **kwargs): self.makeSubtask("inputRecorder") def selectExposures(self, patchRef, skyInfo=None, selectDataList=[]): - """! - @brief Select exposures to coadd + """Select exposures to coadd - Get the corners of the bbox supplied in skyInfo using @ref afwGeom.Box2D and convert the pixel - positions of the bbox corners to sky coordinates using @ref skyInfo.wcs.pixelToSky. Use the - @ref WcsSelectImagesTask_ "WcsSelectImagesTask" to select exposures that lie inside the patch + Get the corners of the bbox supplied in skyInfo using afwGeom.Box2D and convert the pixel + positions of the bbox corners to sky coordinates using skyInfo.wcs.pixelToSky. Use the + "WcsSelectImagesTask" to select exposures that lie inside the patch indicated by the dataRef. - @param[in] patchRef data reference for sky map patch. Must include keys "tract", "patch", - plus the camera-specific filter key (e.g. "filter" or "band") - @param[in] skyInfo geometry for the patch; output from getSkyInfo - @return a list of science exposures to coadd, as butler data references + Parameters + ---------- + patchRef : + data reference for sky map patch. Must include keys "tract", "patch", + plus the camera-specific filter key (e.g. "filter" or "band") + skyInfo : + geometry for the patch; output from getSkyInfo + + Returns + ------- + self.select.runDataRef : + a list of science exposures to coadd, as butler data references """ if skyInfo is None: skyInfo = self.getSkyInfo(patchRef) @@ -126,18 +131,23 @@ def selectExposures(self, patchRef, skyInfo=None, selectDataList=[]): return self.select.runDataRef(patchRef, coordList, selectDataList=selectDataList).dataRefList def getSkyInfo(self, patchRef): - """! - @brief Use @ref getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox + """Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch. - @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch" + Parameters + ---------- + patchRef : + data reference for sky map. Must include keys "tract" and "patch" - @return pipe_base Struct containing: - - skyMap: sky map - - tractInfo: information for chosen tract of sky map - - patchInfo: information about chosen patch of tract - - wcs: WCS of tract - - bbox: outer bbox of patch, as an afwGeom Box2I + Returns + ------- + getSkyInfo : `struct` + pipe_base Struct containing: + - ``skyMap`` : sky map + - ``tractInfo`` : information for chosen tract of sky map + - ``patchInfo`` : information about chosen patch of tract + - ``wcs`` : WCS of tract + - ``bbox`` : outer bbox of patch, as an afwGeom Box2I """ return getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef) @@ -151,7 +161,8 @@ def getCoaddDatasetName(self, warpType="direct"): Returns ------- - CoaddDatasetName : `string` + CoaddDatasetName : + `string` """ suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] return self.config.coaddName + "Coadd" + suffix @@ -192,15 +203,13 @@ def _getMetadataName(self): return "%s_%s_metadata" % (self.config.coaddName, self._DefaultName) def getBadPixelMask(self): - """! - @brief Convenience method to provide the bitmask from the mask plane names + """Convenience method to provide the bitmask from the mask plane names """ return afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes) class SelectDataIdContainer(pipeBase.DataIdContainer): - """! - @brief A dataId container for inputs to be selected. + """A dataId container for inputs to be selected. Read the header (including the size and Wcs) for all specified inputs and pass those along, ultimately for the SelectImagesTask. @@ -224,18 +233,21 @@ def makeDataRefList(self, namespace): def getSkyInfo(coaddName, patchRef): - """! - @brief Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded. - - @param[in] coaddName coadd name; typically one of deep or goodSeeing - @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch" - - @return pipe_base Struct containing: - - skyMap: sky map - - tractInfo: information for chosen tract of sky map - - patchInfo: information about chosen patch of tract - - wcs: WCS of tract - - bbox: outer bbox of patch, as an afwGeom Box2I + """Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded. + + coaddName : + coadd name; typically one of deep or goodSeeing + patchRef : + data reference for sky map. Must include keys "tract" and "patch" + + Returns + ------- + pipe_base Struct containing: + - ``skyMap`` : sky map + - ``tractInfo`` : information for chosen tract of sky map + - ``patchInfo`` : information about chosen patch of tract + - ``wcs`` : WCS of tract + - ``bbox`` : outer bbox of patch, as an afwGeom Box2I """ skyMap = patchRef.get(coaddName + "Coadd_skyMap") tractId = patchRef.dataId["tract"] @@ -255,22 +267,32 @@ def getSkyInfo(coaddName, patchRef): def scaleVariance(maskedImage, maskPlanes, log=None): - """! - @brief Scale the variance in a maskedImage - + """Scale the variance in a maskedImage + + This is deprecated. Use the ScaleVarianceTask instead. + + Parameters + ---------- + maskedImage : + MaskedImage to operate on; variance will be scaled + maskPlanes : + List of mask planes for pixels to reject + log : + Log for reporting the renormalization factor; or None + + Returns + ------- + task.run : + renormalisation factor + + Notes + ----- The variance plane in a convolved or warped image (or a coadd derived from warped images) does not accurately reflect the noise properties of the image because variance has been lost to covariance. This function attempts to correct for this by scaling the variance plane to match the observed variance in the image. This is not perfect (because we're not tracking the covariance) but it's simple and is often good enough. - - @deprecated Use the ScaleVarianceTask instead. - - @param maskedImage MaskedImage to operate on; variance will be scaled - @param maskPlanes List of mask planes for pixels to reject - @param log Log for reporting the renormalization factor; or None - @return renormalisation factor """ config = ScaleVarianceTask.ConfigClass() config.maskPlanes = maskPlanes diff --git a/python/lsst/pipe/tasks/coaddHelpers.py b/python/lsst/pipe/tasks/coaddHelpers.py index 07d185156..96738c210 100644 --- a/python/lsst/pipe/tasks/coaddHelpers.py +++ b/python/lsst/pipe/tasks/coaddHelpers.py @@ -39,9 +39,18 @@ def groupDataRefs(keys, dataRefIterable): The effect is that the data references in each group have the same values for the provided keys. - @param keys: List of keys to consider when grouping (order is important) - @param dataRefIterable: Iterable of data references to group - @return Dict of : + Parameters + ---------- + keys: + List of keys to consider when grouping (order is important) + + dataRefIterable: + Iterable of data references to group + + Returns + ------- + groupDict : + Dict of : """ groupDict = dict() for dataRef in dataRefIterable: @@ -65,9 +74,11 @@ def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepC consist of the subset of CCD exposures from a single camera exposure that potentially overlap the patch. - @return Struct with: - - groups: Dict of : - - keys: List of keys for group tuple + Returns + ------- + Struct with: + - ``groups`` : Dict of : + - ``keys`` : List of keys for group tuple """ butler = patchDataRef.getButler() tempExpKeys = butler.getKeys(datasetType=tempExpDatasetType) @@ -87,9 +98,17 @@ def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepC def getGroupDataId(groupTuple, keys): """Reconstitute a data identifier from a tuple and corresponding keys - @param groupTuple: Tuple with values specifying a group - @param keys: List of keys for group tuple - @return Data identifier dict + Parameters + ---------- + groupTuple: + Tuple with values specifying a group + keys: + List of keys for group tuple + + Returns + ------- + dict : + Data identifier dict """ if len(groupTuple) != len(keys): raise RuntimeError("Number of values (%d) and keys (%d) do not match" % (len(groupTuple), len(keys))) @@ -99,11 +118,21 @@ def getGroupDataId(groupTuple, keys): def getGroupDataRef(butler, datasetType, groupTuple, keys): """Construct a data reference from a tuple and corresponding keys - @param butler: Data butler - @param datasetType: Name of dataset - @param groupTuple: Tuple with values specifying a group - @param keys: List of keys for group tuple - @return Data reference + Parameters + ---------- + butler: + Data butler + datasetType: + Name of dataset + groupTuple: + Tuple with values specifying a group + keys: + List of keys for group tuple + + Returns + ------- + butler.dataRef : + Data reference """ dataId = getGroupDataId(groupTuple, keys) return butler.dataRef(datasetType=datasetType, dataId=dataId) diff --git a/python/lsst/pipe/tasks/coaddInputRecorder.py b/python/lsst/pipe/tasks/coaddInputRecorder.py index ea6b4c488..ff0a9b18d 100644 --- a/python/lsst/pipe/tasks/coaddInputRecorder.py +++ b/python/lsst/pipe/tasks/coaddInputRecorder.py @@ -65,16 +65,31 @@ class CoaddTempExpInputRecorder: CoaddTempExp. This will contain a single 'visit' record for the CoaddTempExp and a number of 'ccd' records. + Parameters + ---------- + task : + The CoaddInputRecorderTask that is utilising us + visitId : + Identifier (integer) for the visit + num : + Number of CCDs for this visit that overlap this + patch (for reserving memory) + Should generally be created by calling CoaddInputRecorderTask.makeCoaddTempExp(). """ def __init__(self, task, visitId, num=0): """Constructor - @param task The CoaddInputRecorderTask that is utilising us - @param visitId Identifier (integer) for the visit - @param num Number of CCDs for this visit that overlap this - patch (for reserving memory) + Parameters + ---------- + task : + The CoaddInputRecorderTask that is utilising us + visitId : + Identifier (integer) for the visit + num : + Number of CCDs for this visit that overlap this + patch (for reserving memory) """ self.task = task self.coaddInputs = self.task.makeCoaddInputs() @@ -87,14 +102,19 @@ def __init__(self, task, visitId, num=0): def addCalExp(self, calExp, ccdId, nGoodPix): """Add a 'ccd' record for a calexp just added to the CoaddTempExp - @param[in] calExp Calibrated exposure just added to the CoaddTempExp, or None in case of - failures that should nonetheless be tracked. Should be the original - calexp, in that it should contain the original Psf and Wcs, not the - warped and/or matched ones. - @param[in] ccdId A unique numeric ID for the Exposure. - @param[in] nGoodPix Number of good pixels this image will contribute to the CoaddTempExp. - If saveEmptyCcds is not set and this value is zero, no record will be - added. + Parameters + ---------- + calExp : + Calibrated exposure just added to the CoaddTempExp, or None in case of + failures that should nonetheless be tracked. Should be the original + calexp, in that it should contain the original Psf and Wcs, not the + warped and/or matched ones. + ccdId : + A unique numeric ID for the Exposure. + nGoodPix : + Number of good pixels this image will contribute to the CoaddTempExp. + If saveEmptyCcds is not set and this value is zero, no record will be + added. """ if nGoodPix == 0 and not self.task.config.saveEmptyCcds: return @@ -117,11 +137,15 @@ def addCalExp(self, calExp, ccdId, nGoodPix): def finish(self, coaddTempExp, nGoodPix=None): """Finish creating the CoaddInputs for a CoaddTempExp. - @param[in,out] coaddTempExp Exposure object from which to obtain the PSF, WCS, and bounding - box for the entry in the 'visits' table. On return, the completed - CoaddInputs object will be attached to it. - @param[in] nGoodPix Total number of good pixels in the CoaddTempExp; ignored unless - saveVisitGoodPix is true. + Parameters + ---------- + coaddTempExp : + Exposure object from which to obtain the PSF, WCS, and bounding + box for the entry in the 'visits' table. On return, the completed + CoaddInputs object will be attached to it. + nGoodPix : + Total number of good pixels in the CoaddTempExp; ignored unless + saveVisitGoodPix is true. """ self._setExposureInfoInRecord(exposure=coaddTempExp, record=self.visitRecord) if self.task.config.saveVisitGoodPix: @@ -137,8 +161,12 @@ def finish(self, coaddTempExp, nGoodPix=None): def _setExposureInfoInRecord(self, exposure, record): """Set exposure info and bbox in an ExposureTable record - @param[in] exposure exposure whose info is to be recorded - @param[in,out] record record of an ExposureTable to set + Parameters + ---------- + exposure : + exposure whose info is to be recorded + record : + record of an ExposureTable to set """ info = exposure.getInfo() record.setPsf(info.getPsf()) @@ -203,6 +231,8 @@ def addVisitToCoadd(self, coaddInputs, coaddTempExp, weight): Note that the passed coaddTempExp may be a subimage, but that this method will only be called for the first subimage + Notes + ----- Returns the record for the visit to allow subclasses to fill in additional fields. Warns and returns None if the inputRecorder catalogs for the coaddTempExp are not usable. """ diff --git a/python/lsst/pipe/tasks/colorterms.py b/python/lsst/pipe/tasks/colorterms.py index ae9fbfe28..3ee4d9619 100644 --- a/python/lsst/pipe/tasks/colorterms.py +++ b/python/lsst/pipe/tasks/colorterms.py @@ -37,10 +37,12 @@ class ColortermNotFoundError(LookupError): class Colorterm(Config): - """!Colorterm correction for one pair of filters + """Colorterm correction for one pair of filters - The transformed magnitude p' is given by - p' = primary + c0 + c1*(primary - secondary) + c2*(primary - secondary)**2 + Notes + ----- + The transformed magnitude p' is given by: + p' = primary + c0 + c1*(primary - secondary) + c2*(primary - secondary)**2 To construct a Colorterm, use keyword arguments: Colorterm(primary=primaryFilterName, secondary=secondaryFilterName, c0=c0value, c1=c1Coeff, c2=c2Coeff) @@ -59,20 +61,35 @@ class Colorterm(Config): c2 = Field(dtype=float, default=0.0, doc="Second-order parameter") def transformSource(self, source): - """!Transform the brightness of a source - - @param[in] source source whose brightness is to be converted; must support get(filterName) - (e.g. source.get("r")) method, as do afw::table::Source and dicts. - @return the transformed source magnitude + """Transform the brightness of a source + + Parameters + ---------- + source : + source whose brightness is to be converted; must support get(filterName) + (e.g. source.get("r")) method, as do afw::table::Source and dicts. + + Returns + ------- + self.transformMags : + the transformed source magnitude """ return self.transformMags(source.get(self.primary), source.get(self.secondary)) def transformMags(self, primary, secondary): - """!Transform brightness - - @param[in] primary brightness in primary filter (magnitude) - @param[in] secondary brightness in secondary filter (magnitude) - @return the transformed brightness (as a magnitude) + """Transform brightness + + Parameters + ---------- + primary : + brightness in primary filter (magnitude) + secondary : + brightness in secondary filter (magnitude) + + Returns + ------- + ``primary + self.c0 + color*(self.c1 + color*self.c2)`` : + the transformed brightness (as a magnitude) """ color = primary - secondary return primary + self.c0 + color*(self.c1 + color*self.c2) @@ -82,20 +99,28 @@ def propagateFluxErrors(self, primaryFluxErr, secondaryFluxErr): class ColortermDict(Config): - """!A mapping of filterName to Colorterm + """A mapping of filterName to Colorterm + Notes + ----- Different reference catalogs may need different ColortermDicts; see ColortermLibrary To construct a ColortermDict use keyword arguments: ColortermDict(data=dataDict) where dataDict is a Python dict of filterName: Colorterm + + Examples + -------- For example: - ColortermDict(data={ - 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883), - 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248), - 'i': Colorterm(primary="i", secondary="z", c0= 0.00130204, c1=-0.16922042, c2=-0.01374245), - }) - The constructor will likely be simplified at some point. + + .. code-block:: none + + ColortermDict(data={ + 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883), + 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248), + 'i': Colorterm(primary="i", secondary="z", c0= 0.00130204, c1=-0.16922042, c2=-0.01374245), + }) + The constructor will likely be simplified at some point. This is subclass of Config. That is a bit of a hack to make it easy to store the data in an appropriate obs_* package as a config override file. In the long term some other @@ -110,30 +135,37 @@ class ColortermDict(Config): class ColortermLibrary(Config): - """!A mapping of photometric reference catalog name or glob to ColortermDict + """A mapping of photometric reference catalog name or glob to ColortermDict + Notes + ----- This allows photometric calibration using a variety of reference catalogs. To construct a ColortermLibrary, use keyword arguments: ColortermLibrary(data=dataDict) where dataDict is a Python dict of catalog_name_or_glob: ColortermDict - For example: - ColortermLibrary(data = { - "hsc*": ColortermDict(data={ - 'g': Colorterm(primary="g", secondary="g"), - 'r': Colorterm(primary="r", secondary="r"), - ... - }), - "sdss*": ColortermDict(data={ - 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883), - 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248), - ... - }), - }) + Examples + -------- + + .. code-block:: none + + For example: + ColortermLibrary(data = { + "hsc": ColortermDict(data={ + 'g': Colorterm(primary="g", secondary="g"), + 'r': Colorterm(primary="r", secondary="r"), + ... + }), + "sdss": ColortermDict(data={ + 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883), + 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248), + ... + }), + }) This is subclass of Config. That is a bit of a hack to make it easy to store the data - in an appropriate obs_* package as a config override file. In the long term some other + in an appropriate obs package as a config override file. In the long term some other means of persistence will be used, at which point the constructor can be made saner. """ data = ConfigDictField( @@ -144,24 +176,36 @@ class ColortermLibrary(Config): ) def getColorterm(self, filterName, photoCatName, doRaise=True): - """!Get the appropriate Colorterm from the library + """Get the appropriate Colorterm from the library Use dict of color terms in the library that matches the photoCatName. If the photoCatName exactly matches an entry in the library, that dict is used; otherwise if the photoCatName matches a single glob (shell syntax, - e.g., "sdss-*" will match "sdss-dr8"), then that is used. If there is no + e.g., "sdss-" will match "sdss-dr8"), then that is used. If there is no exact match and no unique match to the globs, raise an exception. - @param filterName name of filter - @param photoCatName name of photometric reference catalog from which to retrieve the data. + Parameters + ---------- + filterName : + name of filter + photoCatName : + name of photometric reference catalog from which to retrieve the data. This argument is not glob-expanded (but the catalog names in the library are, if no exact match is found). - @param[in] doRaise if True then raise ColortermNotFoundError if no suitable Colorterm found; + doRaise : + if True then raise ColortermNotFoundError if no suitable Colorterm found; if False then return a null Colorterm with filterName as the primary and secondary filter - @return the appropriate Colorterm - @throw ColortermNotFoundError if no suitable Colorterm found and doRaise true; - other exceptions may be raised for unexpected errors, regardless of the value of doRaise + Returns + ------- + ctDict : + the appropriate Colorterm + + Raises + ------ + ColortermNotFoundError + if no suitable Colorterm found and doRaise true; + other exceptions may be raised for unexpected errors, regardless of the value of doRaise """ try: trueRefCatName = None diff --git a/python/lsst/pipe/tasks/dcrAssembleCoadd.py b/python/lsst/pipe/tasks/dcrAssembleCoadd.py index 6dcf34aa9..04992af86 100644 --- a/python/lsst/pipe/tasks/dcrAssembleCoadd.py +++ b/python/lsst/pipe/tasks/dcrAssembleCoadd.py @@ -302,6 +302,8 @@ def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None): """Assemble the coadd. + Notes + ----- Requires additional inputs Struct ``supplementaryData`` to contain a ``templateCoadd`` that serves as the model of the static sky. @@ -321,7 +323,7 @@ def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, Once the ``DcrModel`` reaches convergence or the maximum number of iterations has been reached, fill the metadata for each subfilter - image and make them proper ``coaddExposure``s. + image and make them proper ``coaddExposure`` . Parameters ---------- @@ -335,7 +337,6 @@ def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, The weight to give each input exposure in the coadd supplementaryData : `lsst.pipe.base.Struct` Result struct returned by ``makeSupplementaryData`` with components: - - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) Returns diff --git a/python/lsst/pipe/tasks/exampleCmdLineTask.py b/python/lsst/pipe/tasks/exampleCmdLineTask.py index 3c4f61ea4..64239a363 100755 --- a/python/lsst/pipe/tasks/exampleCmdLineTask.py +++ b/python/lsst/pipe/tasks/exampleCmdLineTask.py @@ -37,7 +37,7 @@ class ExampleCmdLineConfig(pexConfig.Config): - """!Configuration for ExampleCmdLineTask + """Configuration for ExampleCmdLineTask """ stats = pexConfig.ConfigurableField( doc="Subtask to compute statistics of an image", @@ -52,76 +52,78 @@ class ExampleCmdLineConfig(pexConfig.Config): class ExampleCmdLineTask(pipeBase.CmdLineTask): - r"""!Example command-line task that computes simple statistics on an image + # """!Example command-line task that computes simple statistics on an image - \section pipeTasks_ExampleCmdLineTask_Contents Contents + # \section pipeTasks_ExampleCmdLineTask_Contents Contents - - \ref pipeTasks_ExampleCmdLineTask_Purpose - - \ref pipeTasks_ExampleCmdLineTask_Config - - \ref pipeTasks_ExampleCmdLineTask_Debug - - \ref pipeTasks_ExampleCmdLineTask_Example + # - \ref pipeTasks_ExampleCmdLineTask_Purpose + # - \ref pipeTasks_ExampleCmdLineTask_Config + # - \ref pipeTasks_ExampleCmdLineTask_Debug + # - \ref pipeTasks_ExampleCmdLineTask_Example - \section pipeTasks_ExampleCmdLineTask_Purpose Description + # \section pipeTasks_ExampleCmdLineTask_Purpose Description - \copybrief ExampleCmdLineTask + # \copybrief ExampleCmdLineTask - This task was written as an example for the documents \ref pipeTasks_writeTask - and \ref pipeTasks_writeCmdLineTask. - The task reads in a "calexp" (a calibrated science \ref lsst::afw::image::Exposure "exposure"), - computes statistics on the image plane, and logs and returns the statistics. - In addition, if debugging is enabled, it displays the image in ds9. + # This task was written as an example for the documents \ref pipeTasks_writeTask + # and \ref pipeTasks_writeCmdLineTask. + # The task reads in a "calexp" (a calibrated science \ref lsst::afw::image::Exposure "exposure"), + # computes statistics on the image plane, and logs and returns the statistics. + # In addition, if debugging is enabled, it displays the image in ds9. - The image statistics are computed using a subtask, in order to show how to call subtasks and how to - \ref pipeBase_argumentParser_retargetSubtasks "retarget" (replace) them with variant subtasks. + # The image statistics are computed using a subtask, in order to show how to call subtasks and how to + # \ref pipeBase_argumentParser_retargetSubtasks "retarget" (replace) them with variant subtasks. - The main method is \ref ExampleCmdLineTask.runDataRef "runDataRef". + # The main method is \ref ExampleCmdLineTask.runDataRef "runDataRef". - \section pipeTasks_ExampleCmdLineTask_Config Configuration parameters + # \section pipeTasks_ExampleCmdLineTask_Config Configuration parameters - See \ref ExampleCmdLineConfig + # See \ref ExampleCmdLineConfig - \section pipeTasks_ExampleCmdLineTask_Debug Debug variables + # \section pipeTasks_ExampleCmdLineTask_Debug Debug variables - This task supports the following debug variables: -
-
`display` -
If True then display the exposure in ds9 -
+ # This task supports the following debug variables: + #
+ #
`display` + #
If True then display the exposure in ds9 + #
- To enable debugging, see \ref baseDebug. + # To enable debugging, see \ref baseDebug. - \section pipeTasks_ExampleCmdLineTask_Example A complete example of using ExampleCmdLineTask + # \section pipeTasks_ExampleCmdLineTask_Example A complete example of using ExampleCmdLineTask - This code is in examples/exampleCmdLineTask.py, and can be run as follows: - \code - examples/exampleCmdLineTask.py $OBS_TEST_DIR/data/input --id - # that will process all data; you can also try any combination of these flags: - --id filter=g - --config doFail=True --doraise - --show config data - \endcode - """ + # This code is in examples/exampleCmdLineTask.py, and can be run as follows: + # \code + # examples/exampleCmdLineTask.py $OBS_TEST_DIR/data/input --id + # # that will process all data; you can also try any combination of these flags: + # --id filter=g + # --config doFail=True --doraise + # --show config data + # \endcode + # """ ConfigClass = ExampleCmdLineConfig _DefaultName = "exampleTask" def __init__(self, *args, **kwargs): - """Construct an ExampleCmdLineTask - - Call the parent class constructor and make the "stats" subtask from the config field of the same name. - """ pipeBase.CmdLineTask.__init__(self, *args, **kwargs) self.makeSubtask("stats") @pipeBase.timeMethod def runDataRef(self, dataRef): - """!Compute a few statistics on the image plane of an exposure - - @param dataRef: data reference for a calibrated science exposure ("calexp") - @return a pipeBase Struct containing: - - mean: mean of image plane - - meanErr: uncertainty in mean - - stdDev: standard deviation of image plane - - stdDevErr: uncertainty in standard deviation + """Compute a few statistics on the image plane of an exposure + + Parameters + ---------- + dataRef : + data reference for a calibrated science exposure ("calexp") + + Returns + ------- + a pipeBase Struct containing : + - ``mean`` : mean of image plane + - ``meanErr`` : uncertainty in mean + - ``stdDev`` : standard deviation of image plane + - ``stdDevErr`` : uncertainty in standard deviation """ self.log.info("Processing data ID %s" % (dataRef.dataId,)) if self.config.doFail: @@ -143,7 +145,7 @@ def runDataRef(self, dataRef): return self.stats.run(maskedImage) def _getConfigName(self): - """!Get the name prefix for the task config's dataset type, or None to prevent persisting the config + """Get the name prefix for the task config's dataset type, or None to prevent persisting the config This override returns None to avoid persisting metadata for this trivial task. @@ -151,23 +153,25 @@ def _getConfigName(self): The default CmdLineTask._getConfigName returns _DefaultName, which for this task would result in a dataset name of "exampleTask_config". + Notes + ----- Normally you can use the default CmdLineTask._getConfigName, but here are two reasons why you might want to override it: - If you do not want your task to write its config, then have the override return None. - That is done for this example task, because I didn't want to clutter up the - repository with config information for a trivial task. + That is done for this example task, because I didn't want to clutter up the + repository with config information for a trivial task. - If the default name would not be unique. An example is - \ref lsst.pipe.tasks.makeSkyMap.MakeSkyMapTask "MakeSkyMapTask": it makes a - \ref lsst.skymap.SkyMap "sky map" (sky pixelization for a coadd) - for any of several different types of coadd, such as deep or goodSeeing. - As such, the name of the persisted config must include the coadd type in order to be unique. + \ref lsst.pipe.tasks.makeSkyMap.MakeSkyMapTask "MakeSkyMapTask": it makes a + \ref lsst.skymap.SkyMap "sky map" (sky pixelization for a coadd) + for any of several different types of coadd, such as deep or goodSeeing. + As such, the name of the persisted config must include the coadd type in order to be unique. Normally if you override _getConfigName then you override _getMetadataName to match. """ return None def _getMetadataName(self): - """!Get the name prefix for the task metadata's dataset type, or None to prevent persisting metadata + """Get the name prefix for the task metadata's dataset type, or None to prevent persisting metadata This override returns None to avoid persisting metadata for this trivial task. @@ -175,6 +179,8 @@ def _getMetadataName(self): The default CmdLineTask._getConfigName returns _DefaultName, which for this task would result in a dataset name of "exampleTask_metadata". + Notes + ----- See the description of _getConfigName for reasons to override this method. """ return None diff --git a/python/lsst/pipe/tasks/exampleStatsTasks.py b/python/lsst/pipe/tasks/exampleStatsTasks.py index fc00f2d2a..113b18dd1 100755 --- a/python/lsst/pipe/tasks/exampleStatsTasks.py +++ b/python/lsst/pipe/tasks/exampleStatsTasks.py @@ -40,7 +40,7 @@ class ExampleSigmaClippedStatsConfig(pexConfig.Config): - """!Configuration for ExampleSigmaClippedStatsTask + """Configuration for ExampleSigmaClippedStatsTask """ badMaskPlanes = pexConfig.ListField( dtype=str, @@ -61,46 +61,16 @@ class ExampleSigmaClippedStatsConfig(pexConfig.Config): class ExampleSigmaClippedStatsTask(pipeBase.Task): - r"""!Example task to compute sigma-clipped mean and standard deviation of an image - - \section pipeTasks_ExampleSigmaClippedStatsTask_Contents Contents - - - \ref pipeTasks_ExampleSigmaClippedStatsTask_Purpose - - \ref pipeTasks_ExampleSigmaClippedStatsTask_Config - - \ref pipeTasks_ExampleSigmaClippedStatsTask_Debug - - \ref pipeTasks_ExampleSigmaClippedStatsTask_Example - - \section pipeTasks_ExampleSigmaClippedStatsTask_Purpose Description - - \copybrief ExampleSigmaClippedStatsTask + """Example task to compute sigma-clipped mean and standard deviation of an image This is a simple example task designed to be run as a subtask by ExampleCmdLineTask. See also ExampleSimpleStatsTask as a variant that is even simpler. - - The main method is \ref ExampleSigmaClippedStatsTask.run "run". - - \section pipeTasks_ExampleSigmaClippedStatsTask_Config Configuration parameters - - See \ref ExampleSigmaClippedStatsConfig - - \section pipeTasks_ExampleSigmaClippedStatsTask_Debug Debug variables - - This task has no debug variables. - - \section pipeTasks_ExampleSigmaClippedStatsTask_Example A complete example - of using ExampleSigmaClippedStatsTask - - This code is in examples/exampleStatsTask.py (this one example runs both - ExampleSigmaClippedStatsTask and ExampleSimpleStatsTask), and can be run as: - \code - examples/exampleStatsTask.py [fitsFile] - \endcode """ ConfigClass = ExampleSigmaClippedStatsConfig _DefaultName = "exampleSigmaClippedStats" def __init__(self, *args, **kwargs): - """!Construct an ExampleSigmaClippedStatsTask + """Construct an ExampleSigmaClippedStatsTask The init method may compute anything that that does not require data. In this case we create a statistics control object using the config @@ -117,14 +87,19 @@ def __init__(self, *args, **kwargs): @pipeBase.timeMethod def run(self, maskedImage): - """!Compute and return statistics for a masked image - - @param[in] maskedImage: masked image (an lsst::afw::MaskedImage) - @return a pipeBase Struct containing: - - mean: mean of image plane - - meanErr: uncertainty in mean - - stdDev: standard deviation of image plane - - stdDevErr: uncertainty in standard deviation + """Compute and return statistics for a masked image + + Parameters + ---------- + maskedImage : + masked image (an lsst::afw::MaskedImage) + Returns + ------- + a pipeBase Struct containing : + - ``mean`` : mean of image plane + - ``meanErr`` : uncertainty in mean + - ``stdDev`` : standard deviation of image plane + - ``stdDevErr`` : uncertainty in standard deviation """ statObj = afwMath.makeStatistics(maskedImage, afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.ERRORS, self._statsControl) @@ -141,40 +116,12 @@ def run(self, maskedImage): class ExampleSimpleStatsTask(pipeBase.Task): - r"""!Example task to compute mean and standard deviation of an image - - \section pipeTasks_ExampleSimpleStatsTask_Contents Contents - - - \ref pipeTasks_ExampleSimpleStatsTask_Purpose - - \ref pipeTasks_ExampleSimpleStatsTask_Config - - \ref pipeTasks_ExampleSimpleStatsTask_Debug - - \ref pipeTasks_ExampleSimpleStatsTask_Example - - \section pipeTasks_ExampleSimpleStatsTask_Purpose Description - - \copybrief ExampleSimpleStatsTask + """Example task to compute mean and standard deviation of an image This was designed to be run as a subtask by ExampleCmdLineTask. It is about as simple as a task can be; it has no configuration parameters and requires no special initialization. See also ExampleSigmaClippedStatsTask as a variant that is slightly more complicated. - The main method is \ref ExampleSimpleTask.run "run". - - \section pipeTasks_ExampleSimpleStatsTask_Config Configuration parameters - - This task has no configuration parameters. - - \section pipeTasks_ExampleSimpleStatsTask_Debug Debug variables - - This task has no debug variables. - - \section pipeTasks_ExampleSimpleStatsTask_Example A complete example of using ExampleSimpleStatsTask - - This code is in examples/exampleStatsTask.py (this one example runs both - ExampleSigmaClippedStatsTask and ExampleSimpleStatsTask), and can be run as: - \code - examples/exampleStatsTask.py [fitsFile] - \endcode """ # Even a task with no configuration requires setting ConfigClass ConfigClass = pexConfig.Config @@ -190,14 +137,20 @@ class ExampleSimpleStatsTask(pipeBase.Task): # and all of its subtasks whenver the task is run. @pipeBase.timeMethod def run(self, maskedImage): - """!Compute and return statistics for a masked image - - @param[in] maskedImage: masked image (an lsst::afw::MaskedImage) - @return a pipeBase Struct containing: - - mean: mean of image plane - - meanErr: uncertainty in mean - - stdDev: standard deviation of image plane - - stdDevErr: uncertainty in standard deviation + """Compute and return statistics for a masked image + + Parameters + ---------- + maskedImage : + masked image (an lsst::afw::MaskedImage) + + Returns + ------- + a pipeBase Struct containing : + - mean: mean of image plane + - meanErr: uncertainty in mean + - stdDev: standard deviation of image plane + - stdDevErr: uncertainty in standard deviation """ self._statsControl = afwMath.StatisticsControl() statObj = afwMath.makeStatistics(maskedImage, afwMath.MEAN | afwMath.STDEV | afwMath.ERRORS, diff --git a/python/lsst/pipe/tasks/fakes.py b/python/lsst/pipe/tasks/fakes.py index 8e3daa796..e922b60c6 100644 --- a/python/lsst/pipe/tasks/fakes.py +++ b/python/lsst/pipe/tasks/fakes.py @@ -41,21 +41,19 @@ class BaseFakeSourcesTask(lsst.pipe.base.Task, metaclass=abc.ABCMeta): This is an abstract base class (abc) and is not intended to be directly used. To create a fake sources injector, create a child class and re-implement the required methods. + + Subclasses that define their own __init__ should simply forward all arguments to the base + class constructor. They can then assume self.config is an instance of their ConfigClass. + + If an external catalog is used to add sources consistently to multiple overlapping images, + that catalog should generally be loaded and attached to self here, so it can be used + multiple times by the run() method. """ ConfigClass = BaseFakeSourcesConfig _DefaultName = "baseFakeSources" def __init__(self, **kwargs): - """Initialize the Task. - - Subclasses that define their own __init__ should simply forward all arguments to the base - class constructor. They can then assume self.config is an instance of their ConfigClass. - - If an external catalog is used to add sources consistently to multiple overlapping images, - that catalog should generally be loaded and attached to self here, so it can be used - multiple times by the run() method. - """ lsst.pipe.base.Task.__init__(self, **kwargs) lsst.afw.image.Mask[lsst.afw.image.MaskPixel].addMaskPlane(self.config.maskPlaneName) self.bitmask = lsst.afw.image.Mask[lsst.afw.image.MaskPixel]\ diff --git a/python/lsst/pipe/tasks/getRepositoryData.py b/python/lsst/pipe/tasks/getRepositoryData.py index 5e8f9d2c4..f308d2de1 100755 --- a/python/lsst/pipe/tasks/getRepositoryData.py +++ b/python/lsst/pipe/tasks/getRepositoryData.py @@ -44,14 +44,19 @@ def getTargetList(parsedCmd): def __call__(self, dataRefList): """Run GetRepositoryDataTask.run on a single target - @param dataRefList: argument dict for run; contains one key: dataRefList - - @return: - - None if doReturnResults false - - A pipe_base Struct containing these fields if doReturnResults true: - - dataRefList: the argument dict sent to runDataRef - - metadata: task metadata after execution of runDataRef - - result: result returned by task runDataRef + Parameters + ---------- + dataRefList : + argument dict for run; contains one key: dataRefList + + Returns + ------- + None + if doReturnResults false + Struct : + - ``dataRefList`` : the argument dict sent to runDataRef + - ``metadata`` : task metadata after execution of runDataRef + - ``result`` : result returned by task runDataRef """ task = self.TaskClass(config=self.config, log=self.log) result = task.runDataRef(dataRefList) @@ -78,17 +83,26 @@ def __init__(self, *args, **kwargs): def runDataRef(self, dataRefList): """Get data from a repository for a collection of data references - @param dataRefList: a list of data references + Parameters + ---------- + dataRefList : a list of data references """ raise NotImplementedError("subclass must specify a run method") def getIdList(self, dataRefList): """Get a list of data IDs in a form that can be used as dictionary keys - @param dataRefList: a list of data references - @return a pipe_base Struct with fields: - - idKeyTuple: a tuple of dataRef data ID keys - - idValList: a list of data ID value tuples, each tuple contains values in the order in idKeyTuple + Parameters + ---------- + dataRefList : + a list of data references + + Returns + ------- + result : ``lsst.pipe.base.Struct`` + - ``idKeyTuple`` : a tuple of dataRef data ID keys + - ``idValList`` : a list of data ID value tuples, + each tuple contains values in the order in idKeyTuple """ if not dataRefList: raise RuntimeError("No data refs") @@ -107,20 +121,34 @@ def getIdList(self, dataRefList): def getDataList(self, dataRefList, datasetType): """Retrieve a list of data - @param dataRefList: a list of data references - @param datasetType: datasetType of data to be retrieved - @return a list of data, one entry per dataRef in dataRefList (in order) + Parameters + ---------- + dataRefList : + a list of data references + datasetType : + datasetType of data to be retrieved + + Returns + ------- + result : `list` + a list of data, one entry per dataRef in dataRefList (in order) """ return [dataRef.get(datasetType=datasetType) for dataRef in dataRefList] def getMetadataItems(self, dataRefList, datasetType, nameList): """Retrieve a list of dictionaries of metadata - @param dataRefList: a list of data references - @param datasetType: datasetType of metadata (or any object that supports get(name)) - @return a list of dicts of metadata: - - each entry in the list corresponds to a dataRef in dataRefList - - each dict contains name: item of metadata, for each name in nameList; + Parameters + ---------- + dataRefList: a list of data references + datasetType: datasetType of metadata (or any object that supports get(name)) + + Returns + ------- + valList : `list` of `dict` + a list of dicts of metadata: + - each entry in the list corresponds to a dataRef in dataRefList + - each dict contains name: item of metadata, for each name in nameList; numeric and string values will be returned as arrays """ valList = [] diff --git a/python/lsst/pipe/tasks/imageDifference.py b/python/lsst/pipe/tasks/imageDifference.py index 91685c2b7..71bc8d826 100644 --- a/python/lsst/pipe/tasks/imageDifference.py +++ b/python/lsst/pipe/tasks/imageDifference.py @@ -46,6 +46,8 @@ FwhmPerSigma = 2 * math.sqrt(2 * math.log(2)) IqrToSigma = 0.741 +__all__ = ('ImageDifferenceConfig', 'ImageDifferenceTaskRunner', + 'Winter2013ImageDifferenceConfig', 'Winter2013ImageDifferenceTask') class ImageDifferenceConfig(pexConfig.Config): """Config for ImageDifferenceTask @@ -229,16 +231,17 @@ def getTargetList(parsedCmd, **kwargs): class ImageDifferenceTask(pipeBase.CmdLineTask): """Subtract an image from a template and measure the result + + Parameters + ---------- + butler : + Butler object to use in constructing reference object loaders """ ConfigClass = ImageDifferenceConfig RunnerClass = ImageDifferenceTaskRunner _DefaultName = "imageDifference" def __init__(self, butler=None, **kwargs): - """!Construct an ImageDifference Task - - @param[in] butler Butler object to use in constructing reference object loaders - """ pipeBase.CmdLineTask.__init__(self, **kwargs) self.makeSubtask("getTemplate") @@ -299,26 +302,33 @@ def runDataRef(self, sensorRef, templateIdList=None): - detect sources - measure sources - @param sensorRef: sensor-level butler data reference, used for the following data products: - Input only: - - calexp - - psf - - ccdExposureId - - ccdExposureId_bits - - self.config.coaddName + "Coadd_skyMap" - - self.config.coaddName + "Coadd" - Input or output, depending on config: - - self.config.coaddName + "Diff_subtractedExp" - Output, depending on config: - - self.config.coaddName + "Diff_matchedExp" - - self.config.coaddName + "Diff_src" - - @return pipe_base Struct containing these fields: + Parameters + ---------- + sensorRef : + sensor-level butler data reference, used for the following data products: + Input only: + - calexp + - psf + - ccdExposureId + - ccdExposureId_bits + - self.config.coaddName + "Coadd_skyMap" + - self.config.coaddName + "Coadd" + Input or output, depending on config: + - self.config.coaddName + "Diff_subtractedExp" + Output, depending on config: + - self.config.coaddName + "Diff_matchedExp" + - self.config.coaddName + "Diff_src" + + Returns + ------- + result : `lsst.pipe.base.Struct` + pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; - the unpersisted version if subtraction not run but detection run - None if neither subtraction nor detection run (i.e. nothing useful done) + the unpersisted version if subtraction not run but detection run + None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run + """ self.log.info("Processing %s" % (sensorRef.dataId)) @@ -743,7 +753,9 @@ def runDataRef(self, sensorRef, templateIdList=None): def fitAstrometry(self, templateSources, templateExposure, selectSources): """Fit the relative astrometry between templateSources and selectSources - @todo remove this method. It originally fit a new WCS to the template before calling register.run + Notes + ----- + remove this method. It originally fit a new WCS to the template before calling register.run because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. It remains because a subtask overrides it. """ @@ -860,15 +872,14 @@ def setDefaults(self): ImageDifferenceConfig.setDefaults(self) self.getTemplate.retarget(GetCalexpAsTemplateTask) - class Winter2013ImageDifferenceTask(ImageDifferenceTask): - """!Image difference Task used in the Winter 2013 data challege. + """Image difference Task used in the Winter 2013 data challege. Enables testing the effects of registration shifts and scatter. For use with winter 2013 simulated images: Use --templateId visit=88868666 for sparse data - --templateId visit=22222200 for dense data (g) - --templateId visit=11111100 for dense data (i) + --templateId visit=22222200 for dense data (g) + --templateId visit=11111100 for dense data (i) """ ConfigClass = Winter2013ImageDifferenceConfig _DefaultName = "winter2013ImageDifference" diff --git a/python/lsst/pipe/tasks/ingest.py b/python/lsst/pipe/tasks/ingest.py index 3a41d5c3f..4eeb35747 100644 --- a/python/lsst/pipe/tasks/ingest.py +++ b/python/lsst/pipe/tasks/ingest.py @@ -12,6 +12,9 @@ from lsst.pipe.base import Task, InputOnlyArgumentParser from lsst.afw.fits import DEFAULT_HDU +__all__ = ('IngestArgumentParser', 'ParseConfig', 'ParseConfig', 'ParseTask', 'RegisterConfig', + 'RegistryContext', 'RegisterTask', 'IngestConfig', 'IngestTask') + class IngestArgumentParser(InputOnlyArgumentParser): """Argument parser to support ingesting images into the image repository""" @@ -54,8 +57,16 @@ def getInfo(self, filename): Here, we open the image and parse the header, but one could also look at the filename itself and derive information from that, or set values from the configuration. - @param filename Name of file to inspect - @return File properties; list of file properties for each extension + Parameters + ---------- + filename : + Name of file to inspect + + Returns + ------- + phuInfo : + infoList : + File properties; list of file properties for each extension """ md = readMetadata(filename, self.config.hdu) phuInfo = self.getInfoFromMetadata(md) @@ -85,8 +96,16 @@ def getInfo(self, filename): @staticmethod def getExtensionName(md): """ Get the name of an extension. - @param md: PropertySet like one obtained from lsst.afw.fits.readMetadata) - @return Name of the extension if it exists. None otherwise. + + Parameters + ---------- + md : + PropertySet like one obtained from lsst.afw.fits.readMetadata) + + Returns + ------- + result : `None` or `str` + Name of the extension if it exists. None otherwise. """ try: # This returns a tuple @@ -99,15 +118,23 @@ def getInfoFromMetadata(self, md, info=None): """Attempt to pull the desired information out of the header This is done through two mechanisms: - * translation: a property is set directly from the relevant header keyword - * translator: a property is set with the result of calling a method + - translation: a property is set directly from the relevant header keyword + - translator: a property is set with the result of calling a method The translator methods receive the header metadata and should return the appropriate value, or None if the value cannot be determined. - @param md FITS header - @param info File properties, to be supplemented - @return info + Parameters + ---------- + md : + FITS header + info : + File properties, to be supplemented + + Returns + ------- + info : + """ if info is None: info = {} @@ -160,10 +187,19 @@ def translate_filter(self, md): def getDestination(self, butler, info, filename): """Get destination for the file - @param butler Data butler - @param info File properties, used as dataId for the butler - @param filename Input filename - @return Destination filename + Parameters + ---------- + butler : + Data butler + info : + File properties, used as dataId for the butler + filename : + Input filename + + Returns + ------- + raw : + Destination filename """ raw = butler.get("raw_filename", info)[0] # Ensure filename is devoid of cfitsio directions about HDUs @@ -201,16 +237,20 @@ class RegistryContext: An existing registry is copied, so that it may continue to be used while we add to this new registry. Finally, the new registry is moved into the right place. + + Parameters + ---------- + registryName : + Name of registry file + createTableFunc : + Function to create tables + forceCreateTables : + Force the (re-)creation of tables? + permissions : + Permissions to set on database file """ def __init__(self, registryName, createTableFunc, forceCreateTables, permissions): - """Construct a context manager - - @param registryName: Name of registry file - @param createTableFunc: Function to create tables - @param forceCreateTables: Force the (re-)creation of tables? - @param permissions: Permissions to set on database file - """ self.registryName = registryName self.permissions = permissions @@ -264,11 +304,19 @@ class RegisterTask(Task): def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"): """Open the registry and return the connection handle. - @param directory Directory in which the registry file will be placed - @param create Clobber any existing registry and create a new one? - @param dryrun Don't do anything permanent? - @param name Filename of the registry - @return Database connection + directory : + Directory in which the registry file will be placed + create : + Clobber any existing registry and create a new one? + dryrun : + Don't do anything permanent? + name : + Filename of the registry + + Returns + ------- + result : + Database connection """ if dryrun: return fakeContext() @@ -283,8 +331,12 @@ def createTable(self, conn, table=None): One table (typically 'raw') contains information on all files, and the other (typically 'raw_visit') contains information on all visits. - @param conn Database connection - @param table Name of table to create in database + Parameters + ---------- + conn : + Database connection + table : + Name of table to create in database """ if table is None: table = self.config.table @@ -325,9 +377,14 @@ def check(self, conn, info, table=None): def addRow(self, conn, info, dryrun=False, create=False, table=None): """Add a row to the file table (typically 'raw'). - @param conn Database connection - @param info File properties to add to database - @param table Name of table in database + Parameters + ---------- + conn : + Database connection + info : + File properties to add to database + table : + Name of table in database """ if table is None: table = self.config.table @@ -350,8 +407,12 @@ def addVisits(self, conn, dryrun=False, table=None): """Generate the visits table (typically 'raw_visits') from the file table (typically 'raw'). - @param conn Database connection - @param table Name of table in database + Parameters + ---------- + conn : + Database connection + table : + Name of table in database """ if table is None: table = self.config.table @@ -397,11 +458,18 @@ def parseAndRun(cls): def ingest(self, infile, outfile, mode="move", dryrun=False): """Ingest a file into the image repository. - @param infile Name of input file - @param outfile Name of output file (file in repository) - @param mode Mode of ingest (copy/link/move/skip) - @param dryrun Only report what would occur? - @param Success boolean + Parameters + ---------- + infile : `str` + Name of input file + outfile : `str` + Name of output file (file in repository) + mode : + Mode of ingest (copy/link/move/skip) + dryrun : + Only report what would occur? + Success : `bool` + boolean """ if mode == "skip": return True @@ -467,10 +535,15 @@ def isBadId(self, info, badIdList): return False def expandFiles(self, fileNameList): - """!Expand a set of filenames and globs, returning a list of filenames + """Expand a set of filenames and globs, returning a list of filenames - @param fileNameList A list of files and glob patterns + Parameters + ---------- + fileNameList : + A list of files and glob patterns + Notes + ----- N.b. globs obey Posix semantics, so a pattern that matches nothing is returned unchanged """ filenameList = [] @@ -486,11 +559,19 @@ def expandFiles(self, fileNameList): return filenameList def runFile(self, infile, registry, args): - """!Examine and ingest a single file - - @param infile: File to process - @param args: Parsed command-line arguments - @return parsed information from FITS HDUs or None + """Examine and ingest a single file + + Parameters + ---------- + infile : + File to process + args: + Parsed command-line arguments + + Returns + ------- + result : + parsed information from FITS HDUs or None """ if self.isBadFile(infile, args.badFile): self.log.info("Skipping declared bad file %s" % infile) @@ -536,8 +617,12 @@ def run(self, args): def assertCanCopy(fromPath, toPath): """Can I copy a file? Raise an exception is space constraints not met. - @param fromPath Path from which the file is being copied - @param toPath Path to which the file is being copied + Parameters + ---------- + fromPath : + Path from which the file is being copied + toPath : + Path to which the file is being copied """ req = os.stat(fromPath).st_size st = os.statvfs(os.path.dirname(toPath)) diff --git a/python/lsst/pipe/tasks/ingestCalibs.py b/python/lsst/pipe/tasks/ingestCalibs.py index 5dce558be..4d237191a 100644 --- a/python/lsst/pipe/tasks/ingestCalibs.py +++ b/python/lsst/pipe/tasks/ingestCalibs.py @@ -7,6 +7,9 @@ from lsst.pipe.base import InputOnlyArgumentParser from lsst.pipe.tasks.ingest import RegisterTask, ParseTask, RegisterConfig, IngestTask +__all__ = ('CalibsParseTask', 'CalibsRegisterConfig', 'CalibsRegisterTask', + 'IngestCalibsArgumentParser', 'IngestCalibsConfig', 'IngestCalibsTask') + def _convertToDate(dateString): """Convert a string into a date object""" @@ -43,10 +46,19 @@ def getCalibType(self, filename): def getDestination(self, butler, info, filename): """Get destination for the file - @param butler Data butler - @param info File properties, used as dataId for the butler - @param filename Input filename - @return Destination filename + Parameters + ---------- + butler : + Data butler + info : + File properties, used as dataId for the butler + filename : + Input filename + + Returns + ------- + raw : + Destination filename """ # 'tempinfo' was added as part of DM-5466 to strip Nones from info. # The Butler should handle this behind-the-scenes in the future. @@ -98,8 +110,12 @@ def updateValidityRanges(self, conn, validity): """Loop over all tables, filters, and ccdnums, and update the validity ranges in the registry. - @param conn: Database connection - @param validity: Validity range (days) + Parameters + ---------- + conn : + Database connection + validity : + Validity range (days) """ conn.row_factory = sqlite3.Row cursor = conn.cursor() @@ -120,10 +136,16 @@ def fixSubsetValidity(self, conn, table, detectorData, validity): so that the calibration data with whose date is nearest the date of the observation is used. - @param conn: Database connection - @param table: Name of table to be selected - @param detectorData: Values identifying a detector (from columns in self.config.detector) - @param validity: Validity range (days) + Parameters + ---------- + conn : + Database connection + table : + Name of table to be selected + detectorData : + Values identifying a detector (from columns in self.config.detector) + validity : + Validity range (days) """ columns = ", ".join([self.config.calibDate, self.config.validStart, self.config.validEnd]) sql = "SELECT id, %s FROM %s" % (columns, table) diff --git a/python/lsst/pipe/tasks/ingestPgsql.py b/python/lsst/pipe/tasks/ingestPgsql.py index 6a8dc21c2..16c737a28 100644 --- a/python/lsst/pipe/tasks/ingestPgsql.py +++ b/python/lsst/pipe/tasks/ingestPgsql.py @@ -10,17 +10,22 @@ except ImportError: havePgSql = False +__all__ = ('PgsqlRegistryContext', 'PgsqlRegisterTask', 'PgsqlIngestConfig', 'PgsqlIngestTask') + class PgsqlRegistryContext(RegistryContext): """Context manager to provide a pgsql registry + + Parameters + ---------- + registryName : + Name of registry file + createTableFunc : + Function to create tables + forceCreateTables : + Force the (re-)creation of tables? """ def __init__(self, registryName, createTableFunc, forceCreateTables): - """Construct a context manager - - @param registryName: Name of registry file - @param createTableFunc: Function to create tables - @param forceCreateTables: Force the (re-)creation of tables? - """ self.registryName = registryName data = PgsqlRegistry.readYaml(registryName) self.conn = pgsql.connect(host=data["host"], port=data["port"], user=data["user"], @@ -52,10 +57,19 @@ class PgsqlRegisterTask(RegisterTask): def openRegistry(self, directory, create=False, dryrun=False): """Open the registry and return the connection handle. - @param directory Directory in which the registry file will be placed - @param create Clobber any existing registry and create a new one? - @param dryrun Don't do anything permanent? - @return Database connection + Parameters + ---------- + directory : + Directory in which the registry file will be placed + create : + Clobber any existing registry and create a new one? + dryrun : + Don't do anything permanent? + + Returns + ------- + result : + Database connection """ if dryrun: return fakeContext() @@ -72,8 +86,12 @@ def createTable(self, conn, table=None): compared to SQLite (FLOAT instead of DOUBLE, SERIAL instead of AUTOINCREMENT). - @param conn Database connection - @param table Name of table to create in database + Parameters + ---------- + conn : + Database connection + table : + Name of table to create in database """ if table is None: table = self.config.table diff --git a/python/lsst/pipe/tasks/interpImage.py b/python/lsst/pipe/tasks/interpImage.py index 13b5b4719..caeadc807 100644 --- a/python/lsst/pipe/tasks/interpImage.py +++ b/python/lsst/pipe/tasks/interpImage.py @@ -79,11 +79,17 @@ class InterpImageTask(pipeBase.Task): def _setFallbackValue(self, mi=None): """Set the edge fallbackValue for interpolation - @param[in] mi input maksedImage on which to calculate the statistics - Must be provided if fallbackValueType != "USER". - - @return fallbackValue The value set/computed based on the fallbackValueType - and negativeFallbackAllowed config parameters + Parameters + ---------- + mi: + input maksedImage on which to calculate the statistics + Must be provided if fallbackValueType != "USER". + + Returns + ------ + fallbackValue : + The value set/computed based on the fallbackValueType + and negativeFallbackAllowed config parameters """ if self.config.fallbackValueType != 'USER': assert mi, "No maskedImage provided" @@ -111,12 +117,29 @@ def _setFallbackValue(self, mi=None): @pipeBase.timeMethod def run(self, image, planeName=None, fwhmPixels=None, defects=None): - """!Interpolate in place over pixels in a maskedImage marked as bad + """Interpolate in place over pixels in a maskedImage marked as bad Pixels to be interpolated are set by either a mask planeName provided by the caller OR a defects list of type measAlg.DefectListT. If both are provided an exception is raised. + Parameters + ---------- + image : + MaskedImage OR Exposure to be interpolated + planeName : + name of mask plane over which to interpolate + If None, must provide a defects list. + fwhmPixels : + FWHM of core star (pixels) + If None the default is used, where the default + is set to the exposure psf if available + defects : + List of defects of type measAlg.DefectListT + over which to interpolate. + + Notes + ----- Note that the interpolation code in meas_algorithms currently doesn't use the input PSF (though it's a required argument), so it's not important to set the input PSF parameters exactly. This PSF is set @@ -125,15 +148,6 @@ def run(self, image, planeName=None, fwhmPixels=None, defects=None): measAlg.GaussianPsfFactory with the value of fwhmPixels (the value passed in by the caller, or the default defaultFwhm set in measAlg.GaussianPsfFactory if None). - - @param[in,out] image MaskedImage OR Exposure to be interpolated - @param[in] planeName name of mask plane over which to interpolate - If None, must provide a defects list. - @param[in] fwhmPixels FWHM of core star (pixels) - If None the default is used, where the default - is set to the exposure psf if available - @param[in] defects List of defects of type measAlg.DefectListT - over which to interpolate. """ try: maskedImage = image.getMaskedImage() diff --git a/python/lsst/pipe/tasks/makeCoaddTempExp.py b/python/lsst/pipe/tasks/makeCoaddTempExp.py index 82abecf3a..da1c5cdff 100755 --- a/python/lsst/pipe/tasks/makeCoaddTempExp.py +++ b/python/lsst/pipe/tasks/makeCoaddTempExp.py @@ -99,128 +99,148 @@ def setDefaults(self): class MakeCoaddTempExpTask(CoaddBaseTask): - r"""!Warp and optionally PSF-Match calexps onto an a common projection. + """Warp and optionally PSF-Match calexps onto an a common projection. - @anchor MakeCoaddTempExpTask_ - - @section pipe_tasks_makeCoaddTempExp_Contents Contents - - - @ref pipe_tasks_makeCoaddTempExp_Purpose - - @ref pipe_tasks_makeCoaddTempExp_Initialize - - @ref pipe_tasks_makeCoaddTempExp_IO - - @ref pipe_tasks_makeCoaddTempExp_Config - - @ref pipe_tasks_makeCoaddTempExp_Debug - - @ref pipe_tasks_makeCoaddTempExp_Example - - @section pipe_tasks_makeCoaddTempExp_Purpose Description + Notes + ----- + Description Warp and optionally PSF-Match calexps onto a common projection, by performing the following operations: + - Group calexps by visit/run - - For each visit, generate a Warp by calling method @ref makeTempExp. - makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch - on each visit + - For each visit, generate a Warp by calling method makeTempExp. - The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). + makeTempExp loops over the visit's calexps calling WarpAndPsfMatch + on each visit - @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization + The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). - @copydoc \_\_init\_\_ + Task Initialization This task has one special keyword argument: passing reuse=True will cause the task to skip the creation of warps that are already present in the output repositories. - @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task + Invoking the Task This task is primarily designed to be run from the command line. The main method is `runDataRef`, which takes a single butler data reference for the patch(es) to process. - @copydoc run + .. code-block:: none + + Create a Warp from inputs + + We iterate over the multiple calexps in a single exposure to construct + the warp (previously called a coaddTempExp) of that exposure to the + supplied tract/patch. + + Pixels that receive no pixels are set to NAN; this is not correct + (violates LSST algorithms group policy), but will be fixed up by + interpolating after the coaddition. + + @param calexpRefList: List of data references for calexps that (may) + overlap the patch of interest + @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric + information about the patch + @param visitId: integer identifier for visit, for the table that will + produce the CoaddPsf + @return a pipeBase Struct containing: + + - exposures: a dictionary containing the warps requested: + + "direct": direct warp if config.makeDirect + "psfMatched": PSF-matched warp if config.makePsfMatched WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). Only two types are available: direct (for regular Warps/Coadds) and psfMatched (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, for generating likelihood Coadds with Warps that have been correlated with their own PSF. - @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters - - See @ref MakeCoaddTempExpConfig and parameters inherited from - @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink - - @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs + Matching Guide to PSF-Matching Configs To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask - @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink + ``lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask`` ModelPsfMatchTask link is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, padding of the science PSF thumbnail and spatial sampling frequency of the PSF. - *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting + Config Guidelines: The user must specify the size of the model PSF to which to match by setting `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. In general, for a set of input images, this config should equal the FWHM of the visit with the worst seeing. The smallest it should be set to is the median FWHM. The defaults of the other config options offer a reasonable starting point. The following list presents the most common problems that arise from a misconfigured - @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink + ``lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask`` ModelPsfMatchTask and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via - ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: - Problem: Explanation. *Solution* + ```config.warpAndPsfMatch.psfMatch.kernel['AL']``` . Each item in the list is formatted as: + Problem: Explanation. Solution + + Examples + -------- + + Troublshooting PSF-Matching Configuration: - *Troublshooting PSF-Matching Configuration:* - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. - For example:_ - config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 + For example: + + >>> config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 + + Note that increasing the kernel size also increases runtime. - Note that increasing the kernel size also increases runtime. - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution - for matching kernel. _Provide the matcher with more data by either increasing - the spatial sampling by decreasing the spatial cell size,_ - config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 - config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 + for matching kernel. _Provide the matcher with more data by either increasing + the spatial sampling by decreasing the spatial cell size, + + >>> config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 + >>> config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 - _or increasing the padding around the Science PSF, for example:_ + or increasing the padding around the Science PSF, for example: - config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 + >>> config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 - Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the - matching kernel dimensions, thus increasing the number of pixels available to fit - after convolving the PSF with the matching kernel. - Optionally, for debugging the effects of padding, the level of padding may be manually - controlled by setting turning off the automatic padding and setting the number - of pixels by which to pad the PSF: + Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the + matching kernel dimensions, thus increasing the number of pixels available to fit + after convolving the PSF with the matching kernel. + Optionally, for debugging the effects of padding, the level of padding may be manually + controlled by setting turning off the automatic padding and setting the number + of pixels by which to pad the PSF: - config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True - config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 + >>> config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True + >>> config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 - Deconvolution: Matching a large PSF to a smaller PSF produces - a telltale noise pattern which looks like ripples or a brain. - _Increase the size of the requested model PSF. For example:_ - config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. + a telltale noise pattern which looks like ripples or a brain. + Increase the size of the requested model PSF. For example: + + >>> config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. - High frequency (sometimes checkered) noise: The matching basis functions are too small. - _Increase the width of the Gaussian basis functions. For example:_ - config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] - # from default [0.7, 1.5, 3.0] + Increase the width of the Gaussian basis functions. For example: + + config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] + from default [0.7, 1.5, 3.0] - @section pipe_tasks_makeCoaddTempExp_Debug Debug variables + Debug variables MakeCoaddTempExpTask has no debug output, but its subtasks do. - @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask + A complete example of using MakeCoaddTempExpTask This example uses the package ci_hsc to show how MakeCoaddTempExp fits into the larger Data Release Processing. Set up by running: + .. code-block:: none + setup ci_hsc cd $CI_HSC_DIR # if not built already: @@ -230,22 +250,29 @@ class MakeCoaddTempExpTask(CoaddBaseTask): (e.g. by building `ci_hsc` above) to generate a repository of calexps and an output respository with the desired SkyMap. The command, + .. code-block:: none + makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ - --id patch=5,4 tract=0 filter=HSC-I \ - --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ - --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ - --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 + --id patch=5,4 tract=0 filter=HSC-I \ + --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ + --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ + --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 writes a direct and PSF-Matched Warp to - - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and - - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` - respectively. - @note PSF-Matching in this particular dataset would benefit from adding + .. code-block:: none + + - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and + - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` + respectively. + + + PSF-Matching in this particular dataset would benefit from adding `--configfile ./matchingConfig.py` to the command line arguments where `matchingConfig.py` is defined by: - echo " + .. code-block:: none + config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py @@ -262,17 +289,26 @@ def __init__(self, reuse=False, **kwargs): @pipeBase.timeMethod def runDataRef(self, patchRef, selectDataList=[]): - """!Produce Coadd_Warp images by warping and optionally PSF-matching. + """Produce Coadd_Warp images by warping and optionally PSF-matching. - @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", + Parameters + ---------- + patchRef : + data reference for sky map patch. Must include keys "tract", "patch", plus the camera-specific filter key (e.g. "filter" or "band") - @return: dataRefList: a list of data references for the new Coadd_directWarps + + Returns + ------- + dataRefList : + a list of data references for the new Coadd_directWarps if direct or both warp types are requested and Coadd_psfMatchedWarps if only psfMatched warps are requested. - @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. + Notes + ----- + warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. - @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp + warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored (assembleCoadd should determine zeropoint scaling without referring to it). """ @@ -340,16 +376,25 @@ def run(self, calexpRefList, skyInfo, visitId=0): (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. - @param calexpRefList: List of data references for calexps that (may) + Parameters + ---------- + calexpRefList : + List of data references for calexps that (may) overlap the patch of interest - @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric + skyInfo : + Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch - @param visitId: integer identifier for visit, for the table that will + visitId : + integer identifier for visit, for the table that will produce the CoaddPsf - @return a pipeBase Struct containing: - - exposures: a dictionary containing the warps requested: - "direct": direct warp if config.makeDirect - "psfMatched": PSF-matched warp if config.makePsfMatched + + Returns + ------- + result : `lsst.pipe.base.Struct` + a pipeBase Struct containing: + - ``exposures``: a dictionary containing the warps requested: + - "direct": direct warp if config.makeDirect + - "psfMatched": PSF-matched warp if config.makePsfMatched """ warpTypeList = self.getWarpTypeList() diff --git a/python/lsst/pipe/tasks/makeDiscreteSkyMap.py b/python/lsst/pipe/tasks/makeDiscreteSkyMap.py index ecdc29f7f..4c89f337a 100755 --- a/python/lsst/pipe/tasks/makeDiscreteSkyMap.py +++ b/python/lsst/pipe/tasks/makeDiscreteSkyMap.py @@ -30,6 +30,8 @@ from lsst.skymap import DiscreteSkyMap, BaseSkyMap from lsst.pipe.base import ArgumentParser +__all__ = ('MakeDiscreteSkyMapConfig', 'MakeDiscreteSkyMapRunner', 'MakeDiscreteSkyMapTask') + class MakeDiscreteSkyMapConfig(pexConfig.Config): """Config for MakeDiscreteSkyMapTask @@ -66,6 +68,8 @@ def setDefaults(self): class MakeDiscreteSkyMapRunner(pipeBase.TaskRunner): """Run a task with all dataRefs at once, rather than one dataRef at a time. + Notes + ----- Call the run method of the task using two positional arguments: - butler: data butler - dataRefList: list of all dataRefs, @@ -76,14 +80,19 @@ def getTargetList(parsedCmd): def __call__(self, args): """ - @param args Arguments for Task.run() - - @return: - - None if self.doReturnResults false - - A pipe_base Struct containing these fields if self.doReturnResults true: - - dataRef: the provided data reference - - metadata: task metadata after execution of run - - result: result returned by task run, or None if the task fails + Parameters + ---------- + args : + Arguments for Task.run() + + Returns + ------- + result : `None` or `Struct` + None if self.doReturnResults false + A pipe_base Struct containing these fields if self.doReturnResults true: + - ``dataRef`` : the provided data reference + - ``metadata``: task metadata after execution of run + - ``result`` : result returned by task run, or None if the task fails """ butler, dataRefList = args task = self.TaskClass(config=self.config, log=self.log) @@ -116,7 +125,7 @@ def __call__(self, args): class MakeDiscreteSkyMapTask(pipeBase.CmdLineTask): - """!Make a DiscreteSkyMap in a repository, using the bounding box of a set of calexps. + """Make a DiscreteSkyMap in a repository, using the bounding box of a set of calexps. The command-line and run signatures and config are sufficiently different from MakeSkyMapTask that we don't inherit from it, but it is a replacement, so we use the same config/metadata names. @@ -130,12 +139,20 @@ def __init__(self, **kwargs): @pipeBase.timeMethod def runDataRef(self, butler, dataRefList): - """!Make a skymap from the bounds of the given set of calexps. - - @param[in] butler data butler used to save the SkyMap - @param[in] dataRefList dataRefs of calexps used to determine the size and pointing of the SkyMap - @return a pipeBase Struct containing: - - skyMap: the constructed SkyMap + """Make a skymap from the bounds of the given set of calexps. + + Parameters + ---------- + butler : + data butler used to save the SkyMap + dataRefList : + dataRefs of calexps used to determine the size and pointing of the SkyMap + + Returns + ------- + result : + a pipeBase Struct containing: + - ``skyMap`` : the constructed SkyMap """ self.log.info("Extracting bounding boxes of %d images" % len(dataRefList)) points = [] diff --git a/python/lsst/pipe/tasks/makeSkyMap.py b/python/lsst/pipe/tasks/makeSkyMap.py index 02f6d32af..f819e1288 100755 --- a/python/lsst/pipe/tasks/makeSkyMap.py +++ b/python/lsst/pipe/tasks/makeSkyMap.py @@ -27,6 +27,8 @@ import lsst.pipe.base as pipeBase from lsst.skymap import skyMapRegistry +__all__ = ('MakeSkyMapConfig', 'MakeSkyMapRunner', 'MakeSkyMapTask') + class MakeSkyMapConfig(pexConfig.Config): """Config for MakeSkyMapTask @@ -81,7 +83,7 @@ def __call__(self, butler): class MakeSkyMapTask(pipeBase.CmdLineTask): - """!Make a sky map in a repository + """Make a sky map in a repository Making a sky map in a repository is a prerequisite for making a coadd, since the sky map is used as the pixelization for the coadd. @@ -95,11 +97,18 @@ def __init__(self, **kwargs): @pipeBase.timeMethod def runDataRef(self, butler): - """!Make a skymap, persist it (optionally) and log some information about it - - @param[in] butler data butler - @return a pipeBase Struct containing: - - skyMap: the constructed SkyMap + """Make a skymap, persist it (optionally) and log some information about it + + Parameters + ---------- + butler : + data butler + + Returns + ------- + result : + a pipeBase Struct containing: + - ``skyMap``: the constructed SkyMap """ skyMap = self.config.skyMap.apply() self.logSkyMapInfo(skyMap) @@ -110,9 +119,11 @@ def runDataRef(self, butler): ) def logSkyMapInfo(self, skyMap): - """!Log information about a sky map + """Log information about a sky map - @param[in] skyMap sky map (an lsst.skyMap.SkyMap) + Parameters + ---------- + skyMap : `lsst.skyMap.SkyMap` """ self.log.info("sky map has %s tracts" % (len(skyMap),)) for tractInfo in skyMap: diff --git a/python/lsst/pipe/tasks/matchBackgrounds.py b/python/lsst/pipe/tasks/matchBackgrounds.py index 7f0076430..95d07d2b6 100644 --- a/python/lsst/pipe/tasks/matchBackgrounds.py +++ b/python/lsst/pipe/tasks/matchBackgrounds.py @@ -26,6 +26,8 @@ import lsst.pipe.base as pipeBase import lsstDebug +__all__ = ('MatchBackgroundsConfig', 'MatchBackgroundsTask', 'DataRefMatcher') + class MatchBackgroundsConfig(pexConfig.Config): @@ -152,31 +154,46 @@ def run(self, expRefList, expDatasetType, imageScalerList=None, refExpDataRef=No Choose a refExpDataRef automatically if none supplied. - @param[in] expRefList: list of data references to science exposures to be background-matched; + Parameters + ---------- + expRefList : + list of data references to science exposures to be background-matched; all exposures must exist. - @param[in] expDatasetType: dataset type of exposures, e.g. 'goodSeeingCoadd_tempExp' - @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); + expDatasetType : + dataset type of exposures, e.g. 'goodSeeingCoadd_tempExp' + imageScalerList : + list of image scalers (coaddUtils.ImageScaler); if None then the images are not scaled - @param[in] refExpDataRef: data reference for the reference exposure. + refExpDataRef : + data reference for the reference exposure. If None, then this task selects the best exposures from expRefList. if not None then must be one of the exposures in expRefList. - @param[in] refImageScaler: image scaler for reference image; + refImageScaler : + image scaler for reference image; ignored if refExpDataRef is None, else scaling is not performed if None - @return: a pipBase.Struct containing these fields: - - backgroundInfoList: a list of pipeBase.Struct, one per exposure in expRefList, - each of which contains these fields: - - isReference: this is the reference exposure (only one returned Struct will + Returns + ------- + result : `pipeBase.Struct` + a pipeBase.Struct containing these fields: + + - ``backgroundInfoList`` : a list of pipeBase.Struct, one per exposure in expRefList, + each of which contains these fields: + - ``isReference`` : this is the reference exposure (only one returned Struct will contain True for this value, unless the ref exposure is listed multiple times) - - backgroundModel: differential background model (afw.Math.Background or afw.Math.Approximate). + - ``backgroundModel`` : differential background model + (afw.Math.Background or afw.Math.Approximate). Add this to the science exposure to match the reference exposure. - - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); - should be comparable to difference image's mean variance. - - diffImVar: the mean variance of the difference image. - All fields except isReference will be None if isReference True or the fit failed. - - @warning: all exposures must exist on disk + - ``fitRMS`` : rms of the fit. This is the sqrt(mean(residuals**2)). + - ``matchedMSE`` : the MSE of the reference and matched + images: mean((refImage - matchedSciImage)**2); + should be comparable to difference image's mean variance. + - ``diffImVar`` : the mean variance of the difference image. + All fields except isReference will be None if isReference True or the fit failed. + + Notes + ----- + warning: all exposures must exist on disk """ numExp = len(expRefList) @@ -273,16 +290,27 @@ def selectRefExposure(self, expRefList, imageScalerList, expDatasetType): - bestRefWeightVariance - bestRefWeightLevel - @param[in] expRefList: list of data references to exposures. + Parameters + ---------- + expRefList : + list of data references to exposures. Retrieves dataset type specified by expDatasetType. If an exposure is not found, it is skipped with a warning. - @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); + imageScalerList : + list of image scalers (coaddUtils.ImageScaler); must be the same length as expRefList - @param[in] expDatasetType: dataset type of exposure: e.g. 'goodSeeingCoadd_tempExp' - - @return: index of best exposure - - @raise pipeBase.TaskError if none of the exposures in expRefList are found. + dataset : + type of exposure: e.g. 'goodSeeingCoadd_tempExp' + + Returns + ------- + result : + index of best exposure + + Raises + ------ + pipeBase.TaskError + if none of the exposures in expRefList are found. """ self.log.info("Calculating best reference visit") varList = [] @@ -332,6 +360,27 @@ def matchBackgrounds(self, refExposure, sciExposure): """ Match science exposure's background level to that of reference exposure. + Parameters + ---------- + refExposure : + reference exposure + sciExposure : + science exposure; modified by changing the background level + to match that of the reference exposure + + Returns + ------- + result : `pipeBase.Struct` + returns a pipeBase.Struct with fields: + - ``backgroundModel`` : an afw.math.Approximate or an afw.math.Background. + - ``fitRMS`` : rms of the fit. This is the sqrt(mean(residuals**2)). + - ``matchedMSE``: the MSE of the reference and + matched images: mean((refImage - matchedSciImage)**2); + should be comparable to difference image's mean variance. + - ``diffImVar`` : the mean variance of the difference image. + + Notes + ----- Process creates a difference image of the reference exposure minus the science exposure, and then generates an afw.math.Background object. It assumes (but does not require/check) that the mask plane already has detections set. If detections have not been set/masked, sources will bias the @@ -340,16 +389,6 @@ def matchBackgrounds(self, refExposure, sciExposure): or by polynomial interpolation by the Approximate class. This model of difference image is added to the science exposure in memory. Fit diagnostics are also calculated and returned. - - @param[in] refExposure: reference exposure - @param[in,out] sciExposure: science exposure; modified by changing the background level - to match that of the reference exposure - @returns a pipBase.Struct with fields: - - backgroundModel: an afw.math.Approximate or an afw.math.Background. - - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); - should be comparable to difference image's mean variance. - - diffImVar: the mean variance of the difference image. """ if lsstDebug.Info(__name__).savefits: @@ -489,13 +528,22 @@ def _debugPlot(self, X, Y, Z, dZ, modelImage, bbox, model, resids): Saves the fig to lsstDebug.Info(__name__).figpath Displays on screen if lsstDebug.Info(__name__).display = True - @param X: array of x positions - @param Y: array of y positions - @param Z: array of the grid values that were interpolated - @param dZ: array of the error on the grid values - @param modelImage: image ofthe model of the fit - @param model: array of len(Z) containing the grid values predicted by the model - @param resids: Z - model + Parameters + ---------- + X : + array of x positions + Y : + array of y positions + Z : + array of the grid values that were interpolated + dZ : + array of the error on the grid values + modelImage : + image ofthe model of the fit + model : + array of len(Z) containing the grid values predicted by the model + resids : + Z - model """ import matplotlib.pyplot as plt import matplotlib.colors @@ -580,42 +628,62 @@ class DataRefMatcher: Note that this is not exact, but should suffice for this task until there is better support for this kind of thing in the butler. + + Parameters + ---------- + butler : + datasetType : + dataset type to match """ def __init__(self, butler, datasetType): - """Construct a DataRefMatcher - - @param[in] butler - @param[in] datasetType: dataset type to match - """ self._datasetType = datasetType # for diagnostics self._keyNames = butler.getKeys(datasetType) def _makeKey(self, ref): """Return a tuple of values for the specified keyNames - @param[in] ref: data reference + Parameters + ---------- + ref : + data reference - @raise KeyError if ref.dataId is missing a key in keyNames + Raises + ------ + KeyError + if ref.dataId is missing a key in keyNames """ return tuple(ref.dataId[key] for key in self._keyNames) def isMatch(self, ref0, ref1): """Return True if ref0 == ref1 - @param[in] ref0: data ref 0 - @param[in] ref1: data ref 1 - - @raise KeyError if either ID is missing a key in keyNames + Parameters + ---------- + ref0 : + data ref 0 + ref1 : + data ref 1 + + Raises + ------ + KeyError + if either ID is missing a key in keyNames """ return self._makeKey(ref0) == self._makeKey(ref1) def matchList(self, ref0, refList): """Return a list of indices of matches - @return tuple of indices of matches + Returns + ------- + result : `tuple` + tuple of indices of matches - @raise KeyError if any ID is missing a key in keyNames + Raises + ------ + KeyError + if any ID is missing a key in keyNames """ key0 = self._makeKey(ref0) return tuple(ind for ind, ref in enumerate(refList) if self._makeKey(ref) == key0) diff --git a/python/lsst/pipe/tasks/measurePsf.py b/python/lsst/pipe/tasks/measurePsf.py index 50102a783..1a9d6fe37 100644 --- a/python/lsst/pipe/tasks/measurePsf.py +++ b/python/lsst/pipe/tasks/measurePsf.py @@ -26,6 +26,8 @@ import lsst.pex.config as pexConfig import lsst.pipe.base as pipeBase +__all__ = ('MeasurePsfConfig', 'MeasurePsfTask') + class MeasurePsfConfig(pexConfig.Config): starSelector = measAlg.sourceSelectorRegistry.makeField( @@ -54,167 +56,97 @@ class MeasurePsfConfig(pexConfig.Config): class MeasurePsfTask(pipeBase.Task): - r"""! -@anchor MeasurePsfTask_ - -@brief Measure the PSF - -@section pipe_tasks_measurePsf_Contents Contents - - - @ref pipe_tasks_measurePsf_Purpose - - @ref pipe_tasks_measurePsf_Initialize - - @ref pipe_tasks_measurePsf_IO - - @ref pipe_tasks_measurePsf_Config - - @ref pipe_tasks_measurePsf_Debug - - @ref pipe_tasks_measurePsf_Example - -@section pipe_tasks_measurePsf_Purpose Description - -A task that selects stars from a catalog of sources and uses those to measure the PSF. - -The star selector is a subclass of -@ref lsst.meas.algorithms.starSelector.BaseStarSelectorTask "lsst.meas.algorithms.BaseStarSelectorTask" -and the PSF determiner is a sublcass of -@ref lsst.meas.algorithms.psfDeterminer.BasePsfDeterminerTask "lsst.meas.algorithms.BasePsfDeterminerTask" - -@warning -There is no establised set of configuration parameters for these algorithms, so once you start modifying -parameters (as we do in @ref pipe_tasks_measurePsf_Example) your code is no longer portable. - -@section pipe_tasks_measurePsf_Initialize Task initialisation - -@copydoc \_\_init\_\_ - -@section pipe_tasks_measurePsf_IO Invoking the Task - -@copydoc run + """A task that selects stars from a catalog of sources and uses those to measure the PSF. -@section pipe_tasks_measurePsf_Config Configuration parameters + Parameters + ---------- + schema : `lsst.sfw.table.Schema` + An lsst::afw::table::Schema used to create the output lsst.afw.table.SourceCatalog + kwargs : + Keyword arguments passed to lsst.pipe.base.task.Task.__init__. -See @ref MeasurePsfConfig. + Notes + ----- + If schema is not None, 'calib_psf_candidate' and 'calib_psf_used' fields will be added to + identify which stars were employed in the PSF estimation. -@section pipe_tasks_measurePsf_Debug Debug variables + This task can add fields to the schema, so any code calling this task must ensure that + these fields are indeed present in the input table. -The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a -flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. + The star selector is a subclass of + ``lsst.meas.algorithms.starSelector.BaseStarSelectorTask`` "lsst.meas.algorithms.BaseStarSelectorTask" + and the PSF determiner is a sublcass of + ``lsst.meas.algorithms.psfDeterminer.BasePsfDeterminerTask`` "lsst.meas.algorithms.BasePsfDeterminerTask" -
-
@c display -
If True, display debugging plots -
displayExposure -
display the Exposure + spatialCells -
displayPsfCandidates -
show mosaic of candidates -
showBadCandidates -
Include bad candidates -
displayPsfMosaic -
show mosaic of reconstructed PSF(xy) -
displayResiduals -
show residuals -
normalizeResiduals -
Normalise residuals by object amplitude -
+ There is no establised set of configuration parameters for these algorithms, so once you start modifying + parameters (as we do in @ref pipe_tasks_measurePsf_Example) your code is no longer portable. -Additionally you can enable any debug outputs that your chosen star selector and psf determiner support. + @section pipe_tasks_measurePsf_Debug Debug variables -@section pipe_tasks_measurePsf_Example A complete example of using MeasurePsfTask + The ``lsst.pipe.base.cmdLineTask.CmdLineTask`` command line task interface supports a + flag -d to import debug.py from your PYTHONPATH; see baseDebug for more about debug.py files. -This code is in @link measurePsfTask.py@endlink in the examples directory, and can be run as @em e.g. -@code -examples/measurePsfTask.py --ds9 -@endcode -@dontinclude measurePsfTask.py + .. code-block:: none -The example also runs SourceDetectionTask and SourceMeasurementTask; -see @ref meas_algorithms_measurement_Example for more explanation. + display + If True, display debugging plots + displayExposure + display the Exposure + spatialCells + displayPsfCandidates + show mosaic of candidates + showBadCandidates + Include bad candidates + displayPsfMosaic + show mosaic of reconstructed PSF(xy) + displayResiduals + show residuals + normalizeResiduals + Normalise residuals by object amplitude -Import the tasks (there are some other standard imports; read the file to see them all): -@skip SourceDetectionTask -@until MeasurePsfTask + Additionally you can enable any debug outputs that your chosen star selector and psf determiner support. -We need to create the tasks before processing any data as the task constructor -can add an extra column to the schema, but first we need an almost-empty -Schema: + A complete example of using MeasurePsfTask -@skipline makeMinimalSchema + This code is in ``measurePsfTask.py`` in the examples directory, and can be run as e.g. -We can now call the constructors for the tasks we need to find and characterize candidate -PSF stars: + .. code-block:: none -@skip SourceDetectionTask.ConfigClass -@until measureTask + examples/measurePsfTask.py --ds9 -Note that we've chosen a minimal set of measurement plugins: we need the -outputs of @c base_SdssCentroid, @c base_SdssShape and @c base_CircularApertureFlux as -inputs to the PSF measurement algorithm, while @c base_PixelFlags identifies -and flags bad sources (e.g. with pixels too close to the edge) so they can be -removed later. + The example also runs SourceDetectionTask and SourceMeasurementTask; + see ``meas_algorithms_measurement_Example`` for more explanation. -Now we can create and configure the task that we're interested in: + Import the tasks (there are some other standard imports; read the file to see them all): -@skip MeasurePsfTask -@until measurePsfTask -We're now ready to process the data (we could loop over multiple exposures/catalogues using the same -task objects). First create the output table: + To investigate the @ref pipe_tasks_measurePsf_Debug, put something like -@skipline afwTable + .. code-block :: none -And process the image: - -@skip sources = -@until result - -We can then unpack and use the results: - -@skip psf -@until cellSet - -If you specified @c --ds9 you can see the PSF candidates: - -@skip display -@until RED - -
+ import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively -To investigate the @ref pipe_tasks_measurePsf_Debug, put something like -@code{.py} - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + if name == "lsst.pipe.tasks.measurePsf" : + di.display = True + di.displayExposure = False # display the Exposure + spatialCells + di.displayPsfCandidates = True # show mosaic of candidates + di.displayPsfMosaic = True # show mosaic of reconstructed PSF(xy) + di.displayResiduals = True # show residuals + di.showBadCandidates = True # Include bad candidates + di.normalizeResiduals = False # Normalise residuals by object amplitude - if name == "lsst.pipe.tasks.measurePsf" : - di.display = True - di.displayExposure = False # display the Exposure + spatialCells - di.displayPsfCandidates = True # show mosaic of candidates - di.displayPsfMosaic = True # show mosaic of reconstructed PSF(xy) - di.displayResiduals = True # show residuals - di.showBadCandidates = True # Include bad candidates - di.normalizeResiduals = False # Normalise residuals by object amplitude + return di - return di + lsstDebug.Info = DebugInfo - lsstDebug.Info = DebugInfo -@endcode -into your debug.py file and run measurePsfTask.py with the @c --debug flag. + into your debug.py file and run measurePsfTask.py with the --debug flag. """ ConfigClass = MeasurePsfConfig _DefaultName = "measurePsf" def __init__(self, schema=None, **kwargs): - """!Create the detection task. Most arguments are simply passed onto pipe.base.Task. - - @param schema An lsst::afw::table::Schema used to create the output lsst.afw.table.SourceCatalog - @param **kwargs Keyword arguments passed to lsst.pipe.base.task.Task.__init__. - - If schema is not None, 'calib_psf_candidate' and 'calib_psf_used' fields will be added to - identify which stars were employed in the PSF estimation. - - @note This task can add fields to the schema, so any code calling this task must ensure that - these fields are indeed present in the input table. - """ - pipeBase.Task.__init__(self, **kwargs) if schema is not None: self.candidateKey = schema.addField( @@ -238,23 +170,32 @@ def __init__(self, schema=None, **kwargs): @pipeBase.timeMethod def run(self, exposure, sources, expId=0, matches=None): - """!Measure the PSF - - @param[in,out] exposure Exposure to process; measured PSF will be added. - @param[in,out] sources Measured sources on exposure; flag fields will be set marking - stars chosen by the star selector and the PSF determiner if a schema - was passed to the task constructor. - @param[in] expId Exposure id used for generating random seed. - @param[in] matches A list of lsst.afw.table.ReferenceMatch objects - (@em i.e. of lsst.afw.table.Match - with @c first being of type lsst.afw.table.SimpleRecord and @c second - type lsst.afw.table.SourceRecord --- the reference object and detected - object respectively) as returned by @em e.g. the AstrometryTask. - Used by star selectors that choose to refer to an external catalog. - - @return a pipe.base.Struct with fields: - - psf: The measured PSF (also set in the input exposure) - - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates + """Measure the PSF + + Parameters + ---------- + exposure : + Exposure to process; measured PSF will be added. + sources : + Measured sources on exposure; flag fields will be set marking + stars chosen by the star selector and the PSF determiner if a schema + was passed to the task constructor. + expId : + Exposure id used for generating random seed. + matches : + A list of lsst.afw.table.ReferenceMatch objects + (i.e. of lsst.afw.table.Match + with first being of type lsst.afw.table.SimpleRecord and @c second + type lsst.afw.table.SourceRecord --- the reference object and detected + object respectively) as returned by e.g. the AstrometryTask. + Used by star selectors that choose to refer to an external catalog. + + Returns + ------- + result : `pipe.base.Struct` + a pipe.base.Struct with fields: + - ``psf`` : The measured PSF (also set in the input exposure) + - ``cellSet`` : an lsst.afw.math.SpatialCellSet containing the PSF candidates as returned by the psf determiner. """ self.log.info("Measuring PSF") diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index b68c6cb8e..61b23bbc1 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -20,6 +20,20 @@ # the GNU General Public License along with this program. If not, # see . # +""" +New dataset types: +- deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) +- deepCoadd_mergeDet: merged detections (tract, patch) +- deepCoadd_meas: measurements of merged detections (tract, patch, filter) +- deepCoadd_ref: reference sources (tract, patch) +All of these have associated schema catalogs that require no data ID and hold no records. + +In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in +the mergeDet, meas, and ref dataset Footprints: +- deepCoadd_peak_schema +""" +import numpy + from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner, PipelineTask, InitOutputDatasetField, InputDatasetField, OutputDatasetField, @@ -45,27 +59,13 @@ from .multiBandUtils import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory # noqa: F401 -""" -New set types: -* deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) -* deepCoadd_mergeDet: merged detections (tract, patch) -* deepCoadd_meas: measurements of merged detections (tract, patch, filter) -* deepCoadd_ref: reference sources (tract, patch) -All of these have associated *_schema catalogs that require no data ID and hold no records. -In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in -the mergeDet, meas, and ref dataset Footprints: -* deepCoadd_peak_schema -""" ############################################################################################################## class DetectCoaddSourcesConfig(Config): - """! - @anchor DetectCoaddSourcesConfig_ - - @brief Configuration parameters for the DetectCoaddSourcesTask + """Configuration parameters for the DetectCoaddSourcesTask """ doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?") scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling") @@ -132,24 +132,8 @@ def setDefaults(self): ## @} -class DetectCoaddSourcesTask(PipelineTask, CmdLineTask): - r"""! - @anchor DetectCoaddSourcesTask_ - - @brief Detect sources on a coadd - - @section pipe_tasks_multiBand_Contents Contents - - - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose - - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize - - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run - - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config - - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug - - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example - - @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description - - Command-line task that detects sources on a coadd of exposures obtained with a single filter. +class DetectCoaddSourcesTask(CmdLineTask): + """Command-line task that detects sources on a coadd of exposures obtained with a single filter. Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane @@ -159,43 +143,29 @@ class DetectCoaddSourcesTask(PipelineTask, CmdLineTask): After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref SourceDetectionTask_ "detection" subtask. - @par Inputs: - deepCoadd{tract,patch,filter}: ExposureF - @par Outputs: - deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) - @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input - exposure (ExposureF) - @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList - @par Data Unit: + Parameters + ----- + deepCoadd : + {tract,patch,filter}. ExposureF + deepCoadd_det : + {tract,patch,filter}. SourceCatalog (only parent Footprints) + deepCoadd_calexp : + {tract,patch,filter}. Variance scaled, background-subtracted input + exposure (ExposureF) + deepCoadd_calexp_background : + {tract,patch,filter}. BackgroundList + Data Unit : tract, patch, filter + schema : + initial schema for the output catalog, modified-in place to include all + fields set by this task. If None, the source minimal schema will be used. + kwargs : + keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ + Notes + ----- - DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. - You can retarget this subtask if you wish. - - @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization - - @copydoc \_\_init\_\_ - - @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task - - @copydoc run - - @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters - - See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" - - @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables - - The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a - flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py - files. - - DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to - @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for - @ref SourceDetectionTask_ "SourceDetectionTask" for further information. - - @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example - of using DetectCoaddSourcesTask + Examples + -------- DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of the task is to update the background, detect all sources in a single band and generate a set of parent @@ -203,25 +173,25 @@ class DetectCoaddSourcesTask(PipelineTask, CmdLineTask): eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data reference to the coadd to be processed. A list of the available optional arguments can be obtained by calling detectCoaddSources.py with the `--help` command line argument: - @code - detectCoaddSources.py --help - @endcode + + >>> detectCoaddSources.py --help To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: - @code - detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I - @endcode + + + >>> detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I + that will process the HSC-I band data. The results are written to `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. It is also necessary to run: - @code - detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R - @endcode + + >>> detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R + to generate the sources catalogs for the HSC-R band required by the next step in the multi-band - processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". + processing procedure: MergeDetectionsTask_ "MergeDetectionsTask". """ _DefaultName = "detectCoaddSources" ConfigClass = DetectCoaddSourcesConfig @@ -253,14 +223,17 @@ def getInitOutputDatasetTypes(cls, config): return output def __init__(self, schema=None, **kwargs): - """! - @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. + """Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): - @param[in] schema: initial schema for the output catalog, modified-in place to include all - fields set by this task. If None, the source minimal schema will be used. - @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ + Parameters + ---------- + schema : + initial schema for the output catalog, modified-in place to include all + fields set by this task. If None, the source minimal schema will be used. + kwargs : + keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ """ # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree # call structure has been reviewed carefully to be sure super will work as intended. @@ -278,13 +251,15 @@ def getInitOutputDatasets(self): return {"detectionSchema": afwTable.SourceCatalog(self.schema)} def runDataRef(self, patchRef): - """! - @brief Run detection on a coadd. + """Run detection on a coadd. - Invokes @ref run and then uses @ref write to output the + Invokes run and then uses write to output the results. - @param[in] patchRef: data reference for patch + Parameters + ---------- + patchRef : + data reference for patch """ exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True) expId = int(patchRef.get(self.config.coaddName + "CoaddId")) @@ -299,21 +274,28 @@ def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler): return self.run(**inputData) def run(self, exposure, idFactory, expId): - """! - @brief Run detection on an exposure. + """Run detection on an exposure. First scale the variance plane to match the observed variance - using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to + using ScaleVarianceTask. Then invoke the SourceDetectionTask_ "detection" subtask to detect sources. - @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, - depending on configuration). - @param[in] idFactory: IdFactory to set source identifiers - @param[in] expId: Exposure identifier (integer) for RNG seed + Parameters + ---------- + exposure : + Exposure on which to detect (may be backround-subtracted and scaled, + depending on configuration). + idFactory : + IdFactory to set source identifiers + expId : + Exposure identifier (integer) for RNG seed - @return a pipe.base.Struct with fields - - sources: catalog of detections - - backgrounds: list of backgrounds + Returns + ------- + result : `pipe.base.Struct` + a pipe.base.Struct with fields: + - ``sources`` : catalog of detections + - ``backgrounds`` : list of backgrounds """ if self.config.doScaleVariance: varScale = self.scaleVariance.run(exposure.maskedImage) @@ -334,9 +316,12 @@ def write(self, results, patchRef): """! @brief Write out results from runDetection. - @param[in] exposure: Exposure to write out - @param[in] results: Struct returned from runDetection - @param[in] patchRef: data reference for patch + exposure: + Exposure to write out + results: + Struct returned from runDetection + patchRef: + data reference for patch """ coaddName = self.config.coaddName + "Coadd" patchRef.put(results.outputBackgrounds, coaddName + "_calexp_background") @@ -395,27 +380,23 @@ class DeblendCoaddSourcesTask(CmdLineTask): Deblend sources from master catalog in each coadd. This can either be done separately in each band using the HSC-SDSS deblender - (`DeblendCoaddSourcesTask.config.simultaneous==False`) + ( `DeblendCoaddSourcesTask.config.simultaneous==False` ) or use SCARLET to simultaneously fit the blend in all bands - (`DeblendCoaddSourcesTask.config.simultaneous==True`). + ( `DeblendCoaddSourcesTask.config.simultaneous==True` ). The task will set its own `self.schema` atribute to the `Schema` of the output deblended catalog. This will include all fields from the input `Schema`, as well as additional fields from the deblender. - `pipe.tasks.multiband.DeblendCoaddSourcesTask Description - --------------------------------------------------------- - ` - Parameters ---------- butler: `Butler` Butler used to read the input schemas from disk or - construct the reference catalog loader, if `schema` or `peakSchema` or + construct the reference catalog loader, if `schema` or `peakSchema` schema: `Schema` The schema of the merged detection catalog as an input to this task. peakSchema: `Schema` - The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog + The schema of the `PeakRecord` in the `Footprint` in the merged detection catalog """ ConfigClass = DeblendCoaddSourcesConfig RunnerClass = DeblendCoaddSourcesRunner @@ -453,7 +434,7 @@ def getSchemaCatalogs(self): Returns ------- - result: dict + result: `dict` Dictionary of empty catalogs, with catalog names as keys. """ catalog = afwTable.SourceCatalog(self.schema) @@ -471,7 +452,7 @@ def runDataRef(self, patchRefList, psfCache=100): Parameters ---------- - patchRefList: list + patchRefList: `list` List of data references for each filter """ if self.config.simultaneous: @@ -556,9 +537,10 @@ def write(self, dataRef, flux_sources, template_sources=None): def writeMetadata(self, dataRefList): """Write the metadata produced from processing the data. + Parameters ---------- - dataRefList + dataRefList : `list` List of Butler data references used to write the metadata. The metadata is written to dataset type `CmdLineTask._getMetadataName`. """ @@ -577,10 +559,7 @@ def getExposureId(self, dataRef): class MeasureMergedCoaddSourcesConfig(Config): - """! - @anchor MeasureMergedCoaddSourcesConfig_ - - @brief Configuration parameters for the MeasureMergedCoaddSourcesTask + """Configuration parameters for the MeasureMergedCoaddSourcesTask """ inputCatalog = Field(dtype=str, default="deblendedFlux", doc=("Name of the input catalog to use." @@ -651,23 +630,7 @@ def getTargetList(parsedCmd, **kwargs): class MeasureMergedCoaddSourcesTask(CmdLineTask): - r"""! - @anchor MeasureMergedCoaddSourcesTask_ - - @brief Deblend sources from master catalog in each coadd seperately and measure. - - @section pipe_tasks_multiBand_Contents Contents - - - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose - - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize - - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run - - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config - - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug - - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example - - @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description - - Command-line task that uses peaks and footprints from a master catalog to perform deblending and + """Command-line task that uses peaks and footprints from a master catalog to perform deblending and measurement in each coadd. Given a master input catalog of sources (peaks and footprints) or deblender outputs @@ -676,60 +639,36 @@ class MeasureMergedCoaddSourcesTask(CmdLineTask): consistent set of child sources. The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source - properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit + properties are measured and the is-primary flag (indicating sources with no children) is set. Visit flags are propagated to the coadd sources. Optionally, we can match the coadd sources to an external reference catalog. - @par Inputs: - deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog - @n deepCoadd_calexp{tract,patch,filter}: ExposureF - @par Outputs: - deepCoadd_meas{tract,patch,filter}: SourceCatalog - @par Data Unit: - tract, patch, filter - + Parameters + ---------- + deepCoadd_mergeDet{tract,patch} : + deepCoadd_deblend{tract,patch} : + SourceCatalog + deepCoadd_calexp{tract,patch,filter} : + ExposureF + deepCoadd_meas{tract,patch,filter} : + SourceCatalog + Data Unit : + tract, patch, filter + + Notes + ----- MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: -
-
@ref SingleFrameMeasurementTask_ "measurement" -
Measure source properties of deblended sources.
-
@ref SetPrimaryFlagsTask_ "setPrimaryFlags" -
Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are - not at the edge of the field and that have either not been deblended or are the children of deblended - sources
-
@ref PropagateVisitFlagsTask_ "propagateFlags" -
Propagate flags set in individual visits to the coadd.
-
@ref DirectMatchTask_ "match" -
Match input sources to a reference catalog (optional). -
-
- These subtasks may be retargeted as required. - - @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization - - @copydoc \_\_init\_\_ - - @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task - - @copydoc run - - @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters - - See @ref MeasureMergedCoaddSourcesConfig_ - - @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables - - The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a - flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py + The ``lsst.pipe.base.cmdLineTask.CmdLineTask`` command line task interface supports a + flag -d to import debug.py from your PYTHONPATH; see baseDebug for more about debug.py files. MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to the various sub-tasks. See the documetation for individual sub-tasks for more information. - @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using - MeasureMergedCoaddSourcesTask - + Examples + -------- After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. The next stage in the multi-band processing procedure will merge these measurements into a suitable catalog for driving forced photometry. @@ -738,26 +677,26 @@ class MeasureMergedCoaddSourcesTask(CmdLineTask): to be processed. A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the `--help` command line argument: - @code - measureCoaddSources.py --help - @endcode + + >>> measureCoaddSources.py --help + To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished - step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band + step 6 at pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band coadd as follows: - @code - measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I - @endcode + + >>> measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I + This will process the HSC-I band data. The results are written in - `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits + `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits` It is also necessary to run - @code - measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R - @endcode + + >>> measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R + to generate the sources catalogs for the HSC-R band required by the next step in the multi-band - procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". + procedure: "MergeMeasurementsTask" . """ _DefaultName = "measureCoaddSources" ConfigClass = MeasureMergedCoaddSourcesConfig @@ -775,18 +714,25 @@ def _makeArgumentParser(cls): return parser def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs): - """! - @brief Initialize the task. + """Initialize the task. + Parameters + ---------- Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): - @param[in] schema: the schema of the merged detection catalog used as input to this one - @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog - @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference + schema : + the schema of the merged detection catalog used as input to this one + peakSchema : + the schema of the PeakRecords in the Footprints in the merged detection catalog + refObjLoader : + an instance of LoadReferenceObjectsTasks that supplies an external reference catalog. May be None if the loader can be constructed from the butler argument or all steps requiring a reference catalog are disabled. - @param[in] butler: a butler used to read the input schemas from disk or construct the reference + butler : + a butler used to read the input schemas from disk or construct the reference catalog loader, if schema or peakSchema or refObjLoader is None + Notes + ----- The task will set its own self.schema attribute to the schema of the output measurement catalog. This will include all fields from the input schema, as well as additional fields for all the measurements. @@ -816,11 +762,15 @@ def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, self.makeSubtask("catalogCalculation", schema=self.schema) def runDataRef(self, patchRef, psfCache=100): - """! - @brief Deblend and measure. + """Deblend and measure. - @param[in] patchRef: Patch reference. + Parameters + ---------- + patchRef : + Patch reference. + Notes + ----- Set 'is-primary' and related flags. Propagate flags from individual visits. Optionally match the sources to a reference catalog and write the matches. Finally, write the deblended sources and measurements out. @@ -860,12 +810,20 @@ def runDataRef(self, patchRef, psfCache=100): self.write(patchRef, sources) def readSources(self, dataRef): - """! - @brief Read input sources. + """Read input sources. - @param[in] dataRef: Data reference for catalog of merged detections - @return List of sources in merged catalog + Parameters + ---------- + dataRef : + Data reference for catalog of merged detections + + Returns + ------- + sources : `list` + List of sources in merged catalog + Notes + ----- We also need to add columns to hold the measurements we're about to make so we can measure in-place. """ @@ -880,14 +838,18 @@ def readSources(self, dataRef): return sources def writeMatches(self, dataRef, exposure, sources): - """! - @brief Write matches of the sources to the astrometric reference catalog. + """Write matches of the sources to the astrometric reference catalog. We use the Wcs in the exposure to match sources. - @param[in] dataRef: data reference - @param[in] exposure: exposure with Wcs - @param[in] sources: source catalog + Parameters + ---------- + dataRef : + data reference + exposure : + exposure with Wcs + sources : + source catalog """ result = self.match.run(sources, exposure.getInfo().getFilter().getName()) if result.matches: @@ -899,11 +861,14 @@ def writeMatches(self, dataRef, exposure, sources): dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull") def write(self, dataRef, sources): - """! - @brief Write the source catalog. + """Write the source catalog. - @param[in] dataRef: data reference - @param[in] sources: source catalog + Parameters + ---------- + dataRef : + data reference + sources : + source catalog """ dataRef.put(sources, self.config.coaddName + "Coadd_meas") self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId)) diff --git a/python/lsst/pipe/tasks/objectMasks.py b/python/lsst/pipe/tasks/objectMasks.py index 95da770ae..cb39a8806 100644 --- a/python/lsst/pipe/tasks/objectMasks.py +++ b/python/lsst/pipe/tasks/objectMasks.py @@ -4,6 +4,8 @@ import lsst.afw.table as afwTable from lsst.log import Log +__all__ = ('ObjectMaskCatalog',) + class ObjectMaskCatalog: """Class to support bright object masks @@ -40,40 +42,42 @@ def __setitem__(self, i, v): @staticmethod def readFits(fileName, hdu=0, flags=0): - """Read a ds9 region file, returning a ObjectMaskCatalog object - - This method is called "readFits" to fool the butler. The corresponding mapper entry looks like - brightObjectMask: { - template: "deepCoadd/BrightObjectMasks/%(tract)d/BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg" # noqa E501 - python: "lsst.obs.subaru.objectMasks.ObjectMaskCatalog" - persistable: "PurePythonClass" - storage: "FitsCatalogStorage" - } - and this is the only way I know to get it to read a random file type, in this case a ds9 region file. - - This method expects to find files named as BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg - The files should be structured as follows: - - # Description of catalogue as a comment - # CATALOG: catalog-id-string - # TRACT: 0 - # PATCH: 5,4 - # FILTER: HSC-I - - wcs; fk5 - - circle(RA, DEC, RADIUS) # ID: 1, mag: 12.34 - box(RA, DEC, XSIZE, YSIZE, THETA) # ID: 2, mag: 23.45 - ... - - The ", mag: XX.YY" is optional - - The commented lines must be present, with the relevant fields such as tract patch and filter filled - in. The coordinate system must be listed as above. Each patch is specified as a box or circle, with - RA, DEC, and dimensions specified in decimal degrees (with or without an explicit "d"). - - Only (axis-aligned) boxes and circles are currently supported as region definitions. - """ + # """Read a ds9 region file, returning a ObjectMaskCatalog object + + # Notes + # ----- + # This method is called "readFits" to fool the butler. The corresponding mapper entry looks like + # brightObjectMask: { + # template: "deepCoadd/BrightObjectMasks/%(tract)d/BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg" # noqa E501 + # python: "lsst.obs.subaru.objectMasks.ObjectMaskCatalog" + # persistable: "PurePythonClass" + # storage: "FitsCatalogStorage" + # } + # and this is the only way I know to get it to read a random file type, in this case a ds9 region file + + # This method expects to find files named as BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg + # The files should be structured as follows: + + # # Description of catalogue as a comment + # # CATALOG: catalog-id-string + # # TRACT: 0 + # # PATCH: 5,4 + # # FILTER: HSC-I + + # wcs; fk5 + + # circle(RA, DEC, RADIUS) # ID: 1, mag: 12.34 + # box(RA, DEC, XSIZE, YSIZE, THETA) # ID: 2, mag: 23.45 + # ... + + # The ", mag: XX.YY" is optional + + # The commented lines must be present, with the relevant fields such as tract patch and filter filled + # in. The coordinate system must be listed as above. Each patch is specified as a box or circle, with + # RA, DEC, and dimensions specified in decimal degrees (with or without an explicit "d"). + + # Only (axis-aligned) boxes and circles are currently supported as region definitions. + # """ log = Log.getLogger("ObjectMaskCatalog") diff --git a/python/lsst/pipe/tasks/photoCal.py b/python/lsst/pipe/tasks/photoCal.py index fbbae22b1..826451273 100644 --- a/python/lsst/pipe/tasks/photoCal.py +++ b/python/lsst/pipe/tasks/photoCal.py @@ -52,9 +52,9 @@ class PhotoCalConfig(pexConf.Config): default=None, doc=("Apply photometric color terms to reference stars? One of:\n" "None: apply if colorterms and photoCatName are not None;\n" - " fail if color term data is not available for the specified ref catalog and filter.\n" + "fail if color term data is not available for the specified ref catalog and filter.\n" "True: always apply colorterms; fail if color term data is not available for the\n" - " specified reference catalog and filter.\n" + "specified reference catalog and filter.\n" "False: do not apply."), optional=True, ) @@ -122,129 +122,61 @@ def setDefaults(self): ## @} class PhotoCalTask(pipeBase.Task): - r"""! -@anchor PhotoCalTask_ + """Calculate an Exposure's zero-point given a set of flux measurements + of stars matched to an input catalogue. -@brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. + Notes + ----- + The type of flux to use is specified by PhotoCalConfig.fluxField. -@section pipe_tasks_photocal_Contents Contents + The algorithm clips outliers iteratively, with parameters set in the configuration. - - @ref pipe_tasks_photocal_Purpose - - @ref pipe_tasks_photocal_Initialize - - @ref pipe_tasks_photocal_IO - - @ref pipe_tasks_photocal_Config - - @ref pipe_tasks_photocal_Debug - - @ref pipe_tasks_photocal_Example + This task can adds fields to the schema, so any code calling this task must ensure that + these columns are indeed present in the input match list; see @ref pipe_tasks_photocal_Example -@section pipe_tasks_photocal_Purpose Description + Debug variables -@copybrief PhotoCalTask + The available variables in PhotoCalTask are: -Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. -The type of flux to use is specified by PhotoCalConfig.fluxField. + display : + If True enable other debug outputs + displaySources : + If True, display the exposure on ds9's frame 1 and overlay the source catalogue. -The algorithm clips outliers iteratively, with parameters set in the configuration. + red o : + Reserved objects + green o : + Objects used in the photometric calibration -@note This task can adds fields to the schema, so any code calling this task must ensure that -these columns are indeed present in the input match list; see @ref pipe_tasks_photocal_Example + scatterPlot : + Make a scatter plot of flux v. reference magnitude as a function of reference magnitude: -@section pipe_tasks_photocal_Initialize Task initialisation + - good objects in blue + - rejected objects in red -@copydoc \_\_init\_\_ + (if scatterPlot is 2 or more, prompt to continue after each iteration) -@section pipe_tasks_photocal_IO Inputs/Outputs to the run method + To investigate the @ref pipe_tasks_photocal_Debug, put something like -@copydoc run + .. code-block:: none -@section pipe_tasks_photocal_Config Configuration parameters -See @ref PhotoCalConfig - -@section pipe_tasks_photocal_Debug Debug variables - -The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a -flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. - -The available variables in PhotoCalTask are: -
-
@c display -
If True enable other debug outputs -
@c displaySources -
If True, display the exposure on ds9's frame 1 and overlay the source catalogue. -
-
red o -
Reserved objects -
green o -
Objects used in the photometric calibration -
-
@c scatterPlot -
Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. - - good objects in blue - - rejected objects in red - (if @c scatterPlot is 2 or more, prompt to continue after each iteration) -
- -@section pipe_tasks_photocal_Example A complete example of using PhotoCalTask - -This code is in @link examples/photoCalTask.py@endlink, and can be run as @em e.g. -@code -examples/photoCalTask.py -@endcode -@dontinclude photoCalTask.py - -Import the tasks (there are some other standard imports; read the file for details) -@skipline from lsst.pipe.tasks.astrometry -@skipline measPhotocal - -We need to create both our tasks before processing any data as the task constructors -can add extra columns to the schema which we get from the input catalogue, @c scrCat: -@skipline getSchema - -Astrometry first: -@skip AstrometryTask.ConfigClass -@until aTask -(that @c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, -so we tell it to use the @c r band) - -Then photometry: -@skip measPhotocal -@until pTask - -If the schema has indeed changed we need to add the new columns to the source table -(yes; this should be easier!) -@skip srcCat -@until srcCat = cat - -We're now ready to process the data (we could loop over multiple exposures/catalogues using the same -task objects): -@skip matches -@until result - -We can then unpack and use the results: -@skip calib -@until np.log + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + if name.endswith(".PhotoCal"): + di.display = 1 -
-To investigate the @ref pipe_tasks_photocal_Debug, put something like -@code{.py} - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively - if name.endswith(".PhotoCal"): - di.display = 1 + return di - return di + lsstDebug.Info = DebugInfo - lsstDebug.Info = DebugInfo -@endcode -into your debug.py file and run photoCalTask.py with the @c --debug flag. + into your debug.py file and run photoCalTask.py with the @c --debug flag. """ ConfigClass = PhotoCalConfig _DefaultName = "photoCal" def __init__(self, refObjLoader, schema=None, **kwds): - """!Create the photometric calibration task. See PhotoCalTask.init for documentation - """ pipeBase.Task.__init__(self, **kwds) self.scatterPlot = None self.fig = None @@ -282,19 +214,30 @@ def getSourceKeys(self, schema): @pipeBase.timeMethod def extractMagArrays(self, matches, filterName, sourceKeys): - """!Extract magnitude and magnitude error arrays from the given matches. - - @param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink - @param[in] filterName Name of filter being calibrated - @param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() - - @return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays - where magErr is an error in the magnitude; the error in srcMag - refMag - If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as - magErr is what is later used to determine the zero point. - Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes - (1 or 2 strings) - @note These magnitude arrays are the @em inputs to the photometric calibration, some may have been + """Extract magnitude and magnitude error arrays from the given matches. + + Parameters + ---------- + matches : `lsst.afw.table.ReferenceMatchVector` + Reference/source matches, a lsst::afw::table::ReferenceMatchVector + filterName : + Name of filter being calibrated + sourceKeys : + Struct of source catalog keys, as returned by getSourceKeys() + + Returns + ------- + result : `Struct` + Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays + where magErr is an error in the magnitude; the error in srcMag - refMag + If nonzero, config.magErrFloor will be added to magErr only (not srcMagErr or refMagErr), as + magErr is what is later used to determine the zero point. + Struct also contains refFluxFieldList: a list of field names of the reference + catalog used for fluxes (1 or 2 strings) + + Notes + ----- + These magnitude arrays are the @em inputs to the photometric calibration, some may have been discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) """ srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m in matches]) @@ -402,47 +345,60 @@ def extractMagArrays(self, matches, filterName, sourceKeys): @pipeBase.timeMethod def run(self, exposure, sourceCat, expId=0): - """!Do photometric calibration - select matches to use and (possibly iteratively) compute + """Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point. - @param[in] exposure Exposure upon which the sources in the matches were detected. - @param[in] sourceCat A catalog of sources to use in the calibration - (@em i.e. a list of lsst.afw.table.Match with - @c first being of type lsst.afw.table.SimpleRecord and @c second type lsst.afw.table.SourceRecord --- - the reference object and matched object respectively). - (will not be modified except to set the outputField if requested.). - - @return Struct of: - - calib ------- @link lsst::afw::image::Calib@endlink object containing the zero point - - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays - - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. - - zp ---------- Photometric zero point (mag) - - sigma ------- Standard deviation of fit of photometric zero point (mag) - - ngood ------- Number of sources used to fit photometric zero point + Parameters + ---------- + exposure : + Exposure upon which the sources in the matches were detected. + sourceCat : + A catalog of sources to use in the calibration + (i.e. a list of lsst.afw.table.Match with + first being of type lsst.afw.table.SimpleRecord and second type lsst.afw.table.SourceRecord + the reference object and matched object respectively). + (will not be modified except to set the outputField if requested.). + Returns + ------- + result : `Struct` + Struct of: + - ``calib`` : lsst::afw::image::Calib object containing the zero point + - ``arrays`` : Magnitude arrays returned be PhotoCalTask.extractMagArrays + - ``matchesFinal`` : ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. + - ``zp`` : Photometric zero point (mag) + - ``sigma`` : Standard deviation of fit of photometric zero point (mag) + - ``ngood`` : Number of sources used to fit photometric zero point + + Notes + ----- The exposure is only used to provide the name of the filter being calibrated (it may also be used to generate debugging plots). The reference objects: - - Must include a field @c photometric; True for objects which should be considered as - photometric standards - - Must include a field @c flux; the flux used to impose a magnitude limit and also to calibrate - the data to (unless a color term is specified, in which case ColorTerm.primary is used; - See https://jira.lsstcorp.org/browse/DM-933) - - May include a field @c stargal; if present, True means that the object is a star - - May include a field @c var; if present, True means that the object is variable + - Must include a field @c photometric; True for objects which should be considered as + photometric standards + - Must include a field @c flux; the flux used to impose a magnitude limit and also to calibrate + the data to (unless a color term is specified, in which case ColorTerm.primary is used; + See https://jira.lsstcorp.org/browse/DM-933) + - May include a field @c stargal; if present, True means that the object is a star + - May include a field @c var; if present, True means that the object is variable The measured sources: - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration - @throws RuntimeError with the following strings: + Raises + ------ + RuntimeError + with the following strings: + + .. code-block:: none + + No matches to use for photocal + No matches are available (perhaps no sources/references were selected by the matcher). + No reference stars are available + No matches are available from which to extract magnitudes. -
-
No matches to use for photocal -
No matches are available (perhaps no sources/references were selected by the matcher). -
No reference stars are available -
No matches are available from which to extract magnitudes. -
""" import lsstDebug @@ -521,21 +477,26 @@ def displaySources(self, exposure, matches, reserved, frame=1): ds9.dot("o", x, y, size=4, frame=frame, ctype=ctype) def getZeroPoint(self, src, ref, srcErr=None, zp0=None): - """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) + """Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) + Returns + ------- + result : `Struct` + Struct of: + - ``zp`` : Photometric zero point (mag) + - ``sigma`` : Standard deviation of fit of zero point (mag) + - ``ngood`` : Number of sources used to fit zero point + + Notes + ----- We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: - 1. We use the median/interquartile range to estimate the position to clip around, and the + - We use the median/interquartile range to estimate the position to clip around, and the "sigma" to use. - 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently + - We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently large estimate will prevent the clipping from ever taking effect. - 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude + - Rather than start with the median we start with a crude mode. This means that a set of magnitude residuals with a tight core and asymmetrical outliers will start in the core. We use the width of - this core to set our maximum sigma (see 2.) - - @return Struct of: - - zp ---------- Photometric zero point (mag) - - sigma ------- Standard deviation of fit of zero point (mag) - - ngood ------- Number of sources used to fit zero point + this core to set our maximum sigma (see second bullet) """ sigmaMax = self.config.sigmaMax diff --git a/python/lsst/pipe/tasks/processCcd.py b/python/lsst/pipe/tasks/processCcd.py index 480e674e4..6463b1899 100644 --- a/python/lsst/pipe/tasks/processCcd.py +++ b/python/lsst/pipe/tasks/processCcd.py @@ -30,6 +30,7 @@ class ProcessCcdConfig(pexConfig.Config): """Config for ProcessCcd""" + isr = pexConfig.ConfigurableField( target=IsrTask, doc="""Task to perform instrumental signature removal or load a post-ISR image; ISR consists of: @@ -77,54 +78,47 @@ def setDefaults(self): class ProcessCcdTask(pipeBase.CmdLineTask): - r"""!Assemble raw data, fit the PSF, detect and measure, and fit WCS and zero-point - - @anchor ProcessCcdTask_ - - @section pipe_tasks_processCcd_Contents Contents - - - @ref pipe_tasks_processCcd_Purpose - - @ref pipe_tasks_processCcd_Initialize - - @ref pipe_tasks_processCcd_IO - - @ref pipe_tasks_processCcd_Config - - @ref pipe_tasks_processCcd_Debug - - @ref pipe_tasks_processCcd_Example - - @section pipe_tasks_processCcd_Purpose Description + """Assemble raw data, fit the PSF, detect and measure, and fit WCS and zero-point + + Parameters + ---------- + butler : + The butler is passed to the refObjLoader constructor in case it is + needed. Ignored if the refObjLoader argument provides a loader directly. + psfRefObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an + external reference catalog for image characterization. An example of when this would + be used is when a CatalogStarSelector is used. May be None if the desired loader can + be constructed from the butler argument or all steps requiring a catalog are disabled. + astromRefObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an + external reference catalog for astrometric calibration. May be None if the desired + loader can be constructed from the butler argument or all steps requiring a reference + catalog are disabled. + photoRefObjLoader : + An instance of LoadReferenceObjectsTasks that supplies an + external reference catalog for photometric calibration. May be None if the desired + loader can be constructed from the butler argument or all steps requiring a reference + catalog are disabled. + kwargs : + other keyword arguments for lsst.pipe.base.CmdLineTask + + Notes + ----- Perform the following operations: - Call isr to unpersist raw data and assemble it into a post-ISR exposure - Call charImage subtract background, fit a PSF model, repair cosmic rays, - detect and measure bright sources, and measure aperture correction + detect and measure bright sources, and measure aperture correction - Call calibrate to perform deep detection, deblending and single-frame measurement, - refine the WCS and fit the photometric zero-point - - @section pipe_tasks_processCcd_Initialize Task initialisation - - @copydoc \_\_init\_\_ - - @section pipe_tasks_processCcd_IO Invoking the Task - - This task is primarily designed to be run from the command line. - - The main method is `runDataRef`, which takes a single butler data reference for the raw input data. - - @section pipe_tasks_processCcd_Config Configuration parameters - - See @ref ProcessCcdConfig - - @section pipe_tasks_processCcd_Debug Debug variables - - ProcessCcdTask has no debug output, but its subtasks do. - - @section pipe_tasks_processCcd_Example A complete example of using ProcessCcdTask + refine the WCS and fit the photometric zero-point The following commands will process all raw data in obs_test's data repository. Note: be sure to specify an `--output` that does not already exist: - setup obs_test - setup pipe_tasks - processCcd.py $OBS_TEST_DIR/data/input --output processCcdOut --id + - setup obs_test + - setup pipe_tasks + - processCcd.py $OBS_TEST_DIR/data/input --output processCcdOut --id The data is read from the small repository in the `obs_test` package and written `./processCcdOut` (or whatever output you specified). Specifying `--id` with no values processes all data. @@ -136,23 +130,6 @@ class ProcessCcdTask(pipeBase.CmdLineTask): def __init__(self, butler=None, psfRefObjLoader=None, astromRefObjLoader=None, photoRefObjLoader=None, **kwargs): - """! - @param[in] butler The butler is passed to the refObjLoader constructor in case it is - needed. Ignored if the refObjLoader argument provides a loader directly. - @param[in] psfRefObjLoader An instance of LoadReferenceObjectsTasks that supplies an - external reference catalog for image characterization. An example of when this would - be used is when a CatalogStarSelector is used. May be None if the desired loader can - be constructed from the butler argument or all steps requiring a catalog are disabled. - @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks that supplies an - external reference catalog for astrometric calibration. May be None if the desired - loader can be constructed from the butler argument or all steps requiring a reference - catalog are disabled. - @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks that supplies an - external reference catalog for photometric calibration. May be None if the desired - loader can be constructed from the butler argument or all steps requiring a reference - catalog are disabled. - @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask - """ pipeBase.CmdLineTask.__init__(self, **kwargs) self.makeSubtask("isr") self.makeSubtask("charImage", butler=butler, refObjLoader=psfRefObjLoader) @@ -168,15 +145,21 @@ def runDataRef(self, sensorRef): - characterize image to estimate PSF and background - calibrate astrometry and photometry - @param sensorRef: butler data reference for raw data - - @return pipe_base Struct containing these fields: - - charRes: object returned by image characterization task; an lsst.pipe.base.Struct - that will include "background" and "sourceCat" fields - - calibRes: object returned by calibration task: an lsst.pipe.base.Struct - that will include "background" and "sourceCat" fields - - exposure: final exposure (an lsst.afw.image.ExposureF) - - background: final background model (an lsst.afw.math.BackgroundList) + Parameters + ---------- + sensorRef : + butler data reference for raw data + + Returns + ------- + result : `Struct` + pipe_base Struct containing these fields: + - ``charRes`` : object returned by image characterization task; an lsst.pipe.base.Struct + that will include "background" and "sourceCat" fields + - ``calibRes`` : object returned by calibration task: an lsst.pipe.base.Struct + that will include "background" and "sourceCat" fields + - ``exposure`` : final exposure (an lsst.afw.image.ExposureF) + - ``background`` : final background model (an lsst.afw.math.BackgroundList) """ self.log.info("Processing %s" % (sensorRef.dataId)) @@ -207,13 +190,22 @@ def runDataRef(self, sensorRef): @classmethod def _makeArgumentParser(cls): - """!Create and return an argument parser + """Create and return an argument parser + + Parameters + ---------- + cls : + the class object - @param[in] cls the class object - @return the argument parser for this task. + Returns + ------- + parser : + the argument parser for this task. + Notes + ----- This override is used to delay making the data ref list until the dataset type is known; - this is done in @ref parseAndRun. + this is done in parseAndRun. """ parser = pipeBase.ArgumentParser(name=cls._DefaultName) parser.add_id_argument(name="--id", diff --git a/python/lsst/pipe/tasks/propagateVisitFlags.py b/python/lsst/pipe/tasks/propagateVisitFlags.py index 36bc25b30..21db27b8f 100644 --- a/python/lsst/pipe/tasks/propagateVisitFlags.py +++ b/python/lsst/pipe/tasks/propagateVisitFlags.py @@ -28,7 +28,7 @@ class PropagateVisitFlagsConfig(Config): - """!Configuration for propagating flags to coadd""" + """Configuration for propagating flags to coadd""" flags = DictField(keytype=str, itemtype=float, default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, "calib_psf_reserved": 0.2, "calib_astrometry_used": 0.2, "calib_photometry_used": 0.2, @@ -48,24 +48,10 @@ class PropagateVisitFlagsConfig(Config): ## \} class PropagateVisitFlagsTask(Task): - r"""!Task to propagate flags from single-frame measurements to coadd measurements - -\anchor PropagateVisitFlagsTask_ - -\brief Propagate flags from individual visit measurements to coadd measurements - -\section pipe_tasks_propagateVisitFlagsTask_Contents Contents - - - \ref pipe_tasks_propagateVisitFlagsTask_Description - - \ref pipe_tasks_propagateVisitFlagsTask_Initialization - - \ref pipe_tasks_propagateVisitFlagsTask_Config - - \ref pipe_tasks_propagateVisitFlagsTask_Use - - \ref pipe_tasks_propagateVisitFlagsTask_Example - -\section pipe_tasks_propagateVisitFlagsTask_Description Description - -\copybrief PropagateVisitFlagsTask + """Task to propagate flags from single-frame measurements to coadd measurements +Notes +----- We want to be able to set a flag for sources on the coadds using flags that were determined from the individual visits. A common example is sources that were used for PSF determination, since we do not do any PSF determination @@ -94,38 +80,29 @@ class PropagateVisitFlagsTask(Task): The relative occurrence accounts for the edge of the field-of-view of the camera, but does not include chip gaps, bad or saturated pixels, etc. -\section pipe_tasks_propagateVisitFlagsTask_Initialization Initialization +Initialization Beyond the usual Task initialization, PropagateVisitFlagsTask also requires a schema for the catalog that is being constructed. -\section pipe_tasks_propagateVisitFlagsTask_Config Configuration parameters - -See \ref PropagateVisitFlagsConfig - -\section pipe_tasks_propagateVisitFlagsTask_Use Use - The 'run' method (described below) is the entry-point for operations. The 'getCcdInputs' staticmethod is provided as a convenience for retrieving the 'ccdInputs' (CCD inputs table) from an Exposure. -\copydoc run - -\section pipe_tasks_propagateVisitFlagsTask_Example Example - -\code{.py} -# Requires: -# * butler: data butler, for retrieving the CCD catalogs -# * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog) -# * coaddExposure: coadd (lsst.afw.image.Exposure) -from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig -config = PropagateVisitFlagsConfig() -config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag -config.matchRadius = 0.5 # Matching radius in arcsec -task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config) -ccdInputs = task.getCcdInputs(coaddExposure) -task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs()) -\endcode +.. code-block :: none + + # Requires: + # * butler: data butler, for retrieving the CCD catalogs + # * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog) + # * coaddExposure: coadd (lsst.afw.image.Exposure) + from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig + config = PropagateVisitFlagsConfig() + config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag + config.matchRadius = 0.5 # Matching radius in arcsec + task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config) + ccdInputs = task.getCcdInputs(coaddExposure) + task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs()) + """ ConfigClass = PropagateVisitFlagsConfig @@ -141,7 +118,7 @@ def getCcdInputs(coaddExposure): return coaddExposure.getInfo().getCoaddInputs().ccds def run(self, butler, coaddSources, ccdInputs, coaddWcs): - """!Propagate flags from individual visit measurements to coadd + """Propagate flags from individual visit measurements to coadd This requires matching the coadd source catalog to each of the catalogs from the inputs, and thresholding on the number of times a source is diff --git a/python/lsst/pipe/tasks/registerImage.py b/python/lsst/pipe/tasks/registerImage.py index 47b581b28..ed57de881 100644 --- a/python/lsst/pipe/tasks/registerImage.py +++ b/python/lsst/pipe/tasks/registerImage.py @@ -63,13 +63,23 @@ def run(self, inputSources, inputWcs, inputBBox, templateSources): 'matchRadius' of the configuration in order to facilitate source matching. We fit a new Wcs, but do NOT set it in the input exposure. - @param inputSources: Sources from input exposure - @param inputWcs: Wcs of input exposure - @param inputBBox: Bounding box of input exposure - @param templateSources: Sources from template exposure - @return Struct(matches: Matches between sources, - wcs: Wcs for input in frame of template, - ) + Parameters + ---------- + inputSources : + Sources from input exposure + inputWcs : + Wcs of input exposure + inputBBox : + Bounding box of input exposure + templateSources : + Sources from template exposure + + Returns + ------- + result : `Struct` + Struct with contents: + - ``matches`` : Matches between sources, + - ``wcs`` : Wcs for input in frame of template """ matches = self.matchSources(inputSources, templateSources) wcs = self.fitWcs(matches, inputWcs, inputBBox) @@ -81,9 +91,17 @@ def matchSources(self, inputSources, templateSources): The order of the input arguments matters (because the later Wcs fitting assumes a particular order). - @param inputSources: Source catalog of the input frame - @param templateSources: Source of the target frame - @return Match list + Parameters + ---------- + inputSources : + Source catalog of the input frame + templateSources : + Source of the target frame + + Returns + ------- + matches : + Match list """ matches = afwTable.matchRaDec(templateSources, inputSources, self.config.matchRadius*afwGeom.arcseconds) @@ -98,10 +116,19 @@ def fitWcs(self, matches, inputWcs, inputBBox): The fitting includes iterative sigma-clipping. - @param matches: List of matches (first is target, second is input) - @param inputWcs: Original input Wcs - @param inputBBox: Bounding box of input image - @return Wcs + Parameters + ---------- + matches : + List of matches (first is target, second is input) + inputWcs : + Original input Wcs + inputBBox : + Bounding box of input image + + Returns + ------- + wcs : + """ copyMatches = type(matches)(matches) refCoordKey = copyMatches[0].first.getTable().getCoordKey() @@ -141,11 +168,21 @@ def warpExposure(self, inputExp, newWcs, templateWcs, templateBBox): and other metadata), but we do not attempt to warp these to the template frame. - @param inputExp: Input exposure, to be warped - @param newWcs: Revised Wcs for input exposure - @param templateWcs: Target Wcs - @param templateBBox: Target bounding box - @return Warped exposure + Parameters + ---------- + inputExp : + Input exposure, to be warped + newWcs : + Revised Wcs for input exposure + templateWcs : + Target Wcs + templateBBox : + Target bounding box + + Returns + ------- + alignedExp : + Warped exposure """ warper = Warper.fromConfig(self.config.warper) copyExp = inputExp.Factory(inputExp.getMaskedImage(), newWcs) @@ -159,11 +196,21 @@ def warpSources(self, inputSources, newWcs, templateWcs, templateBBox): interest between the two frames. We therefore update only the sky and pixel coordinates. - @param inputSources: Sources on input exposure, to be warped - @param newWcs: Revised Wcs for input exposure - @param templateWcs: Target Wcs - @param templateBBox: Target bounding box - @return Warped sources + Parameters + ---------- + inputSources : + Sources on input exposure, to be warped + newWcs : + Revised Wcs for input exposure + templateWcs : + Target Wcs + templateBBox : + Target bounding box + + Returns + ------- + alignedSources : + Warped sources """ alignedSources = inputSources.copy(True) if not isinstance(templateBBox, afwGeom.Box2D): diff --git a/python/lsst/pipe/tasks/repair.py b/python/lsst/pipe/tasks/repair.py index 59d38a912..2305dde1b 100644 --- a/python/lsst/pipe/tasks/repair.py +++ b/python/lsst/pipe/tasks/repair.py @@ -28,6 +28,8 @@ from lsst.afw.display import getDisplay from lsst.pipe.tasks.interpImage import InterpImageTask +__all__ = ('RepairConfig', 'RepairTask') + class RepairConfig(pexConfig.Config): doInterpolate = pexConfig.Field( @@ -63,108 +65,39 @@ def setDefaults(self): class RepairTask(pipeBase.Task): - r"""! - @anchor RepairTask_ - - @brief Interpolate over defects in an exposure and handle cosmic rays + """This task operates on an lsst.afw.image.Exposure in place to interpolate over a set of + lsst.meas.algorithms.Defect objects. + It will also, optionally, find and interpolate any cosmic rays in the lsst.afw.image.Exposure. - @section pipe_tasks_repair_Contents Contents + Notes + ----- + The available debug variables in RepairTask are: - - @ref pipe_tasks_repair_Purpose - - @ref pipe_tasks_repair_Initialize - - @ref pipe_tasks_repair_IO - - @ref pipe_tasks_repair_Config - - @ref pipe_tasks_repair_Debug - - @ref pipe_tasks_repair_Example + display : + A dictionary containing debug point names as keys with frame number as value. Valid keys are: + repair.before : + display image before any repair is done + repair.after : + display image after cosmic ray and defect correction + displayCR : + If True, display the exposure on ds9's frame 1 and overlay bounding boxes around detects CRs. - @section pipe_tasks_repair_Purpose Description + Examples + -------- + To investigate the pipe_tasks_repair_Debug, put something like - @copybrief RepairTask + .. code-block :: none - This task operates on an lsst.afw.image.Exposure in place to interpolate over a set of - lsst.meas.algorithms.Defect objects. - It will also, optionally, find and interpolate any cosmic rays in the lsst.afw.image.Exposure. - - @section pipe_tasks_repair_Initialize Task initialization - - See: lsst.pipe.base.task.Task.__init__ - - @section pipe_tasks_repair_IO Inputs/Outputs to the run method - - @copydoc run - - @section pipe_tasks_repair_Config Configuration parameters - - See @ref RepairConfig - - @section pipe_tasks_repair_Debug Debug variables - - The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a - flag @c -d to import @b debug.py from your @c PYTHONPATH; see - Using lsstDebug to control debugging output for more about @b debug.py files. - - The available variables in RepairTask are: -
-
@c display -
A dictionary containing debug point names as keys with frame number as value. Valid keys are: -
-
repair.before -
display image before any repair is done -
repair.after -
display image after cosmic ray and defect correction -
-
@c displayCR -
If True, display the exposure on ds9's frame 1 and overlay bounding boxes around detects CRs. -
- @section pipe_tasks_repair_Example A complete example of using RepairTask - - This code is in runRepair.py in the examples directory, and can be run as @em e.g. - @code - examples/runRepair.py - @endcode - @dontinclude runRepair.py - Import the task. There are other imports. Read the source file for more info. - @skipline RepairTask - - For this example, we manufacture a test image to run on. - - First, create a pure Poisson noise image and a Psf to go with it. The mask plane - and variance can be constructed at the same time. - @skip poisson - @until mask - - Inject some cosmic rays and generate the Exposure. Exposures are MaskedImages (image + mask + variance) - with other metadata (e.g. Psf and Wcs objects). - @skip some CRs - @until setPsf - - Defects are represented as bad columns of random lengths. A defect list must be constructed to pass - on to the RepairTask. - @bug This is addressed in DM-963 - - @skip addDefects - @until push_back - - Finally, the exposure can be repaired. Create an instance of the task and run it. The exposure - is modified in place. - @skip RepairTask - @until repair.run - -
- To investigate the @ref pipe_tasks_repair_Debug, put something like - @code{.py} - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively - if name == "lsst.pipe.tasks.repair": - di.display = {'repair.before':2, 'repair.after':3} - di.displayCR = True - return di + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + if name == "lsst.pipe.tasks.repair": + di.display = {'repair.before':2, 'repair.after':3} + di.displayCR = True + return di lsstDebug.Info = DebugInfo - @endcode - into your debug.py file and run runRepair.py with the @c --debug flag. + into your debug.py file and run runRepair.py with the --debug flag. Conversion notes: @@ -180,22 +113,34 @@ def __init__(self, **kwargs): @pipeBase.timeMethod def run(self, exposure, defects=None, keepCRs=None): - """!Repair an Exposure's defects and cosmic rays - - @param[in, out] exposure lsst.afw.image.Exposure to process. Exposure must have a valid Psf. - Modified in place. - @param[in] defects an lsst.meas.algorithms.DefectListT object. If None, do no - defect correction. - @param[in] keepCRs don't interpolate over the CR pixels (defer to RepairConfig if None) - - @throws AssertionError with the following strings: - -
-
No exposure provided -
The object provided as exposure evaluates to False -
No PSF provided -
The Exposure has no associated Psf -
+ """Repair an Exposure's defects and cosmic rays + + Parameters + ---------- + exposure : `lsst.afw.image.Exposure` + Exposure must have a valid Psf. + Modified in place. + defects :`lsst.meas.algorithms.DefectListT` + If None, do no + defect correction. + keepCRs : + don't interpolate over the CR pixels (defer to RepairConfig if None) + + Raises + ------ + AssertionError + with the following strings: + + Notes + ----- + + .. code-block:: none + + No exposure provided + The object provided as exposure evaluates to False + No PSF provided + The Exposure has no associated Psf + """ assert exposure, "No exposure provided" psf = exposure.getPsf() @@ -218,8 +163,10 @@ def run(self, exposure, defects=None, keepCRs=None): def cosmicRay(self, exposure, keepCRs=None): """Mask cosmic rays - @param[in,out] exposure Exposure to process - @param[in] keepCRs Don't interpolate over the CR pixels (defer to pex_config if None) + exposure : + Exposure to process + keepCRs : + Don't interpolate over the CR pixels (defer to pex_config if None) """ import lsstDebug display = lsstDebug.Info(__name__).display diff --git a/python/lsst/pipe/tasks/repositoryIterator.py b/python/lsst/pipe/tasks/repositoryIterator.py index 46c86ad50..1147c957f 100644 --- a/python/lsst/pipe/tasks/repositoryIterator.py +++ b/python/lsst/pipe/tasks/repositoryIterator.py @@ -34,11 +34,21 @@ def _getDTypeList(keyTuple, valTuple): """Construct a numpy dtype for a data ID or repository ID - @param[in] keyTuple: ID key names, in order - @param[in] valTuple: a value tuple - @return numpy dtype as a list - - @warning: this guesses at string length (STR_PADDING + length of string in valTuple); + Parameters + ---------- + keyTuple : + ID key names, in order + valTuple : + a value tuple + + Returns + ------- + typeList : `list` + numpy dtype as a list + + Notes + ----- + warning: this guesses at string length (STR_PADDING + length of string in valTuple); longer strings will be truncated when inserted into numpy structured arrays """ typeList = [] @@ -54,6 +64,15 @@ def _getDTypeList(keyTuple, valTuple): class SourceData: """Accumulate a set of measurements from a set of source tables + Parameters + ---------- + datasetType : + dataset type for source + sourceKeyTuple : + list of keys of data items to extract from the source tables + + Notes + ----- To use: - specify the desired source measurements when constructing this object - call addSourceMetrics for each repository you harvest data from @@ -61,24 +80,23 @@ class SourceData: Data available after calling finalize: - self.sourceArr: a numpy structured array of shape (num repositories, num sources) - containing named columns for: - - source ID - - each data ID key - - each item of data extracted from the source table + containing named columns for: + - source ID + - each data ID key + - each item of data extracted from the source table - self.sourceIdDict: a dict of (source ID: index of axis 1 of self.sourceArr) - self.repoArr: a numpy structured array of shape (num repositories,) - containing a named column for each repository key (see RepositoryIterator) + containing a named column for each repository key (see RepositoryIterator) + + sources that had non-finite data (e.g. NaN) for every value extracted are silently omitted - @note: sources that had non-finite data (e.g. NaN) for every value extracted are silently omitted + Raises + ------ + RuntimeError + if sourceKeyTuple is empty """ def __init__(self, datasetType, sourceKeyTuple): - """ - @param[in] datasetType: dataset type for source - @param[in] sourceKeyTuple: list of keys of data items to extract from the source tables - - @raise RuntimeError if sourceKeyTuple is empty - """ if len(sourceKeyTuple) < 1: raise RuntimeError("Must specify at least one key in sourceKeyTuple") self.datasetType = datasetType @@ -105,22 +123,36 @@ def _getSourceMetrics(self, idKeyTuple, idValList, sourceTableList): Extracts a set of source measurements (specified by sourceKeyTuple) from a list of source tables (one per data ID) and saves them as a dict of source ID: list of data - @param[in] idKeyTuple: a tuple of data ID keys; must be the same for each call - @param[in] idValList: a list of data ID value tuples; + Parameters + ---------- + idKeyTuple : + a tuple of data ID keys; must be the same for each call + idValList : + a list of data ID value tuples; each tuple contains values in the order in idKeyTuple - @param[in] sourceTableList: a list of source tables, one per entry in idValList + sourceTableList : + a list of source tables, one per entry in idValList - @return a dict of source id: data id tuple + source data tuple + Returns + ------- + dataDict : `dict` + a dict of source id: data id tuple + source data tuple where source data tuple order matches sourceKeyTuple and data id tuple matches self._idKeyTuple (which is set from the first idKeyTuple) - @raise RuntimeError if idKeyTuple is different than it was for the first call. + Raises + ------ + RuntimeError + if idKeyTuple is different than it was for the first call. + Notes + ----- GetRepositoryDataTask.run returns idKeyTuple and idValList; you can easily make a subclass of GetRepositoryDataTask that also returns sourceTableList. Updates instance variables: - self._idKeyTuple if not already set. + """ if self._idKeyTuple is None: self._idKeyTuple = tuple(idKeyTuple) @@ -155,17 +187,33 @@ def addSourceMetrics(self, repoInfo, idKeyTuple, idValList, sourceTableList): Once you have accumulated all source measurements, call finalize to process the data. - @param[in] repoInfo: a RepositoryInfo instance - @param[in] idKeyTuple: a tuple of data ID keys; must be the same for each call - @param[in] idValList: a list of data ID value tuples; + Parameters + ---------- + repoInfo : + a RepositoryInfo instance + idKeyTuple : + a tuple of data ID keys; must be the same for each call + idValList : + a list of data ID value tuples; each tuple contains values in the order in idKeyTuple - @param[in] sourceTableList: a list of source tables, one per entry in idValList + sourceTableList : + a list of source tables, one per entry in idValList + + Raises + ------ + RuntimeError + if idKeyTuple is different than it was for the first call. + + Returns + ------- - @raise RuntimeError if idKeyTuple is different than it was for the first call. + result : `len(dataDict)` + number of sources + Notes + ----- Accumulates the data in temporary cache self._tempDataList. - @return number of sources """ if self._repoKeyTuple is None: self._repoKeyTuple = repoInfo.keyTuple @@ -228,14 +276,16 @@ def __init__(self, keyTuple, valTuple, dtype, name): class RepositoryIterator: """Iterate over a set of data repositories that use a naming convention based on parameter values + + Parameters + ---------- + formatStr : + format string using dictionary notation, e.g.: "%(foo)s_%(bar)d" + dataDict : + name=valueList pairs """ def __init__(self, formatStr, **dataDict): - """Construct a repository iterator from a dict of name: valueList - - @param[in] formatStr: format string using dictionary notation, e.g.: "%(foo)s_%(bar)d" - @param[in] **dataDict: name=valueList pairs - """ self._formatStr = formatStr self._keyTuple = tuple(sorted(dataDict.keys())) self._valListOfLists = [numpy.array(dataDict[key]) for key in self._keyTuple] @@ -260,7 +310,10 @@ def __len__(self): def format(self, valDict): """Return formatted string for a specified value dictionary - @param[in] valDict: a dict of key: value pairs that identify a repository + Parameters + ---------- + valDict : + a dict of key: value pairs that identify a repository """ return self._formatStr % valDict diff --git a/python/lsst/pipe/tasks/scaleVariance.py b/python/lsst/pipe/tasks/scaleVariance.py index 2ca0a2743..500e4051b 100644 --- a/python/lsst/pipe/tasks/scaleVariance.py +++ b/python/lsst/pipe/tasks/scaleVariance.py @@ -26,6 +26,8 @@ from lsst.pipe.base import Task from lsst.meas.algorithms import SubtractBackgroundTask +__all__ = ('ScaleVarianceConfig', 'ScaleVarianceTask') + class ScaleVarianceConfig(Config): background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction") diff --git a/python/lsst/pipe/tasks/scaleZeroPoint.py b/python/lsst/pipe/tasks/scaleZeroPoint.py index baf796808..ce1a53fd1 100644 --- a/python/lsst/pipe/tasks/scaleZeroPoint.py +++ b/python/lsst/pipe/tasks/scaleZeroPoint.py @@ -26,20 +26,22 @@ import lsst.pipe.base as pipeBase from lsst.pipe.tasks.selectImages import BaseSelectImagesTask -__all__ = ["ImageScaler", "SpatialImageScaler", "ScaleZeroPointTask"] +__all__ = ("ImageScaler", "SpatialImageScaler", "ScaleZeroPointTask", "ScaleZeroPointConfig", + "SpatialScaleZeroPointConfig", "SpatialScaleZeroPointTask") class ImageScaler: """A class that scales an image This version uses a single scalar. Fancier versions may use a spatially varying scale. + + Parameters + ---------- + scale : + scale correction to apply (see scaleMaskedImage); """ def __init__(self, scale=1.0): - """Construct an ImageScaler - - @param[in] scale: scale correction to apply (see scaleMaskedImage); - """ self._scale = scale def scaleMaskedImage(self, maskedImage): @@ -57,18 +59,25 @@ class SpatialImageScaler(ImageScaler): Interpolates only when scaleMaskedImage() or getInterpImage() is called. Currently the only type of 'interpolation' implemented is CONSTANT which calculates the mean. + + Parameters + ---------- + interpStyle : + interpolation style (CONSTANT is only option) + xList : + list of X pixel positions + yList : + list of Y pixel positions + scaleList : + list of multiplicative scale factors at (x,y) + + Raises + ------ + RuntimeError + if the lists have different lengths """ def __init__(self, interpStyle, xList, yList, scaleList): - """Constructor - - @param[in] interpStyle: interpolation style (CONSTANT is only option) - @param[in] xList: list of X pixel positions - @param[in] yList: list of Y pixel positions - @param[in] scaleList: list of multiplicative scale factors at (x,y) - - @raise RuntimeError if the lists have different lengths - """ if len(xList) != len(yList) or len(xList) != len(scaleList): raise RuntimeError( "len(xList)=%s len(yList)=%s, len(scaleList)=%s but all lists must have the same length" % @@ -82,7 +91,10 @@ def __init__(self, interpStyle, xList, yList, scaleList): def scaleMaskedImage(self, maskedImage): """Apply scale correction to the specified masked image - @param[in,out] image to scale; scale is applied in place + Parameters + ---------- + maskedImage : + image to scale; scale is applied in place """ scale = self.getInterpImage(maskedImage.getBBox()) maskedImage *= scale @@ -90,7 +102,10 @@ def scaleMaskedImage(self, maskedImage): def getInterpImage(self, bbox): """Return an image containing the scale correction with same bounding box as supplied. - @param[in] bbox: integer bounding box for image (afwGeom.Box2I) + Parameters + ---------- + bbox : + integer bounding box for image (afwGeom.Box2I) """ npoints = len(self._xList) @@ -138,8 +153,6 @@ class ScaleZeroPointTask(pipeBase.Task): _DefaultName = "scaleZeroPoint" def __init__(self, *args, **kwargs): - """Construct a ScaleZeroPointTask - """ pipeBase.Task.__init__(self, *args, **kwargs) # flux at mag=0 is 10^(zeroPoint/2.5) because m = -2.5*log10(F/F0) @@ -150,12 +163,19 @@ def __init__(self, *args, **kwargs): def run(self, exposure, dataRef=None): """Scale the specified exposure to the desired photometric zeropoint - @param[in,out] exposure: exposure to scale; masked image is scaled in place - @param[in] dataRef: dataRef for exposure. - Not used, but in API so that users can switch between spatially variant - and invariant tasks - @return a pipeBase.Struct containing: - - imageScaler: the image scaling object used to scale exposure + Parameters + ---------- + exposure : + exposure to scale; masked image is scaled in place + dataRef : + dataRef for exposure. + Not used, but in API so that users can switch between spatially variant + and invariant tasks + Returns + ------- + result ; `pipeBase.Struct` + a pipeBase.Struct containing: + - ``imageScaler`` : the image scaling object used to scale exposure """ imageScaler = self.computeImageScaler(exposure=exposure, dataRef=dataRef) mi = exposure.getMaskedImage() @@ -167,10 +187,14 @@ def run(self, exposure, dataRef=None): def computeImageScaler(self, exposure, dataRef=None): """Compute image scaling object for a given exposure. - @param[in] exposure: exposure for which scaling is desired - @param[in] dataRef: dataRef for exposure. - Not used, but in API so that users can switch between spatially variant - and invariant tasks + Parameters + ---------- + exposure : + exposure for which scaling is desired + dataRef : + dataRef for exposure. + Not used, but in API so that users can switch between spatially variant + and invariant tasks """ scale = self.scaleFromCalib(exposure.getCalib()).scale return ImageScaler(scale) @@ -178,7 +202,10 @@ def computeImageScaler(self, exposure, dataRef=None): def getCalib(self): """Get desired Calib - @return calibration (lsst.afw.image.Calib) with fluxMag0 set appropriately for config.zeroPoint + Returns + ------- + result : `self._calib` + calibration (lsst.afw.image.Calib) with fluxMag0 set appropriately for config.zeroPoint """ return self._calib @@ -187,13 +214,18 @@ def scaleFromCalib(self, calib): Compute scale, such that if pixelCalib describes the photometric zeropoint of a pixel then the following scales that pixel to the photometric zeropoint specified by config.zeroPoint: - scale = computeScale(pixelCalib) - pixel *= scale - - @return a pipeBase.Struct containing: - - scale, as described above. - - @note: returns a struct to leave room for scaleErr in a future implementation. + scale = computeScale(pixelCalib) + pixel = scale + + Returns + ------- + result : `pipeBase.Struct` + a pipeBase.Struct containing: + - ``scale`` : computeScale(pixelCalib) + + Notes + ----- + returns a struct to leave room for scaleErr in a future implementation. """ fluxAtZeroPoint = calib.getFlux(self.config.zeroPoint) return pipeBase.Struct( @@ -205,9 +237,18 @@ def scaleFromFluxMag0(self, fluxMag0): This is a wrapper around scaleFromCalib, which see for more information - @param[in] fluxMag0 - @return a pipeBase.Struct containing: - - scale, as described in scaleFromCalib. + Parameters + ---------- + fluxMag0 : + + Returns + ------- + result : `pipeBase.Struct` + a pipeBase.Struct containing: + + - ``scale`` : Compute scale, such that if pixelCalib describes the photometric + zeropoint of a pixel then the following scales that pixel to the photometric + zeropoint specified by config.zeroPoint: """ calib = afwImage.Calib() calib.setFluxMag0(fluxMag0) @@ -227,11 +268,18 @@ def __init__(self, *args, **kwargs): def run(self, exposure, dataRef): """Scale the specified exposure to the desired photometric zeropoint - @param[in,out] exposure: exposure to scale; masked image is scaled in place - @param[in] dataRef: dataRef for exposure - - @return a pipeBase.Struct containing: - - imageScaler: the image scaling object used to scale exposure + Parameters + ---------- + exposure : + exposure to scale; masked image is scaled in place + dataRef : + dataRef for exposure + + Returns + ------- + result : `pipeBase.Struct` + a pipeBase.Struct containing: + - ``imageScaler`` : the image scaling object used to scale exposure """ imageScaler = self.computeImageScaler(exposure=exposure, dataRef=dataRef) mi = exposure.getMaskedImage() @@ -243,10 +291,17 @@ def run(self, exposure, dataRef): def computeImageScaler(self, exposure, dataRef): """Compute image scaling object for a given exposure. - @param[in] exposure: exposure for which scaling is desired. Only wcs and bbox are used. - @param[in] dataRef: dataRef of exposure - dataRef.dataId used to retrieve all applicable fluxMag0's from a database. - @return a SpatialImageScaler + Parameters + ---------- + exposure : + exposure for which scaling is desired. Only wcs and bbox are used. + dataRef : + dataRef of exposure + dataRef.dataId used to retrieve all applicable fluxMag0's from a database. + + Returns + ------- + SpatialImageScaler """ wcs = exposure.getWcs() diff --git a/python/lsst/pipe/tasks/selectImages.py b/python/lsst/pipe/tasks/selectImages.py index f7345aa5e..2b72c98db 100644 --- a/python/lsst/pipe/tasks/selectImages.py +++ b/python/lsst/pipe/tasks/selectImages.py @@ -53,16 +53,17 @@ class DatabaseSelectImagesConfig(pexConfig.Config): class BaseExposureInfo(pipeBase.Struct): """Data about a selected exposure + + Parameters + ---------- + dataId : + data ID of exposure (a dict) + coordList : + ICRS coordinates of the corners of the exposure (list of lsst.afw.geom.SpherePoint) + plus any others items that are desired """ def __init__(self, dataId, coordList): - """Create exposure information that can be used to generate data references - - The object has the following fields: - - dataId: data ID of exposure (a dict) - - coordList: ICRS coordinates of the corners of the exposure (list of lsst.afw.geom.SpherePoint) - plus any others items that are desired - """ super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList) @@ -76,21 +77,30 @@ class BaseSelectImagesTask(pipeBase.Task): def run(self, coordList): """Select images suitable for coaddition in a particular region - @param[in] coordList: list of coordinates defining region of interest; if None then select all images - subclasses may add additional keyword arguments, as required + Parameters + ---------- + coordList : + list of coordinates defining region of interest; if None then select all images + subclasses may add additional keyword arguments, as required - @return a pipeBase Struct containing: - - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo), - which have at least the following fields: - - dataId: data ID dictionary - - coordList: ICRS coordinates of the corners of the exposure (list of lsst.afw.geom.SpherePoint) + Returns + ------- + result : `pipeBase.Struct` + return a pipeBase Struct containing: + - ``exposureInfoList`` : a list of exposure information objects (subclasses of BaseExposureInfo), + which have at least the following fields: + - ``dataId`` : data ID dictionary + - ``coordList`` : ICRS coordinates of the corners of the exposure (list of lsst.afw.geom.SpherePoint) """ raise NotImplementedError() def _runArgDictFromDataId(self, dataId): """Extract keyword arguments for run (other than coordList) from a data ID - @return keyword arguments for run (other than coordList), as a dict + Returns + ------- + result : + keyword arguments for run (other than coordList), as a dict """ raise NotImplementedError() @@ -102,13 +112,23 @@ def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[] be used to further restrict the selection, providing the user with additional control over the selection. - @param[in] dataRef: data reference; must contain any extra keys needed by the subclass - @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky - @param[in] makeDataRefList: if True, return dataRefList - @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection - @return a pipeBase Struct containing: - - exposureInfoList: a list of objects derived from ExposureInfo - - dataRefList: a list of data references (None if makeDataRefList False) + Parameters + ---------- + dataRef : + data reference; must contain any extra keys needed by the subclass + coordList : + list of coordinates defining region of interest; if None, search the whole sky + makeDataRefList : + if True, return dataRefList + selectDataList : + List of SelectStruct with dataRefs to consider for selection + + Returns + ------- + result : `pipeBase.Struct` + return a pipeBase Struct containing: + - ``exposureInfoList``: a list of objects derived from ExposureInfo + - ``dataRefList``: a list of data references (None if makeDataRefList False) """ runArgDict = self._runArgDictFromDataId(dataRef.dataId) exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList @@ -181,10 +201,16 @@ def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[] directly because the standard for the inputs to ConvexPolygon are pretty high and we don't want to be responsible for reaching them. - @param dataRef: Data reference for coadd/tempExp (with tract, patch) - @param coordList: List of ICRS coordinates (lsst.afw.geom.SpherePoint) specifying boundary of patch - @param makeDataRefList: Construct a list of data references? - @param selectDataList: List of SelectStruct, to consider for selection + Parameters + ---------- + dataRef : + Data reference for coadd/tempExp (with tract, patch) + coordList : + List of ICRS coordinates (lsst.afw.geom.SpherePoint) specifying boundary of patch + makeDataRefList : + Construct a list of data references? + selectDataList : + List of SelectStruct, to consider for selection """ dataRefList = [] exposureInfoList = [] @@ -272,15 +298,21 @@ def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[] adaptive second moments of the star and the PSF. The criteria are: - - the median of the ellipticty residuals - - the robust scatter of the size residuals (using the median absolute deviation) - - the robust scatter of the size residuals scaled by the square of - the median size - - @param dataRef: Data reference for coadd/tempExp (with tract, patch) - @param coordList: List of ICRS coordinates (lsst.afw.geom.SpherePoint) specifying boundary of patch - @param makeDataRefList: Construct a list of data references? - @param selectDataList: List of SelectStruct, to consider for selection + - the median of the ellipticty residuals + - the robust scatter of the size residuals (using the median absolute deviation) + - the robust scatter of the size residuals scaled by the square of + the median size + + Parameters + ---------- + dataRef : + Data reference for coadd/tempExp (with tract, patch) + coordList : + List of ICRS coordinates (lsst.afw.geom.SpherePoint) specifying boundary of patch + makeDataRefList : + Construct a list of data references? + selectDataList : + List of SelectStruct, to consider for selection """ result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList, selectDataList) @@ -385,10 +417,10 @@ def runDataRef(self, dataRef, coordList, makeDataRefList=True, result : `lsst.pipe.base.Struct` Result struct with components: - ``exposureList``: the selected exposures - (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`). + (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`). - ``dataRefList``: the optional data references corresponding to - each element of ``exposureList`` - (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`). + each element of ``exposureList`` + (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`). """ if self.config.nImagesMax <= 0: raise RuntimeError(f"nImagesMax must be greater than zero: {self.config.nImagesMax}") diff --git a/python/lsst/pipe/tasks/setConfigFromEups.py b/python/lsst/pipe/tasks/setConfigFromEups.py index d04301c8a..e8a903ba0 100644 --- a/python/lsst/pipe/tasks/setConfigFromEups.py +++ b/python/lsst/pipe/tasks/setConfigFromEups.py @@ -24,12 +24,21 @@ def setAstrometryConfigFromEups(config, menu): """Set the astrometry configuration according to the astrometry_net_data being used The menu is a dict mapping the astrometry_net_data version to a dict of configuration - values to apply. The menu key may also be a glob. For example: - menu = { "ps1*": {}, # No changes - "ps1-without-y": { "solver.filterMap": {"y": "z"} }, # No y-band in this specific version - "sdss*": { "solver.filterMap": {"y": "z"} }, # No y-band, use z instead - "2mass*": { "solver.filterMap": {"y": "J"} }, # No y-band, use J instead - } + values to apply. The menu key may also be a glob. + + + Examples + -------- + For example: + + .. code-block:: none + + menu = { "ps1": {}, # No changes + "ps1-without-y": { "solver.filterMap": {"y": "z"} }, # No y-band in this specific version + "sdss": { "solver.filterMap": {"y": "z"} }, # No y-band, use z instead + "2mass": { "solver.filterMap": {"y": "J"} }, # No y-band, use J instead + } + """ version = getAndVersion() diff --git a/python/lsst/pipe/tasks/setPrimaryFlags.py b/python/lsst/pipe/tasks/setPrimaryFlags.py index b1143bfc3..3c4abc3a1 100644 --- a/python/lsst/pipe/tasks/setPrimaryFlags.py +++ b/python/lsst/pipe/tasks/setPrimaryFlags.py @@ -25,6 +25,8 @@ from lsst.pipe.base import Task from lsst.afw.geom import Box2D +__all__ = ('SetPrimaryFlagsConfig', 'SetPrimaryFlagsTask') + class SetPrimaryFlagsConfig(Config): nChildKeyName = Field(dtype=str, default="deblend_nChild", @@ -57,13 +59,20 @@ def __init__(self, schema, **kwargs): def run(self, sources, skyMap, tractInfo, patchInfo, includeDeblend=True): """Set is-primary and related flags on sources - @param[in,out] sources a SourceTable + Parameters + ---------- + sources : + a SourceTable - reads centroid fields and an nChild field - writes is-patch-inner, is-tract-inner and is-primary flags - @param[in] skyMap sky tessellation object (subclass of lsst.skymap.BaseSkyMap) - @param[in] tractInfo tract object (subclass of lsst.skymap.TractInfo) - @param[in] patchInfo patch object (subclass of lsst.skymap.PatchInfo) - @param[in] includeDeblend include deblend information in isPrimary? + skyMap : + sky tessellation object (subclass of lsst.skymap.BaseSkyMap) + tractInfo : + tract object (subclass of lsst.skymap.TractInfo) + patchInfo : + patch object (subclass of lsst.skymap.PatchInfo) + includeDeblend : + include deblend information in isPrimary? """ nChildKey = None if includeDeblend: diff --git a/python/lsst/pipe/tasks/snapCombine.py b/python/lsst/pipe/tasks/snapCombine.py index 6e204964d..50b87e7ac 100644 --- a/python/lsst/pipe/tasks/snapCombine.py +++ b/python/lsst/pipe/tasks/snapCombine.py @@ -37,7 +37,7 @@ class InitialPsfConfig(pexConfig.Config): - """!Describes the initial PSF used for detection and measurement before we do PSF determination.""" + """Describes the initial PSF used for detection and measurement before we do PSF determination.""" model = pexConfig.ChoiceField( dtype=str, @@ -133,34 +133,26 @@ def validate(self): class SnapCombineTask(pipeBase.Task): - r"""! - \anchor SnapCombineTask_ - - \brief Combine snaps. + """ + The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a + flag -d to import debug.py from your PYTHONPATH; see + "http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html" + Using lsstDebug to control debugging output for more about debug.py files. - \section pipe_tasks_snapcombine_Contents Contents + Notes + ----- + The available variables in SnapCombineTask are: - - \ref pipe_tasks_snapcombine_Debug + display + A dictionary containing debug point names as keys with frame number as value. Valid keys are: - \section pipe_tasks_snapcombine_Debug Debug variables + .. code-block:: none - The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a - flag \c -d to import \b debug.py from your \c PYTHONPATH; see - Using lsstDebug to control debugging output for more about \b debug.py files. + repair0 + Display the first snap after repairing. + repair1 + Display the second snap after repairing. - The available variables in SnapCombineTask are: -
-
\c display -
A dictionary containing debug point names as keys with frame number as value. Valid keys are: -
-
repair0 -
Display the first snap after repairing. -
repair1 -
Display the second snap after repairing. -
-
-
""" ConfigClass = SnapCombineConfig _DefaultName = "snapCombine" @@ -179,12 +171,21 @@ def __init__(self, *args, **kwargs): def run(self, snap0, snap1, defects=None): """Combine two snaps - @param[in] snap0: snapshot exposure 0 - @param[in] snap1: snapshot exposure 1 - @defects[in] defect list (for repair task) - @return a pipe_base Struct with fields: - - exposure: snap-combined exposure - - sources: detected sources, or None if detection not performed + Parameters + ---------- + snap0 : + snapshot exposure 0 + snap1 : + snapshot exposure 1 + defect : + list (for repair task) + + Returns + ------- + result : `Struct` + a pipe_base Struct with fields: + - ``exposure`` : snap-combined exposure + - ``sources`` : detected sources, or None if detection not performed """ # initialize optional outputs sources = None @@ -249,9 +250,17 @@ def run(self, snap0, snap1, defects=None): def addSnaps(self, snap0, snap1): """Add two snap exposures together, returning a new exposure - @param[in] snap0 snap exposure 0 - @param[in] snap1 snap exposure 1 - @return combined exposure + Parameters + ---------- + snap0 : + snap exposure 0 + snap1 : + snap exposure 1 + + Returns + ------- + combinedExp : + combined exposure """ self.log.info("snapCombine addSnaps") @@ -288,12 +297,19 @@ def fixMetadata(self, combinedMetadata, metadata0, metadata1): which have data type restrictions. To handle other data types (such as sexagesimal positions and ISO dates) you must supplement this method with your own code. - @param[in,out] combinedMetadata metadata of combined exposure; + Parameters + ---------- + combinedMetadata : + metadata of combined exposure; on input this is a deep copy of metadata0 (a PropertySet) - @param[in] metadata0 metadata of snap0 (a PropertySet) - @param[in] metadata1 metadata of snap1 (a PropertySet) - - @note the inputs are presently PropertySets due to ticket #2542. However, in some sense + metadata0 : + metadata of snap0 (a PropertySet) + metadata1 : + metadata of snap1 (a PropertySet) + + Notes + ----- + the inputs are presently PropertySets due to ticket #2542. However, in some sense they are just PropertyLists that are missing some methods. In particular: comments and order are preserved if you alter an existing value with set(key, value). """ @@ -325,8 +341,14 @@ def fixMetadata(self, combinedMetadata, metadata0, metadata1): def makeInitialPsf(self, exposure, fwhmPix=None): """Initialise the detection procedure by setting the PSF and WCS - @param exposure Exposure to process - @return PSF, WCS + Parameters + ---------- + exposure : Exposure to process + + Returns + ------- + PSF : + WCS : """ assert exposure, "No exposure provided" wcs = exposure.getWcs() diff --git a/python/lsst/pipe/tasks/transformMeasurement.py b/python/lsst/pipe/tasks/transformMeasurement.py index 7f535d8fb..45bfd6e10 100644 --- a/python/lsst/pipe/tasks/transformMeasurement.py +++ b/python/lsst/pipe/tasks/transformMeasurement.py @@ -28,7 +28,7 @@ def makeContiguous(catalog): - """!Return a version of the input catalog which is contiguous in memory.""" + """Return a version of the input catalog which is contiguous in memory.""" if not catalog.isContiguous(): return catalog.copy(deep=True) else: @@ -36,7 +36,7 @@ def makeContiguous(catalog): class TransformConfig(pexConfig.Config): - """!Configuration for TransformTask.""" + """Configuration for TransformTask.""" copyFields = pexConfig.ListField( dtype=str, doc="Fields to copy from input to output catalog without transformation", @@ -52,24 +52,27 @@ class TransformConfig(pexConfig.Config): class TransformTask(pipeBase.Task): - r"""! - \anchor TransformTask_ - - \brief Transform a SourceCatalog containing raw measurements to calibrated form. - - \section pipe_tasks_transform_Contents Contents - - - \ref pipe_tasks_transform_purpose - - \ref pipe_tasks_transform_initialize - - \ref pipe_tasks_transform_invoke - - \section pipe_tasks_transform_purpose Description - - Given a set of measurement algorithms with their associated configuration, + """Given a set of measurement algorithms with their associated configuration, the table of source measurements they have produced, and information about an associated WCS and calibration, transform the raw measurement output to a calibrated form. + Parameters + ---------- + measConfig : + Configuration for the measurement task which + produced the measurments being transformed. + inputSchema : + The schema of the input catalog. + outputDataset : + The butler dataset type of the output catalog. + args : + Passed through to pipeBase.Task.__init__() + kwargs : + Passed through to pipeBase.Task.__init__() + + Notes + ----- Transformations are defined on a per-measurement-plugin basis. In addition, a configurable set of fields may be simply copied from the input to the output catalog. @@ -82,28 +85,11 @@ class TransformTask(pipeBase.Task): functionality for reading or writing data from a Butler: rather, per-dataset-type command line tasks are provided to obtain the appropriate information from a Butler (or elsewhere) and then delegate to this task. - - \section pipe_tasks_transform_initialize Task initialization - - \copydoc \_\_init\_\_ - - \section pipe_tasks_transform_invoke Task invocation - - \copydoc run """ ConfigClass = TransformConfig _DefaultName = "transform" def __init__(self, measConfig, inputSchema, outputDataset, *args, **kwargs): - """!Initialize TransformTask. - - @param[in] measConfig Configuration for the measurement task which - produced the measurments being transformed. - @param[in] inputSchema The schema of the input catalog. - @param[in] outputDataset The butler dataset type of the output catalog. - @param[in] *args Passed through to pipeBase.Task.__init__() - @param[in] *kwargs Passed through to pipeBase.Task.__init__() - """ pipeBase.Task.__init__(self, *args, **kwargs) # This task can be used to generate multiple different output dataset types. We @@ -124,18 +110,26 @@ def __init__(self, measConfig, inputSchema, outputDataset, *args, **kwargs): self.transforms.append(transformClass(config, name, self.mapper)) def getSchemaCatalogs(self): - """!Return a dict containing an empty catalog representative of this task's output.""" + """Return a dict containing an empty catalog representative of this task's output.""" transformedSrc = afwTable.BaseCatalog(self.mapper.getOutputSchema()) return {self.outputDataset: transformedSrc} def run(self, inputCat, wcs, calib): - """!Transform raw source measurements to calibrated quantities. - - @param[in] inputCat SourceCatalog of sources to transform. - @param[in] wcs The world coordinate system under which transformations will take place. - @param[in] calib The calibration under which transformations will take place. - - @return A BaseCatalog containing the transformed measurements. + """Transform raw source measurements to calibrated quantities. + + Parameters + ---------- + inputCat : + SourceCatalog of sources to transform. + wcs : + The world coordinate system under which transformations will take place. + calib : + The calibration under which transformations will take place. + + Returns + ------- + outputcat : + A BaseCatalog containing the transformed measurements. """ outputCat = afwTable.BaseCatalog(self.mapper.getOutputSchema()) outputCat.extend(inputCat, mapper=self.mapper) @@ -151,7 +145,7 @@ def run(self, inputCat, wcs, calib): class RunTransformConfig(pexConfig.Config): - """!Configuration for RunTransformTaskBase derivatives.""" + """Configuration for RunTransformTaskBase derivatives.""" transform = pexConfig.ConfigurableField( doc="Subtask which performs transformations", target=TransformTask @@ -163,32 +157,19 @@ class RunTransformConfig(pexConfig.Config): class RunTransformTaskBase(pipeBase.CmdLineTask): - r"""! - \anchor RunTransformTaskBase_ - - \brief Command line interface for TransformTask. - - \section pipe_tasks_transform_Contents Contents - - - \ref pipe_tasks_runtransform_purpose - - \ref pipe_tasks_runtransform_invoke - - \section pipe_tasks_runtransform_purpose Description + """Command line interface for TransformTask. + Notes + ----- Provides a command-line task which can be used to run TransformTask. - - Loads a plugin registry based on configuration; - Loads configuration for the measurement task which was applied from a repository; - Loads the SourceCatalog input schema from a repository; - For each input dataRef, reads the SourceCatalog, WCS and calibration from the - repository and executes TransformTask. + repository and executes TransformTask. This is not a fully-fledged command line task: it requires specialization to a particular source type by defining the variables indicated below. - - \section pipe_tasks_runtransform_invoke Task invocation - - \copydoc run """ RunnerClass = pipeBase.ButlerInitializedTaskRunner ConfigClass = RunTransformConfig @@ -207,7 +188,7 @@ class RunTransformTaskBase(pipeBase.CmdLineTask): @property def inputSchemaType(self): - """! + """ The Butler dataset type for the schema of the input source catalog. By default, we append `_schema` to the input source type. Subclasses may customize @@ -217,17 +198,17 @@ def inputSchemaType(self): @property def outputDataset(self): - """! + """ The Butler dataset type for the schema of the output catalog. - By default, we prepend `transformed_` to the input source type. Subclasses may + By default, we prepend `transformed` to the input source type. Subclasses may customize if required. """ return 'transformed_' + self.sourceType @property def measurementConfig(self): - """! + """ The configuration of the measurement operation used to generate the input catalog. By default we look for `measurement` under the root configuration of the @@ -245,14 +226,20 @@ def __init__(self, *args, **kwargs): @pipeBase.timeMethod def runDataRef(self, dataRef): - """!Transform the source catalog referred to by dataRef. + """Transform the source catalog referred to by dataRef. - The result is both returned and written as dataset type "transformed_" + the input + The result is both returned and written as dataset type "transformed" + the input source dataset type to the provided dataRef. - @param[in] dataRef Data reference for source catalog & calibrated exposure. + Parameters + ---------- + dataRef : + Data reference for source catalog & calibrated exposure. - @returns A BaseCatalog containing the transformed measurements. + Returns + ------- + outputCat : + A BaseCatalog containing the transformed measurements. """ inputCat = dataRef.get(self.sourceType) wcs = dataRef.get(self.calexpType).getWcs() @@ -270,12 +257,9 @@ def runDataRef(self, dataRef): ## \} class SrcTransformTask(RunTransformTaskBase): - """! - \anchor SrcTransformTask_ + """Transform ``src`` measuremenents to calibrated form. - \brief Transform ``src`` measuremenents to calibrated form. - - This is a specialization of \ref RunTransformTaskBase_ "RunTransformTaskBase" which + This is a specialization of RunTransformTaskBase "RunTransformTaskBase" which operates on ``src`` measurements. Refer to the parent documentation for details. """ _DefaultName = "transformSrcMeasurement" @@ -295,12 +279,9 @@ def measurementConfig(self): ## \} class ForcedSrcTransformTask(RunTransformTaskBase): - """! - \anchor ForcedSrcTransformTask_ - - \brief Transform ``forced_src`` measuremenents to calibrated form. + """Transform ``forced_src`` measuremenents to calibrated form. - This is a specialization of \ref RunTransformTaskBase_ "RunTransformTaskBase" which + This is a specialization of RunTransformTaskBase "RunTransformTaskBase" which operates on ``forced_src`` measurements. Refer to the parent documentation for details. """ _DefaultName = "transformForcedSrcMeasurement" @@ -316,12 +297,9 @@ class ForcedSrcTransformTask(RunTransformTaskBase): ## \} class CoaddSrcTransformTask(RunTransformTaskBase): - """! - \anchor CoaddSrcTransformTask_ - - \brief Transform measuremenents made on coadds to calibrated form. + """Transform measuremenents made on coadds to calibrated form. - This is a specialization of \ref RunTransformTaskBase_ "RunTransformTaskBase" which + This is a specialization of RunTransformTaskBase "RunTransformTaskBase" which operates on measurements made on coadds. Refer to the parent documentation for details. """ _DefaultName = "transformCoaddSrcMeasurement" From 07a000984f74aa7585720f831bc3fd040b9dba60 Mon Sep 17 00:00:00 2001 From: Gabor Kovacs Date: Mon, 17 Dec 2018 21:36:04 -0800 Subject: [PATCH 2/6] Manual documentation and __all__ updates following rebase to master. See ticket comment on details. Minor style corrections. --- python/lsst/pipe/tasks/colorterms.py | 6 ++++-- python/lsst/pipe/tasks/imageDifference.py | 6 ++++-- python/lsst/pipe/tasks/ingest.py | 4 ++-- python/lsst/pipe/tasks/ingestPgsql.py | 2 +- python/lsst/pipe/tasks/makeDiscreteSkyMap.py | 2 +- python/lsst/pipe/tasks/makeSkyMap.py | 2 +- python/lsst/pipe/tasks/matchBackgrounds.py | 2 +- python/lsst/pipe/tasks/measurePsf.py | 2 +- python/lsst/pipe/tasks/multiBand.py | 8 +------- python/lsst/pipe/tasks/objectMasks.py | 2 +- python/lsst/pipe/tasks/repair.py | 2 +- python/lsst/pipe/tasks/scaleVariance.py | 2 +- python/lsst/pipe/tasks/scaleZeroPoint.py | 4 ++-- python/lsst/pipe/tasks/setConfigFromEups.py | 6 +++--- python/lsst/pipe/tasks/setPrimaryFlags.py | 2 +- python/lsst/pipe/tasks/snapCombine.py | 3 ++- python/lsst/pipe/tasks/transformMeasurement.py | 9 +++------ 17 files changed, 30 insertions(+), 34 deletions(-) diff --git a/python/lsst/pipe/tasks/colorterms.py b/python/lsst/pipe/tasks/colorterms.py index 3ee4d9619..cba6e030d 100644 --- a/python/lsst/pipe/tasks/colorterms.py +++ b/python/lsst/pipe/tasks/colorterms.py @@ -158,8 +158,10 @@ class ColortermLibrary(Config): ... }), "sdss": ColortermDict(data={ - 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883), - 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248), + 'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, + c2=-0.00726883), + 'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, + c2=-0.03068248), ... }), }) diff --git a/python/lsst/pipe/tasks/imageDifference.py b/python/lsst/pipe/tasks/imageDifference.py index 71bc8d826..6ae3f9949 100644 --- a/python/lsst/pipe/tasks/imageDifference.py +++ b/python/lsst/pipe/tasks/imageDifference.py @@ -46,8 +46,9 @@ FwhmPerSigma = 2 * math.sqrt(2 * math.log(2)) IqrToSigma = 0.741 -__all__ = ('ImageDifferenceConfig', 'ImageDifferenceTaskRunner', - 'Winter2013ImageDifferenceConfig', 'Winter2013ImageDifferenceTask') +__all__ = ['ImageDifferenceConfig', 'ImageDifferenceTaskRunner', + 'Winter2013ImageDifferenceConfig', 'Winter2013ImageDifferenceTask'] + class ImageDifferenceConfig(pexConfig.Config): """Config for ImageDifferenceTask @@ -872,6 +873,7 @@ def setDefaults(self): ImageDifferenceConfig.setDefaults(self) self.getTemplate.retarget(GetCalexpAsTemplateTask) + class Winter2013ImageDifferenceTask(ImageDifferenceTask): """Image difference Task used in the Winter 2013 data challege. Enables testing the effects of registration shifts and scatter. diff --git a/python/lsst/pipe/tasks/ingest.py b/python/lsst/pipe/tasks/ingest.py index 4eeb35747..5a8d8e347 100644 --- a/python/lsst/pipe/tasks/ingest.py +++ b/python/lsst/pipe/tasks/ingest.py @@ -12,8 +12,8 @@ from lsst.pipe.base import Task, InputOnlyArgumentParser from lsst.afw.fits import DEFAULT_HDU -__all__ = ('IngestArgumentParser', 'ParseConfig', 'ParseConfig', 'ParseTask', 'RegisterConfig', - 'RegistryContext', 'RegisterTask', 'IngestConfig', 'IngestTask') +__all__ = ['IngestArgumentParser', 'ParseConfig', 'ParseConfig', 'ParseTask', 'RegisterConfig', + 'RegistryContext', 'RegisterTask', 'IngestConfig', 'IngestTask'] class IngestArgumentParser(InputOnlyArgumentParser): diff --git a/python/lsst/pipe/tasks/ingestPgsql.py b/python/lsst/pipe/tasks/ingestPgsql.py index 16c737a28..db8870d13 100644 --- a/python/lsst/pipe/tasks/ingestPgsql.py +++ b/python/lsst/pipe/tasks/ingestPgsql.py @@ -10,7 +10,7 @@ except ImportError: havePgSql = False -__all__ = ('PgsqlRegistryContext', 'PgsqlRegisterTask', 'PgsqlIngestConfig', 'PgsqlIngestTask') +__all__ = ['PgsqlRegistryContext', 'PgsqlRegisterTask', 'PgsqlIngestConfig', 'PgsqlIngestTask'] class PgsqlRegistryContext(RegistryContext): diff --git a/python/lsst/pipe/tasks/makeDiscreteSkyMap.py b/python/lsst/pipe/tasks/makeDiscreteSkyMap.py index 4c89f337a..e42eeec0a 100755 --- a/python/lsst/pipe/tasks/makeDiscreteSkyMap.py +++ b/python/lsst/pipe/tasks/makeDiscreteSkyMap.py @@ -30,7 +30,7 @@ from lsst.skymap import DiscreteSkyMap, BaseSkyMap from lsst.pipe.base import ArgumentParser -__all__ = ('MakeDiscreteSkyMapConfig', 'MakeDiscreteSkyMapRunner', 'MakeDiscreteSkyMapTask') +__all__ = ['MakeDiscreteSkyMapConfig', 'MakeDiscreteSkyMapRunner', 'MakeDiscreteSkyMapTask'] class MakeDiscreteSkyMapConfig(pexConfig.Config): diff --git a/python/lsst/pipe/tasks/makeSkyMap.py b/python/lsst/pipe/tasks/makeSkyMap.py index f819e1288..f19c212b3 100755 --- a/python/lsst/pipe/tasks/makeSkyMap.py +++ b/python/lsst/pipe/tasks/makeSkyMap.py @@ -27,7 +27,7 @@ import lsst.pipe.base as pipeBase from lsst.skymap import skyMapRegistry -__all__ = ('MakeSkyMapConfig', 'MakeSkyMapRunner', 'MakeSkyMapTask') +__all__ = ['MakeSkyMapConfig', 'MakeSkyMapRunner', 'MakeSkyMapTask'] class MakeSkyMapConfig(pexConfig.Config): diff --git a/python/lsst/pipe/tasks/matchBackgrounds.py b/python/lsst/pipe/tasks/matchBackgrounds.py index 95d07d2b6..246bb9546 100644 --- a/python/lsst/pipe/tasks/matchBackgrounds.py +++ b/python/lsst/pipe/tasks/matchBackgrounds.py @@ -26,7 +26,7 @@ import lsst.pipe.base as pipeBase import lsstDebug -__all__ = ('MatchBackgroundsConfig', 'MatchBackgroundsTask', 'DataRefMatcher') +__all__ = ['MatchBackgroundsConfig', 'MatchBackgroundsTask', 'DataRefMatcher'] class MatchBackgroundsConfig(pexConfig.Config): diff --git a/python/lsst/pipe/tasks/measurePsf.py b/python/lsst/pipe/tasks/measurePsf.py index 1a9d6fe37..a3958e16b 100644 --- a/python/lsst/pipe/tasks/measurePsf.py +++ b/python/lsst/pipe/tasks/measurePsf.py @@ -26,7 +26,7 @@ import lsst.pex.config as pexConfig import lsst.pipe.base as pipeBase -__all__ = ('MeasurePsfConfig', 'MeasurePsfTask') +__all__ = ['MeasurePsfConfig', 'MeasurePsfTask'] class MeasurePsfConfig(pexConfig.Config): diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 61b23bbc1..8950fba2d 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -32,11 +32,10 @@ the mergeDet, meas, and ref dataset Footprints: - deepCoadd_peak_schema """ -import numpy from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner, - PipelineTask, InitOutputDatasetField, InputDatasetField, OutputDatasetField, + InitOutputDatasetField, InputDatasetField, OutputDatasetField, QuantumConfig) from lsst.pex.config import Config, Field, ConfigurableField from lsst.meas.algorithms import DynamicDetectionTask @@ -59,11 +58,6 @@ from .multiBandUtils import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory # noqa: F401 - - - -############################################################################################################## - class DetectCoaddSourcesConfig(Config): """Configuration parameters for the DetectCoaddSourcesTask """ diff --git a/python/lsst/pipe/tasks/objectMasks.py b/python/lsst/pipe/tasks/objectMasks.py index cb39a8806..1efbc5b79 100644 --- a/python/lsst/pipe/tasks/objectMasks.py +++ b/python/lsst/pipe/tasks/objectMasks.py @@ -4,7 +4,7 @@ import lsst.afw.table as afwTable from lsst.log import Log -__all__ = ('ObjectMaskCatalog',) +__all__ = ['ObjectMaskCatalog', ] class ObjectMaskCatalog: diff --git a/python/lsst/pipe/tasks/repair.py b/python/lsst/pipe/tasks/repair.py index 2305dde1b..4a86d59c8 100644 --- a/python/lsst/pipe/tasks/repair.py +++ b/python/lsst/pipe/tasks/repair.py @@ -28,7 +28,7 @@ from lsst.afw.display import getDisplay from lsst.pipe.tasks.interpImage import InterpImageTask -__all__ = ('RepairConfig', 'RepairTask') +__all__ = ['RepairConfig', 'RepairTask'] class RepairConfig(pexConfig.Config): diff --git a/python/lsst/pipe/tasks/scaleVariance.py b/python/lsst/pipe/tasks/scaleVariance.py index 500e4051b..33052763b 100644 --- a/python/lsst/pipe/tasks/scaleVariance.py +++ b/python/lsst/pipe/tasks/scaleVariance.py @@ -26,7 +26,7 @@ from lsst.pipe.base import Task from lsst.meas.algorithms import SubtractBackgroundTask -__all__ = ('ScaleVarianceConfig', 'ScaleVarianceTask') +__all__ = ['ScaleVarianceConfig', 'ScaleVarianceTask'] class ScaleVarianceConfig(Config): diff --git a/python/lsst/pipe/tasks/scaleZeroPoint.py b/python/lsst/pipe/tasks/scaleZeroPoint.py index ce1a53fd1..6ee82ef02 100644 --- a/python/lsst/pipe/tasks/scaleZeroPoint.py +++ b/python/lsst/pipe/tasks/scaleZeroPoint.py @@ -26,8 +26,8 @@ import lsst.pipe.base as pipeBase from lsst.pipe.tasks.selectImages import BaseSelectImagesTask -__all__ = ("ImageScaler", "SpatialImageScaler", "ScaleZeroPointTask", "ScaleZeroPointConfig", - "SpatialScaleZeroPointConfig", "SpatialScaleZeroPointTask") +__all__ = ["ImageScaler", "SpatialImageScaler", "ScaleZeroPointTask", "ScaleZeroPointConfig", + "SpatialScaleZeroPointConfig", "SpatialScaleZeroPointTask"] class ImageScaler: diff --git a/python/lsst/pipe/tasks/setConfigFromEups.py b/python/lsst/pipe/tasks/setConfigFromEups.py index e8a903ba0..9ad929287 100644 --- a/python/lsst/pipe/tasks/setConfigFromEups.py +++ b/python/lsst/pipe/tasks/setConfigFromEups.py @@ -33,10 +33,10 @@ def setAstrometryConfigFromEups(config, menu): .. code-block:: none - menu = { "ps1": {}, # No changes + menu = { "ps1*": {}, # No changes "ps1-without-y": { "solver.filterMap": {"y": "z"} }, # No y-band in this specific version - "sdss": { "solver.filterMap": {"y": "z"} }, # No y-band, use z instead - "2mass": { "solver.filterMap": {"y": "J"} }, # No y-band, use J instead + "sdss*": { "solver.filterMap": {"y": "z"} }, # No y-band, use z instead + "2mass*": { "solver.filterMap": {"y": "J"} }, # No y-band, use J instead } """ diff --git a/python/lsst/pipe/tasks/setPrimaryFlags.py b/python/lsst/pipe/tasks/setPrimaryFlags.py index 3c4abc3a1..f439273ae 100644 --- a/python/lsst/pipe/tasks/setPrimaryFlags.py +++ b/python/lsst/pipe/tasks/setPrimaryFlags.py @@ -25,7 +25,7 @@ from lsst.pipe.base import Task from lsst.afw.geom import Box2D -__all__ = ('SetPrimaryFlagsConfig', 'SetPrimaryFlagsTask') +__all__ = ['SetPrimaryFlagsConfig', 'SetPrimaryFlagsTask'] class SetPrimaryFlagsConfig(Config): diff --git a/python/lsst/pipe/tasks/snapCombine.py b/python/lsst/pipe/tasks/snapCombine.py index 50b87e7ac..e962aac10 100644 --- a/python/lsst/pipe/tasks/snapCombine.py +++ b/python/lsst/pipe/tasks/snapCombine.py @@ -133,7 +133,8 @@ def validate(self): class SnapCombineTask(pipeBase.Task): - """ + """Combine snaps. + The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a flag -d to import debug.py from your PYTHONPATH; see "http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html" diff --git a/python/lsst/pipe/tasks/transformMeasurement.py b/python/lsst/pipe/tasks/transformMeasurement.py index 45bfd6e10..068cef4e1 100644 --- a/python/lsst/pipe/tasks/transformMeasurement.py +++ b/python/lsst/pipe/tasks/transformMeasurement.py @@ -188,8 +188,7 @@ class RunTransformTaskBase(pipeBase.CmdLineTask): @property def inputSchemaType(self): - """ - The Butler dataset type for the schema of the input source catalog. + """The Butler dataset type for the schema of the input source catalog. By default, we append `_schema` to the input source type. Subclasses may customize if required. @@ -198,8 +197,7 @@ def inputSchemaType(self): @property def outputDataset(self): - """ - The Butler dataset type for the schema of the output catalog. + """The Butler dataset type for the schema of the output catalog. By default, we prepend `transformed` to the input source type. Subclasses may customize if required. @@ -208,8 +206,7 @@ def outputDataset(self): @property def measurementConfig(self): - """ - The configuration of the measurement operation used to generate the input catalog. + """The configuration of the measurement operation used to generate the input catalog. By default we look for `measurement` under the root configuration of the generating task. Subclasses may customize this (e.g. to `calibrate.measurement`) From cb14665ae618177f74377dc53b8b08184c268ffc Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Wed, 19 Dec 2018 16:55:40 -0800 Subject: [PATCH 3/6] Standardize task doc boilerplate --- doc/lsst.pipe.tasks/index.rst | 41 +------------------ ...crMultiBand.DeblendDcrCoaddSourcesTask.rst | 31 +++++++++----- ....tasks.ingestCalibs.CalibsRegisterTask.rst | 2 +- ...pipe.tasks.interpImage.InterpImageTask.rst | 2 +- ...ctImages.BestSeeingWcsSelectImagesTask.rst | 2 +- ...ks.selectImages.PsfWcsSelectImagesTask.rst | 6 +++ 6 files changed, 31 insertions(+), 53 deletions(-) diff --git a/doc/lsst.pipe.tasks/index.rst b/doc/lsst.pipe.tasks/index.rst index 369d6bf1f..76d24b476 100644 --- a/doc/lsst.pipe.tasks/index.rst +++ b/doc/lsst.pipe.tasks/index.rst @@ -71,43 +71,4 @@ Configurations Python API reference ==================== -.. automodapi:: lsst.pipe.tasks.assembleCoadd -.. automodapi:: lsst.pipe.tasks.calibrate -.. automodapi:: lsst.pipe.tasks.characterizeImage -.. automodapi:: lsst.pipe.tasks.coaddBase -.. automodapi:: lsst.pipe.tasks.coaddHelpers -.. automodapi:: lsst.pipe.tasks.coaddInputRecorder -.. automodapi:: lsst.pipe.tasks.colorterms -.. automodapi:: lsst.pipe.tasks.dcrAssembleCoadd -.. automodapi:: lsst.pipe.tasks.dcrMultiBand -.. automodapi:: lsst.pipe.tasks.exampleCmdLineTask -.. automodapi:: lsst.pipe.tasks.exampleStatsTasks -.. automodapi:: lsst.pipe.tasks.fakes -.. automodapi:: lsst.pipe.tasks.getRepositoryData -.. automodapi:: lsst.pipe.tasks.imageDifference -.. automodapi:: lsst.pipe.tasks.ingestCalibs -.. automodapi:: lsst.pipe.tasks.ingestPgsql -.. automodapi:: lsst.pipe.tasks.ingest -.. automodapi:: lsst.pipe.tasks.interpImage -.. automodapi:: lsst.pipe.tasks.makeCoaddTempExp -.. automodapi:: lsst.pipe.tasks.makeDiscreteSkyMap -.. automodapi:: lsst.pipe.tasks.makeSkyMap -.. automodapi:: lsst.pipe.tasks.matchBackgrounds -.. automodapi:: lsst.pipe.tasks.measurePsf -.. automodapi:: lsst.pipe.tasks.multiBand -.. automodapi:: lsst.pipe.tasks.photoCal -.. automodapi:: lsst.pipe.tasks.processCcd -.. automodapi:: lsst.pipe.tasks.propagateVisitFlags -.. automodapi:: lsst.pipe.tasks.registerImage -.. automodapi:: lsst.pipe.tasks.repair -.. automodapi:: lsst.pipe.tasks.repositoryIterator -.. automodapi:: lsst.pipe.tasks.scaleVariance -.. automodapi:: lsst.pipe.tasks.scaleZeroPoint -.. automodapi:: lsst.pipe.tasks.selectImages -.. automodapi:: lsst.pipe.tasks.setConfigFromEups -.. automodapi:: lsst.pipe.tasks.setPrimaryFlags -.. automodapi:: lsst.pipe.tasks.snapCombine -.. automodapi:: lsst.pipe.tasks.transformMeasurement -.. automodapi:: lsst.pipe.tasks.version -.. automodapi:: lsst.pipe.tasks.warpAndPsfMatch -.. automodapi:: lsst.pipe.tasks.objectMasks \ No newline at end of file +.. automodapi:: lsst.pipe.tasks diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask.rst index d8aa62551..87a0f92a3 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask.rst @@ -1,15 +1,26 @@ -.. .. lsst-task-topic:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask +.. lsst-task-topic:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask ########################## DeblendDcrCoaddSourcesTask ########################## -.. Retargetable subtasks -.. ===================== -.. -.. .. lsst-task-config-subtasks:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask -.. -.. Configuration fields -.. ==================== -.. -.. .. lsst-task-config-fields:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask +.. _lsst.pipe.tasks.dcrAssembleCoadd.DeblendDcrCoaddSourcesTask-api: + +Python API summary +================== + +.. lsst-task-api-summary:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask + +.. _lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask-subtasks: + +Retargetable subtasks +===================== + +.. lsst-task-config-subtasks:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask + +.. _lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask-configs: + +Configuration fields +==================== + +.. lsst-task-config-fields:: lsst.pipe.tasks.dcrMultiBand.DeblendDcrCoaddSourcesTask diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.rst index 8ad311b12..e8db3864a 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.rst @@ -18,7 +18,7 @@ Retargetable subtasks .. lsst-task-config-subtasks:: lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask -.. _lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask-fields: +.. _lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask-configs: Configuration fields ==================== diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.interpImage.InterpImageTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.interpImage.InterpImageTask.rst index d0bf2765c..a88bb5e23 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.interpImage.InterpImageTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.interpImage.InterpImageTask.rst @@ -18,7 +18,7 @@ Retargetable subtasks .. lsst-task-config-subtasks:: lsst.pipe.tasks.interpImage.InterpImageTask -.. _lsst.pipe.tasks.interpImage.InterpImageTask-fields: +.. _lsst.pipe.tasks.interpImage.InterpImageTask-configs: Configuration fields ==================== diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask.rst index bf164c89a..6aa45d1da 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask.rst @@ -18,7 +18,7 @@ Retargetable subtasks .. lsst-task-config-subtasks:: lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask -.. _lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask-fields: +.. _lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask-configs: Configuration fields ==================== diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask.rst index 2b8e359d6..33e7e46cb 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask.rst @@ -4,16 +4,22 @@ PsfWcsSelectImagesTask ###################### +.. _lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask-api: + Python API summary ================== .. lsst-task-api-summary:: lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask +.. _lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask-subtasks: + Retargetable subtasks ===================== .. lsst-task-config-subtasks:: lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask +.. _lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask-configs: + Configuration fields ==================== From f544e3f967f12c58bd472c2c483de8586c3c2ce0 Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Wed, 19 Dec 2018 16:56:21 -0800 Subject: [PATCH 4/6] Move assembleCoadd task docs to new home --- ....tasks.assembleCoadd.AssembleCoaddTask.rst | 114 ++++++ ...mbleCoadd.CompareWarpAssembleCoaddTask.rst | 130 ++++++ ...ssembleCoadd.SafeClipAssembleCoaddTask.rst | 118 ++++++ python/lsst/pipe/tasks/assembleCoadd.py | 374 ++---------------- 4 files changed, 384 insertions(+), 352 deletions(-) diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask.rst index 27855c06b..34dd2099e 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask.rst @@ -6,6 +6,28 @@ AssembleCoaddTask Assemble a coadded image from a set of ``CoaddTempExps``. +.. _lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask-summary: + +Processing summary +================== + +We want to assemble a coadded image from a set of Warps (also called +coadded temporary exposures or ``coaddTempExps``). +Each input Warp covers a patch on the sky and corresponds to a single +run/visit/exposure of the covered patch. We provide the task with a list +of Warps (``selectDataList``) from which it selects Warps that cover the +specified patch (pointed at by ``dataRef``). + +Each Warp that goes into a coadd will typically have an independent +photometric zero-point. Therefore, we must scale each Warp to set it to +a common photometric zeropoint. WarpType may be one of 'direct' or +'psfMatched', and the boolean configs ``config.makeDirect`` and +``config.makePsfMatched`` set which of the warp types will be coadded. +The coadd is computed as a mean with optional outlier rejection. +Criteria for outlier rejection are set in ``AssembleCoaddConfig``. +Finally, Warps can have bad 'NaN' pixels which received no input from the +source calExps. We interpolate over these bad (NaN) pixels. + .. _lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask-api: Python API summary @@ -26,3 +48,95 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask + +.. _lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask-examples: + +Examples +======== + +`AssembleCoaddTask` assembles a set of warped images into a coadded image. +The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` +with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two +inputs: a data reference to the tract patch and filter to be coadded, and +a list of Warps to attempt to coadd. These are specified using ``--id`` and +``--selectId``, respectively: + +.. code-block:: none + + --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] + --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] + +Only the Warps that cover the specified tract and patch will be coadded. +A list of the available optional arguments can be obtained by calling +``assembleCoadd.py`` with the ``--help`` command line argument: + +.. code-block:: none + + assembleCoadd.py --help + +To demonstrate usage of the `AssembleCoaddTask` in the larger context of +multi-band processing, we will generate the HSC-I & -R band coadds from +HSC engineering test data provided in the ``ci_hsc`` package. To begin, +assuming that the lsst stack has been already set up, we must set up the +obs_subaru and ``ci_hsc`` packages. This defines the environment variable +``$CI_HSC_DIR`` and points at the location of the package. The raw HSC +data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the +coadds, we must first + +- processCcd +- process the individual ccds in $CI_HSC_RAW to produce calibrated exposures +- makeSkyMap +- create a skymap that covers the area of the sky present in the raw exposures +- makeCoaddTempExp +- warp the individual calibrated exposures to the tangent plane of the coadd + +We can perform all of these steps by running + +.. code-block:: none + + $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 + +This will produce warped exposures for each visit. To coadd the warped +data, we call assembleCoadd.py as follows: + +.. code-block:: none + + assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ + --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ + --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ + --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ + --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ + --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ + --selectId visit=903988 ccd=24 + +that will process the HSC-I band data. The results are written in +``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. + +You may also choose to run: + +.. code-block:: none + + scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 + assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ + --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ + --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ + --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ + --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ + --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ + --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 + +to generate the coadd for the HSC-R band if you are interested in +following multiBand Coadd processing as discussed in `pipeTasks_multiBand` +(but note that normally, one would use the `SafeClipAssembleCoaddTask` +rather than `AssembleCoaddTask` to make the coadd. + +.. _lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask-debug: + +Debugging +========= + +The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a +flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see +`baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has +no debug variables of its own. Some of the subtasks may support debug +variables. See the documentation for the subtasks for further information. diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.rst index 6059d8de7..8eeb1b75e 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.rst @@ -4,6 +4,44 @@ CompareWarpAssembleCoaddTask ############################ +.. _lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask-summary: + +Processing summary +================== + +In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., +we clip outliers). The problem with doing this is that when computing the +coadd PSF at a given location, individual visit PSFs from visits with +outlier pixels contribute to the coadd PSF and cannot be treated correctly. +In this task, we correct for this behavior by creating a new badMaskPlane +'CLIPPED' which marks pixels in the individual warps suspected to contain +an artifact. We populate this plane on the input warps by comparing +PSF-matched warps with a PSF-matched median coadd which serves as a +model of the static sky. Any group of pixels that deviates from the +PSF-matched template coadd by more than config.detect.threshold sigma, +is an artifact candidate. The candidates are then filtered to remove +variable sources and sources that are difficult to subtract such as +bright stars. This filter is configured using the config parameters +``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is +the maximum fraction of epochs that the deviation can appear in and still +be considered an artifact. The spatialThreshold is the maximum fraction of +pixels in the footprint of the deviation that appear in other epochs +(where other epochs is defined by the temporalThreshold). If the deviant +region meets this criteria of having a significant percentage of pixels +that deviate in only a few epochs, these pixels have the 'CLIPPED' bit +set in the mask. These regions will not contribute to the final coadd. +Furthermore, any routine to determine the coadd PSF can now be cognizant +of clipped regions. + +Note that the algorithm implemented by this task is +preliminary and works correctly for HSC data. Parameter modifications and +or considerable redesigning of the algorithm is likely required for other +surveys. + +``CompareWarpAssembleCoaddTask`` sub-classes +``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` +as a subtask to generate the TemplateCoadd (the model of the static sky). + .. _lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask-api: Python API summary @@ -24,3 +62,95 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask + +.. _lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask-examples: + +Examples +======== + +``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a +coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running +``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. +Usage of ``assembleCoadd.py`` expects a data reference to the tract patch +and filter to be coadded (specified using +'--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') +along with a list of coaddTempExps to attempt to coadd (specified using +'--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). +Only the warps that cover the specified tract and patch will be coadded. +A list of the available optional arguments can be obtained by calling +``assembleCoadd.py`` with the ``--help`` command line argument: + +.. code-block:: none + + assembleCoadd.py --help + +To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger +context of multi-band processing, we will generate the HSC-I & -R band +oadds from HSC engineering test data provided in the ``ci_hsc`` package. +To begin, assuming that the lsst stack has been already set up, we must +set up the ``obs_subaru`` and ``ci_hsc`` packages. +This defines the environment variable ``$CI_HSC_DIR`` and points at the +location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` +directory. To begin assembling the coadds, we must first + + - processCcd + process the individual ccds in $CI_HSC_RAW to produce calibrated exposures + - makeSkyMap + create a skymap that covers the area of the sky present in the raw exposures + - makeCoaddTempExp + warp the individual calibrated exposures to the tangent plane of the coadd + +We can perform all of these steps by running + +.. code-block:: none + + $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 + +This will produce warped ``coaddTempExps`` for each visit. To coadd the +warped data, we call ``assembleCoadd.py`` as follows: + +.. code-block:: none + + assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ + --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ + --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ + --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ + --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ + --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ + --selectId visit=903988 ccd=24 + +This will process the HSC-I band data. The results are written in +``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. + +.. _lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask-debug: + +Debugging +========= + +The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a +flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see +``baseDebug`` for more about ``debug.py`` files. + +This task supports the following debug variables: + +- ``saveCountIm`` + If True then save the Epoch Count Image as a fits file in the `figPath` +- ``figPath`` + Path to save the debug fits images and figures + +For example, put something like: + +.. code-block:: python + + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) + if name == "lsst.pipe.tasks.assembleCoadd": + di.saveCountIm = True + di.figPath = "/desired/path/to/debugging/output/images" + return di + lsstDebug.Info = DebugInfo + +into your ``debug.py`` file and run ``assemebleCoadd.py`` with the +``--debug`` flag. Some subtasks may have their own debug variables; +see individual Task documentation. diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.rst index 64d27a57b..7f3d13b21 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.rst @@ -4,6 +4,34 @@ SafeClipAssembleCoaddTask ######################### +.. _lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask-summary: + +Processing summary +================== + +In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., +we clip outliers). The problem with doing this is that when computing the +coadd PSF at a given location, individual visit PSFs from visits with +outlier pixels contribute to the coadd PSF and cannot be treated correctly. +In this task, we correct for this behavior by creating a new +``badMaskPlane`` 'CLIPPED'. We populate this plane on the input +coaddTempExps and the final coadd where + + i. difference imaging suggests that there is an outlier and + ii. this outlier appears on only one or two images. + +Such regions will not contribute to the final coadd. Furthermore, any +routine to determine the coadd PSF can now be cognizant of clipped regions. +Note that the algorithm implemented by this task is preliminary and works +correctly for HSC data. Parameter modifications and or considerable +redesigning of the algorithm is likley required for other surveys. + +``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` +"clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. +You can retarget the ``SourceDetectionTask`` "clipDetection" subtask +if you wish. + + .. _lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask-api: Python API summary @@ -24,3 +52,93 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask + +.. _lsst.pipe.tasks.assembleCoadd.AssembleCoaddTask-examples: + +Examples +======== + +`SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` +images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by +running assembleCoadd.py *without* the flag '--legacyCoadd'. + +Usage of ``assembleCoadd.py`` expects a data reference to the tract patch +and filter to be coadded (specified using +'--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') +along with a list of coaddTempExps to attempt to coadd (specified using +'--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). +Only the coaddTempExps that cover the specified tract and patch will be +coadded. A list of the available optional arguments can be obtained by +calling assembleCoadd.py with the --help command line argument: + +.. code-block:: none + + assembleCoadd.py --help + +To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger +context of multi-band processing, we will generate the HSC-I & -R band +coadds from HSC engineering test data provided in the ci_hsc package. +To begin, assuming that the lsst stack has been already set up, we must +set up the obs_subaru and ci_hsc packages. This defines the environment +variable $CI_HSC_DIR and points at the location of the package. The raw +HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling +the coadds, we must first + +- ``processCcd`` + process the individual ccds in $CI_HSC_RAW to produce calibrated exposures +- ``makeSkyMap`` + create a skymap that covers the area of the sky present in the raw exposures +- ``makeCoaddTempExp`` + warp the individual calibrated exposures to the tangent plane of the coadd + +We can perform all of these steps by running + +.. code-block:: none + + $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 + +This will produce warped coaddTempExps for each visit. To coadd the +warped data, we call ``assembleCoadd.py`` as follows: + +.. code-block:: none + + assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ + --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ + --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ + --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ + --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ + --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ + --selectId visit=903988 ccd=24 + +This will process the HSC-I band data. The results are written in +``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. + +You may also choose to run: + +.. code-block:: none + + scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn + assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ + --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ + --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ + --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ + --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ + --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ + --selectId visit=903346 ccd=12 + +to generate the coadd for the HSC-R band if you are interested in following +multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. + + +.. _lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask-debug: + +Debugging +========= + +The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a +flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; +see `baseDebug` for more about ``debug.py`` files. +`SafeClipAssembleCoaddTask` has no debug variables of its own. +The ``SourceDetectionTask`` "clipDetection" subtasks may support debug +variables. See the documetation for `SourceDetectionTask` "clipDetection" +for further information. diff --git a/python/lsst/pipe/tasks/assembleCoadd.py b/python/lsst/pipe/tasks/assembleCoadd.py index 1a73590c0..4dc7c1d19 100755 --- a/python/lsst/pipe/tasks/assembleCoadd.py +++ b/python/lsst/pipe/tasks/assembleCoadd.py @@ -178,117 +178,6 @@ def validate(self): class AssembleCoaddTask(CoaddBaseTask): """Assemble a coadded image from a set of warps (coadded temporary exposures). - - We want to assemble a coadded image from a set of Warps (also called - coadded temporary exposures or ``coaddTempExps``). - Each input Warp covers a patch on the sky and corresponds to a single - run/visit/exposure of the covered patch. We provide the task with a list - of Warps (``selectDataList``) from which it selects Warps that cover the - specified patch (pointed at by ``dataRef``). - Each Warp that goes into a coadd will typically have an independent - photometric zero-point. Therefore, we must scale each Warp to set it to - a common photometric zeropoint. WarpType may be one of 'direct' or - 'psfMatched', and the boolean configs `config.makeDirect` and - `config.makePsfMatched` set which of the warp types will be coadded. - The coadd is computed as a mean with optional outlier rejection. - Criteria for outlier rejection are set in `AssembleCoaddConfig`. - Finally, Warps can have bad 'NaN' pixels which received no input from the - source calExps. We interpolate over these bad (NaN) pixels. - - `AssembleCoaddTask` uses several sub-tasks. These are - - - `ScaleZeroPointTask` - - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp - - `InterpImageTask` - - interpolate across bad pixels (NaN) in the final coadd - - You can retarget these subtasks if you wish. - - Notes - ----- - The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a - flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see - `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has - no debug variables of its own. Some of the subtasks may support debug - variables. See the documentation for the subtasks for further information. - - Examples - -------- - `AssembleCoaddTask` assembles a set of warped images into a coadded image. - The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` - with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two - inputs: a data reference to the tract patch and filter to be coadded, and - a list of Warps to attempt to coadd. These are specified using ``--id`` and - ``--selectId``, respectively: - - .. code-block:: none - - --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] - --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] - - Only the Warps that cover the specified tract and patch will be coadded. - A list of the available optional arguments can be obtained by calling - ``assembleCoadd.py`` with the ``--help`` command line argument: - - .. code-block:: none - - assembleCoadd.py --help - - To demonstrate usage of the `AssembleCoaddTask` in the larger context of - multi-band processing, we will generate the HSC-I & -R band coadds from - HSC engineering test data provided in the ``ci_hsc`` package. To begin, - assuming that the lsst stack has been already set up, we must set up the - obs_subaru and ``ci_hsc`` packages. This defines the environment variable - ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC - data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the - coadds, we must first - - - processCcd - - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures - - makeSkyMap - - create a skymap that covers the area of the sky present in the raw exposures - - makeCoaddTempExp - - warp the individual calibrated exposures to the tangent plane of the coadd - - We can perform all of these steps by running - - .. code-block:: none - - $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 - - This will produce warped exposures for each visit. To coadd the warped - data, we call assembleCoadd.py as follows: - - .. code-block:: none - - assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ - --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ - --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ - --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ - --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ - --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ - --selectId visit=903988 ccd=24 - - that will process the HSC-I band data. The results are written in - ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. - - You may also choose to run: - - .. code-block:: none - - scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 - assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ - --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ - --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ - --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ - --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ - --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ - --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 - - to generate the coadd for the HSC-R band if you are interested in - following multiBand Coadd processing as discussed in `pipeTasks_multiBand` - (but note that normally, one would use the `SafeClipAssembleCoaddTask` - rather than `AssembleCoaddTask` to make the coadd. """ ConfigClass = AssembleCoaddConfig _DefaultName = "assembleCoadd" @@ -1103,111 +992,6 @@ def validate(self): class SafeClipAssembleCoaddTask(AssembleCoaddTask): """Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas with potential artifacts. - - In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., - we clip outliers). The problem with doing this is that when computing the - coadd PSF at a given location, individual visit PSFs from visits with - outlier pixels contribute to the coadd PSF and cannot be treated correctly. - In this task, we correct for this behavior by creating a new - ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input - coaddTempExps and the final coadd where - - i. difference imaging suggests that there is an outlier and - ii. this outlier appears on only one or two images. - - Such regions will not contribute to the final coadd. Furthermore, any - routine to determine the coadd PSF can now be cognizant of clipped regions. - Note that the algorithm implemented by this task is preliminary and works - correctly for HSC data. Parameter modifications and or considerable - redesigning of the algorithm is likley required for other surveys. - - ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` - "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. - You can retarget the ``SourceDetectionTask`` "clipDetection" subtask - if you wish. - - Notes - ----- - The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a - flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; - see `baseDebug` for more about ``debug.py`` files. - `SafeClipAssembleCoaddTask` has no debug variables of its own. - The ``SourceDetectionTask`` "clipDetection" subtasks may support debug - variables. See the documetation for `SourceDetectionTask` "clipDetection" - for further information. - - Examples - -------- - `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` - images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by - running assembleCoadd.py *without* the flag '--legacyCoadd'. - - Usage of ``assembleCoadd.py`` expects a data reference to the tract patch - and filter to be coadded (specified using - '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') - along with a list of coaddTempExps to attempt to coadd (specified using - '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). - Only the coaddTempExps that cover the specified tract and patch will be - coadded. A list of the available optional arguments can be obtained by - calling assembleCoadd.py with the --help command line argument: - - .. code-block:: none - - assembleCoadd.py --help - - To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger - context of multi-band processing, we will generate the HSC-I & -R band - coadds from HSC engineering test data provided in the ci_hsc package. - To begin, assuming that the lsst stack has been already set up, we must - set up the obs_subaru and ci_hsc packages. This defines the environment - variable $CI_HSC_DIR and points at the location of the package. The raw - HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling - the coadds, we must first - - - ``processCcd`` - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures - - ``makeSkyMap`` - create a skymap that covers the area of the sky present in the raw exposures - - ``makeCoaddTempExp`` - warp the individual calibrated exposures to the tangent plane of the coadd - - We can perform all of these steps by running - - .. code-block:: none - - $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 - - This will produce warped coaddTempExps for each visit. To coadd the - warped data, we call ``assembleCoadd.py`` as follows: - - .. code-block:: none - - assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ - --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ - --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ - --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ - --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ - --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ - --selectId visit=903988 ccd=24 - - This will process the HSC-I band data. The results are written in - ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. - - You may also choose to run: - - .. code-block:: none - - scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn - assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ - --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ - --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ - --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ - --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ - --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ - --selectId visit=903346 ccd=12 - - to generate the coadd for the HSC-R band if you are interested in following - multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. """ ConfigClass = SafeClipAssembleCoaddConfig _DefaultName = "safeClipAssembleCoadd" @@ -1220,25 +1004,6 @@ def __init__(self, *args, **kwargs): def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): """Assemble the coadd for a region. - Compute the difference of coadds created with and without outlier - rejection to identify coadd pixels that have outlier values in some - individual visits. - Detect clipped regions on the difference image and mark these regions - on the one or two individual coaddTempExps where they occur if there - is significant overlap between the clipped region and a source. This - leaves us with a set of footprints from the difference image that have - been identified as having occured on just one or two individual visits. - However, these footprints were generated from a difference image. It - is conceivable for a large diffuse source to have become broken up - into multiple footprints acrosss the coadd difference in this process. - Determine the clipped region from all overlapping footprints from the - detected sources in each visit - these are big footprints. - Combine the small and big clipped footprints and mark them on a new - bad mask plane. - Generate the coadd using `AssembleCoaddTask.run` without outlier - removal. Clipped footprints will no longer make it into the coadd - because they are marked in the new bad mask plane. - Parameters ---------- skyInfo : `lsst.pipe.base.Struct` @@ -1260,6 +1025,28 @@ def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwa Notes ----- + Compute the difference of coadds created with and without outlier + rejection to identify coadd pixels that have outlier values in some + individual visits. + + Detect clipped regions on the difference image and mark these regions + on the one or two individual coaddTempExps where they occur if there + is significant overlap between the clipped region and a source. This + leaves us with a set of footprints from the difference image that have + been identified as having occured on just one or two individual visits. + However, these footprints were generated from a difference image. It + is conceivable for a large diffuse source to have become broken up + into multiple footprints acrosss the coadd difference in this process. + Determine the clipped region from all overlapping footprints from the + detected sources in each visit - these are big footprints. + + Combine the small and big clipped footprints and mark them on a new + bad mask plane. + + Generate the coadd using `AssembleCoaddTask.run` without outlier + removal. Clipped footprints will no longer make it into the coadd + because they are marked in the new bad mask plane. + args and kwargs are passed but ignored in order to match the call signature expected by the parent task. """ @@ -1643,123 +1430,6 @@ def setDefaults(self): class CompareWarpAssembleCoaddTask(AssembleCoaddTask): """Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing PSF-matched warps. - - In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., - we clip outliers). The problem with doing this is that when computing the - coadd PSF at a given location, individual visit PSFs from visits with - outlier pixels contribute to the coadd PSF and cannot be treated correctly. - In this task, we correct for this behavior by creating a new badMaskPlane - 'CLIPPED' which marks pixels in the individual warps suspected to contain - an artifact. We populate this plane on the input warps by comparing - PSF-matched warps with a PSF-matched median coadd which serves as a - model of the static sky. Any group of pixels that deviates from the - PSF-matched template coadd by more than config.detect.threshold sigma, - is an artifact candidate. The candidates are then filtered to remove - variable sources and sources that are difficult to subtract such as - bright stars. This filter is configured using the config parameters - ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is - the maximum fraction of epochs that the deviation can appear in and still - be considered an artifact. The spatialThreshold is the maximum fraction of - pixels in the footprint of the deviation that appear in other epochs - (where other epochs is defined by the temporalThreshold). If the deviant - region meets this criteria of having a significant percentage of pixels - that deviate in only a few epochs, these pixels have the 'CLIPPED' bit - set in the mask. These regions will not contribute to the final coadd. - Furthermore, any routine to determine the coadd PSF can now be cognizant - of clipped regions. Note that the algorithm implemented by this task is - preliminary and works correctly for HSC data. Parameter modifications and - or considerable redesigning of the algorithm is likley required for other - surveys. - - ``CompareWarpAssembleCoaddTask`` sub-classes - ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` - as a subtask to generate the TemplateCoadd (the model of the static sky). - - Notes - ----- - The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a - flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see - ``baseDebug`` for more about ``debug.py`` files. - - This task supports the following debug variables: - - - ``saveCountIm`` - If True then save the Epoch Count Image as a fits file in the `figPath` - - ``figPath`` - Path to save the debug fits images and figures - - For example, put something like: - - .. code-block:: python - - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) - if name == "lsst.pipe.tasks.assembleCoadd": - di.saveCountIm = True - di.figPath = "/desired/path/to/debugging/output/images" - return di - lsstDebug.Info = DebugInfo - - into your ``debug.py`` file and run ``assemebleCoadd.py`` with the - ``--debug`` flag. Some subtasks may have their own debug variables; - see individual Task documentation. - - Examples - -------- - ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a - coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running - ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. - Usage of ``assembleCoadd.py`` expects a data reference to the tract patch - and filter to be coadded (specified using - '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') - along with a list of coaddTempExps to attempt to coadd (specified using - '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). - Only the warps that cover the specified tract and patch will be coadded. - A list of the available optional arguments can be obtained by calling - ``assembleCoadd.py`` with the ``--help`` command line argument: - - .. code-block:: none - - assembleCoadd.py --help - - To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger - context of multi-band processing, we will generate the HSC-I & -R band - oadds from HSC engineering test data provided in the ``ci_hsc`` package. - To begin, assuming that the lsst stack has been already set up, we must - set up the ``obs_subaru`` and ``ci_hsc`` packages. - This defines the environment variable ``$CI_HSC_DIR`` and points at the - location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` - directory. To begin assembling the coadds, we must first - - - processCcd - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures - - makeSkyMap - create a skymap that covers the area of the sky present in the raw exposures - - makeCoaddTempExp - warp the individual calibrated exposures to the tangent plane of the coadd - - We can perform all of these steps by running - - .. code-block:: none - - $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 - - This will produce warped ``coaddTempExps`` for each visit. To coadd the - warped data, we call ``assembleCoadd.py`` as follows: - - .. code-block:: none - - assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ - --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ - --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ - --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ - --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ - --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ - --selectId visit=903988 ccd=24 - - This will process the HSC-I band data. The results are written in - ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. """ ConfigClass = CompareWarpAssembleCoaddConfig _DefaultName = "compareWarpAssembleCoadd" From 070a1613d8261f49cacbb6f45b8a8461cbb2cba9 Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Wed, 19 Dec 2018 16:56:48 -0800 Subject: [PATCH 5/6] Clean up calibrate numpydoc --- ...sst.pipe.tasks.calibrate.CalibrateTask.rst | 76 +++++ python/lsst/pipe/tasks/calibrate.py | 291 ++++++------------ 2 files changed, 167 insertions(+), 200 deletions(-) diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.calibrate.CalibrateTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.calibrate.CalibrateTask.rst index 734a5eb38..f7571d207 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.calibrate.CalibrateTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.calibrate.CalibrateTask.rst @@ -4,6 +4,40 @@ CalibrateTask ############# +.. _lsst.pipe.tasks.calibrate.CalibrateTask-summary: + +Processing summary +================== + +Given an exposure with a good PSF model and aperture correction map +(e.g. as provided by ``CharacterizeImageTask``), perform the following +operations: + +- Run detection and measurement +- Run astrometry subtask to fit an improved WCS +- Run photoCal subtask to fit the exposure's photometric zero-point + +Invoking the Task + +If you want this task to unpersist inputs or persist outputs, then call +the `runDataRef` method (a wrapper around the `run` method). + +If you already have the inputs unpersisted and do not want to persist the +output then it is more direct to call the `run` method: + +Quantities set in exposure Metadata + +Exposure metadata + +.. code-block:: none + + MAGZERO_RMS MAGZERO's RMS == sigma reported by photoCal task + ERO_NOBJ Number of stars used == ngood reported by photoCal + task + COLORTERM1 (always 0.0) + COLORTERM2 (always 0.0) + COLORTERM3 (always 0.0) + .. _lsst.pipe.tasks.calibrate.CalibrateTask-api: Python API summary @@ -24,3 +58,45 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.calibrate.CalibrateTask + +.. _lsst.pipe.tasks.calibrate.CalibrateTask-debug: + +Debugging +========= + +The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task +interface supports a flag +`--debug` to import `debug.py` from your `$PYTHONPATH`; see ``baseDebug`` +for more about `debug.py`. + +CalibrateTask has a debug dictionary containing one key: + +.. code-block:: none + + calibrate + frame (an int; <= 0 to not display) in which to display the exposure, + sources and matches. See ``lsst.meas.astrom.displayAstrometry`` for + the meaning of the various symbols. + +For example, put something like: + +.. code-block:: py + + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would + # call us recursively + if name == "lsst.pipe.tasks.calibrate": + di.display = dict( + calibrate = 1, + ) + + return di + + lsstDebug.Info = DebugInfo + +into your `debug.py` file and run `calibrateTask.py` with the `--debug` +flag. + +Some subtasks may have their own debug variables; see individual Task +documentation. diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 4432f162e..4b3fc9474 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -42,7 +42,8 @@ class CalibrateConfig(pexConfig.Config): - """Config for CalibrateTask""" + """Config for CalibrateTask. + """ doWrite = pexConfig.Field( dtype=bool, default=True, @@ -172,16 +173,9 @@ def setDefaults(self): # aperture correction should already be measured -## \addtogroup LSST_task_documentation -## \{ -## \page CalibrateTask -## \ref CalibrateTask_ "CalibrateTask" -## \copybrief CalibrateTask -## \} - class CalibrateTask(pipeBase.CmdLineTask): """Calibrate an exposure: measure sources and perform astrometric and - photometric calibration + photometric calibration. Parameters ---------- @@ -189,139 +183,32 @@ class CalibrateTask(pipeBase.CmdLineTask): The butler is passed to the refObjLoader constructor in case it is needed. Ignored if the refObjLoader argument provides a loader directly. - astromRefObjLoader : - An instance of LoadReferenceObjectsTasks + astromRefObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` + An instance of LoadReferenceObjectsTask that supplies an external reference catalog for astrometric calibration. May be None if the desired loader can be constructed from the butler argument or all steps requiring a reference catalog are disabled. - photoRefObjLoader : - An instance of LoadReferenceObjectsTasks + photoRefObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` + An instance of LoadReferenceObjectsTask that supplies an external reference catalog for photometric calibration. May be None if the desired loader can be constructed from the butler argument or all steps requiring a reference catalog are disabled. - icSourceSchema : - schema for icSource catalog, or None. + icSourceSchema : `lsst.afw.table.SourceCatalog` + Schema for icSource catalog, or None. Schema values specified in config.icSourceFieldsToCopy will be taken from this schema. If set to None, no values will be propagated from the icSourceCatalog - kwargs : - other keyword arguments for - lsst.pipe.base.CmdLineTask - - Notes - ----- - Given an exposure with a good PSF model and aperture correction map - (e.g. as provided by @ref CharacterizeImageTask), perform the following - operations: - - - Run detection and measurement - - Run astrometry subtask to fit an improved WCS - - Run photoCal subtask to fit the exposure's photometric zero-point - - Invoking the Task - - If you want this task to unpersist inputs or persist outputs, then call - the `runDataRef` method (a wrapper around the `run` method). - - If you already have the inputs unpersisted and do not want to persist the - output then it is more direct to call the `run` method: - - Quantities set in exposure Metadata - - Exposure metadata - - .. code-block:: none - - MAGZERO_RMS MAGZERO's RMS == sigma reported by photoCal task - ERO_NOBJ Number of stars used == ngood reported by photoCal - task - COLORTERM1 (always 0.0) - COLORTERM2 (always 0.0) - COLORTERM3 (always 0.0) - - Debug variables - - The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task - interface supports a flag - `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug - for more about `debug.py`. - - CalibrateTask has a debug dictionary containing one key: - - .. code-block:: none - - calibrate - frame (an int; <= 0 to not display) in which to display the exposure, - sources and matches. See @ref lsst.meas.astrom.displayAstrometry for - the meaning of the various symbols. - - - Examples - -------- - For example, put something like: - - .. code-block:: py - - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would - # call us recursively - if name == "lsst.pipe.tasks.calibrate": - di.display = dict( - calibrate = 1, - ) - - return di - - lsstDebug.Info = DebugInfo - - into your `debug.py` file and run `calibrateTask.py` with the `--debug` - flag. - - Some subtasks may have their own debug variables; see individual Task - documentation. + **kwargs + Other keyword arguments for `lsst.pipe.base.CmdLineTask` """ - - # Example description used to live here, removed 2-20-2017 as per - # https://jira.lsstcorp.org/browse/DM-9520 - ConfigClass = CalibrateConfig _DefaultName = "calibrate" RunnerClass = pipeBase.ButlerInitializedTaskRunner def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, **kwargs): - """Construct a CalibrateTask - - Parameters - ---------- - butler : - The butler is passed to the refObjLoader constructor - in case it is needed. Ignored if the refObjLoader argument - provides a loader directly. - astromRefObjLoader : - An instance of LoadReferenceObjectsTasks - that supplies an external reference catalog for astrometric - calibration. May be None if the desired loader can be constructed - from the butler argument or all steps requiring a reference catalog - are disabled. - photoRefObjLoader : - An instance of LoadReferenceObjectsTasks - that supplies an external reference catalog for photometric - calibration. May be None if the desired loader can be constructed - from the butler argument or all steps requiring a reference catalog - are disabled. - icSourceSchema : - schema for icSource catalog, or None. - Schema values specified in config.icSourceFieldsToCopy will be - taken from this schema. If set to None, no values will be - propagated from the icSourceCatalog - kwargs : - other keyword arguments for - lsst.pipe.base.CmdLineTask - """ pipeBase.CmdLineTask.__init__(self, **kwargs) if icSourceSchema is None and butler is not None: @@ -404,51 +291,58 @@ def __init__(self, butler=None, astromRefObjLoader=None, @pipeBase.timeMethod def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True): - """Calibrate an exposure, optionally unpersisting inputs and - persisting outputs. + """Calibrate an exposure, optionally unpersisting (loading) inputs and + persisting outputs. This is a wrapper around the `run` method that unpersists inputs - (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) + (if `doUnpersist` true) and persists outputs (if `config.doWrite` true). Parameters ---------- - dataRef : - butler data reference corresponding to a science - image - exposure : `lsst.afw.image.ExposureF` - characterized exposure, or None to unpersist existing + dataRef : `lsst.daf.persistence.ButlerDataRef` + Butler data reference corresponding to a science image. + exposure : `lsst.afw.image.Exposure` + Characterized exposure, or None to load existing icExp and icBackground. See `run` method for details of what is read and written. - background : lsst.afw.math.BackgroundList - initial model of background already - subtracted from exposure. May be - None if no background has been subtracted, though that is unusual - for calibration. A refined background model is output. Ignored if - exposure is None. - icSourceCat : - catalog from which to copy the fields specified - by icSourceKeys, or None; - doUnpersist : - unpersist data: - - if True, exposure, background and icSourceCat are read from - dataRef and those three arguments must all be None; - - if False the exposure must be provided; background and - icSourceCat are optional. True is intended for running as a - command-line task, False for running as a subtask + background : `lsst.afw.math.BackgroundList` + Initial model of background already subtracted from exposure. + May be None if no background has been subtracted, though that is + unusual for calibration. A refined background model is output. + Ignored if exposure is None. + icSourceCat : `lsst.afw.table.SourceCatalog` + Catalog from which to copy the fields specified by icSourceKeys, + or None. + doUnpersist : `bool` + If True, exposure, background and icSourceCat are read from + dataRef and those three arguments must all be None. + If False, the exposure must be provided; background and + icSourceCat are optional. + True is intended for running as a command-line task, which + loads existing inputs from disk, and False is intended + for running as a subtask, which requires an exposure. Returns ------- - calRes : - same data as the calibrate method + calRes : `lsst.pipe.base.Struct` + Result struct with components: + - ``exposure`` : background-subtracted exposure with refined WCS and Calib + (an `lsst.afw.image.Exposure`) + - ``background`` : model of background subtracted from exposure + (an `lsst.afw.math.BackgroundList`) + - ``sourceCat`` : catalog of measured sources + (an `lsst.afw.table.SourceCatalog`) + - ``astromMatches`` : list of source/refObj matches from the astrometry solver + (a `list` of `lsst.afw.table.ReferenceMatch`) + - ``matchMeta`` : metadata needed to unpersist matches + (an ``lsst.daf.base.PropertyList``) Raises ------ RuntimeError - doUnpersist true; exposure, background - and icSourceCat must all be None - - RuntimeError - doUnpersist false; exposure must be provided + Raised if doUnpersist true but exposure, background + and icSourceCat are not all None; + also raised if doUnpersist false but exposure is not provided. """ self.log.info("Processing %s" % (dataRef.dataId)) @@ -491,37 +385,33 @@ def run(self, exposure, exposureIdInfo=None, background=None, Parameters ---------- exposure : `lsst.afw.image.ExposureF` - exposure to calibrate (an lsst.afw.image.ExposureF or similar); - in: - - MaskedImage - - Psf - - out: - - MaskedImage has background subtracted - - Wcs is replaced - - Calib zero-point is set - - exposureIdInfo : `lsst.obs.base.ExposureIdInfo` + Exposure to calibrate. + exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional ID info for exposure. If not provided, returned SourceCatalog IDs will not be globally unique. background : `lsst.afw.math.BackgroundList` - background model already subtracted from - exposure. May be None if no - background has been subtracted, though that is unusual for - calibration. A refined background model is output. - icSourceCat : - A SourceCatalog from CharacterizeImageTask - from which we can copy some fields. + Initial model of background already subtracted from exposure. + May be None if no background has been subtracted, though that is + unusual for calibration. A refined background model is output. + Ignored if exposure is None. + icSourceCat : `lsst.afw.table.SourceCatalog` + A SourceCatalog from ``CharacterizeImageTask`` from which we can + copy some fields. Returns ------- - result: `lsst.pipe.base.Struct` - - - ``exposure`` : calibrate science exposure with refined WCS and Calib - - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - - ``sourceCat`` : catalog of measured sources - - ``astromMatches`` : list of source/refObj matches from the astrometry solver - + result : `lsst.pipe.base.Struct` + + - ``exposure`` : background-subtracted exposure with refined WCS and Calib + (an `lsst.afw.image.Exposure`) + - ``background`` : model of background subtracted from exposure + (an ``lsst.afw.math.BackgroundList``) + - ``sourceCat`` : catalog of measured sources + (an `lsst.afw.table.SourceCatalog`) + - ``astromMatches`` : list of source/refObj matches from the astrometry solver + (a `list` of `lsst.afw.table.ReferenceMatch`) + - ``matchMeta`` : metadata needed to unpersist matches + (an ``lsst.daf.base.PropertyList``) """ # detect, deblend and measure sources if exposureIdInfo is None: @@ -650,21 +540,22 @@ def run(self, exposure, exposureIdInfo=None, background=None, def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta): - """Write output data to the output repository + """Write output data to the output repository. Parameters ---------- - dataRef : - butler data reference corresponding to a science + dataRef : `lsst.daf.persistence.ButlerDataRef` + Butler data reference corresponding to a science image. exposure : `lsst.afw.image.ExposureF` - exposure to write + Exposure to write. background :`lsst.afw.math.BackgroundList` - background model for exposure - sourceCat : - catalog of measured sources - astromMatches : - list of source/refObj matches from the - astrometry solver + Background model for exposure. + sourceCat : `lsst.afw.table.SourceCatalog` + Catalog of measured sources. + astromMatches : `list` of `lsst.afw.table.ReferenceMatch` + List of source/refObj matches from the astrometry solver. + matchMeta : `lsst.daf.base.PropertyList` + Metadata needed to unpersist matches. """ dataRef.put(sourceCat, "src") if self.config.doWriteMatches and astromMatches is not None: @@ -692,11 +583,11 @@ def setMetadata(self, exposure, photoRes=None): Parameters ---------- - exposure : `lsst.afw.image.ExposureF` - exposure whose metadata is to be set - photoRes : - results of running photoCal; if None then it was - not run + exposure : `lsst.afw.image.Exposure` + Exposure whose metadata is to be set. + photoRes : `lsst.pipe.base.Struct` + Results of running photoCal, optional. + If None, then photoCal was not run. """ if photoRes is None: return @@ -725,10 +616,10 @@ def copyIcSourceFields(self, icSourceCat, sourceCat): Parameters ---------- - icSourceCat : - catalog from which to copy fields - sourceCat : - catalog to which to copy fields + icSourceCat : `lsst.afw.table.SourceCatalog` + Catalog from which to copy fields. + sourceCat : `lsst.afw.table.SourceCatalog` + Catalog to which to copy fields. Notes ----- @@ -738,7 +629,7 @@ def copyIcSourceFields(self, icSourceCat, sourceCat): """ if self.schemaMapper is None: raise RuntimeError("To copy icSource fields you must specify " - "icSourceSchema nd icSourceKeys when " + "icSourceSchema and icSourceKeys when " "constructing this task") if icSourceCat is None or sourceCat is None: raise RuntimeError("icSourceCat and sourceCat must both be " From 2bbbe96d613158da27d59f403d2746c23b496e3f Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Thu, 20 Dec 2018 18:08:42 -0800 Subject: [PATCH 6/6] Numpydoc cleanup for CharacterizeImage, MeasurePsf, and Repair. --- ...haracterizeImage.CharacterizeImageTask.rst | 71 ++++ ...t.pipe.tasks.measurePsf.MeasurePsfTask.rst | 137 +++++++ .../lsst.pipe.tasks.repairTask.RepairTask.rst | 36 ++ python/lsst/pipe/tasks/characterizeImage.py | 342 +++++++----------- python/lsst/pipe/tasks/measurePsf.py | 141 ++------ python/lsst/pipe/tasks/repair.py | 93 ++--- 6 files changed, 445 insertions(+), 375 deletions(-) diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.characterizeImage.CharacterizeImageTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.characterizeImage.CharacterizeImageTask.rst index a79909129..02f2d7b95 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.characterizeImage.CharacterizeImageTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.characterizeImage.CharacterizeImageTask.rst @@ -4,6 +4,23 @@ CharacterizeImageTask ##################### +.. _lsst.pipe.tasks.characterizeImage.CharacterizeImageTask-summary: + +Processing summary +================== + +Given an exposure (typically, e.g., as output by IsrTask): + (1) Iterate over the following config.psfIteration times, or once if + config.doMeasurePsf is False: + - detect and measure bright sources + - do an initial repair of cosmic rays (no interpolation yet) + - measure and subtract background + - do an initial PSF measurement estimate + (2) Update or set final PSF + (3) Do final cosmic ray repair, including interpolation + (4) Perform final measurement with final PSF, including measuring and + applying aperture correction, if applicable + .. _lsst.pipe.tasks.characterizeImage.CharacterizeImageTask-api: Python API summary @@ -24,3 +41,57 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask + +.. _lsst.pipe.tasks.characterizeImage.CharacterizeImageTask-debug: + +Debugging +========= + +The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a flag +`--debug` to import `debug.py` from your `$PYTHONPATH`; see baseDebug for more about `debug.py`. + +CharacterizeImageTask has a debug dictionary with the following keys + +frame: + - `int`: if specified, the frame of first debug image displayed (defaults to 1) + +repair_iter: + - `bool`; if True display image after each repair in the measure PSF loop + +background_iter: + - `bool`; if True display image after each background subtraction in the measure PSF loop + +measure_iter: + - `bool`; if True display image and sources at the end of each iteration of the measure PSF loop + +See lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. + +psf: + - `bool`; if True display image and sources after PSF is measured; + +this will be identical to the final image displayed by measure_iter if measure_iter is true + +repair: + - `bool`; if True display image and sources after final repair + +measure: + - `bool`; if True display image and sources after final measurement + +For example, put something like: + +.. code-block:: none + + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + if name == "lsst.pipe.tasks.characterizeImage": + di.display = dict( + repair = True, + ) + + return di + + lsstDebug.Info = DebugInfo + +into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. +Some subtasks may have their own debug variables; see individual Task documentation. \ No newline at end of file diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.measurePsf.MeasurePsfTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.measurePsf.MeasurePsfTask.rst index 7d308650a..1acd27b45 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.measurePsf.MeasurePsfTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.measurePsf.MeasurePsfTask.rst @@ -24,3 +24,140 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.measurePsf.MeasurePsfTask + +.. _lsst.pipe.tasks.measurePsf.MeasurePsfTask-examples: + +Examples +======== + +A complete example of using MeasurePsfTask is in measurePsfTask.py in the +examples directory, and can be run as, e.g., + +.. code-block:: bash + + examples/measurePsfTask.py --ds9 + +The example also runs SourceDetectionTask and SourceMeasurementTask. +Import the tasks (there are some other standard imports; read the file to see them all): + +.. code-block:: python + + from lsst.meas.algorithms.detection import SourceDetectionTask + from lsst.meas.base import SingleFrameMeasurementTask + from lsst.pipe.tasks.measurePsf import MeasurePsfTask + +We need to create the tasks before processing any data as the task constructor +can add an extra column to the schema, but first we need an almost-empty +Schema: + +.. code-block:: python + + schema = afwTable.SourceTable.makeMinimalSchema() + +We can now call the constructors for the tasks we need to find and characterize candidate +PSF stars: + +.. code-block:: python + + config = SourceDetectionTask.ConfigClass() + config.reEstimateBackground = False + detectionTask = SourceDetectionTask(config=config, schema=schema) + config = SingleFrameMeasurementTask.ConfigClass() + # Use the minimum set of plugins required. + config.plugins.names.clear() + for plugin in ["base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux", "base_PixelFlags"]: + config.plugins.names.add(plugin) + config.plugins["base_CircularApertureFlux"].radii = [7.0] + config.slots.psfFlux = "base_CircularApertureFlux_7_0" # Use of the PSF flux is hardcoded in secondMomentStarSelector + measureTask = SingleFrameMeasurementTask(schema, config=config) + +Note that we've chosen a minimal set of measurement plugins: we need the +outputs of ``base_SdssCentroid``, ``base_SdssShape``, and ``base_CircularApertureFlux`` +as inputs to the PSF measurement algorithm, while ``base_PixelFlags`` identifies +and flags bad sources (e.g. with pixels too close to the edge) so they can be +removed later. + +Now we can create and configure the task that we're interested in: + +.. code-block:: python + + config = MeasurePsfTask.ConfigClass() + psfDeterminer = config.psfDeterminer.apply() + psfDeterminer.config.sizeCellX = 128 + psfDeterminer.config.sizeCellY = 128 + psfDeterminer.config.spatialOrder = 1 + psfDeterminer.config.nEigenComponents = 3 + measurePsfTask = MeasurePsfTask(config=config, schema=schema) + +We're now ready to process the data (we could loop over multiple exposures/catalogues using the same +task objects). First create the output table: + +.. code-block:: python + + tab = afwTable.SourceTable.make(schema) + +And process the image: + +.. code-block:: python + + sources = detectionTask.run(tab, exposure, sigma=2).sources + measureTask.measure(exposure, sources) + result = measurePsfTask.run(exposure, sources) + +We can then unpack and use the results: + +.. code-block:: python + + psf = result.psf + cellSet = result.cellSet + +.. _lsst.pipe.tasks.measurePsf.MeasurePsfTask-debug: + +Debugging +========= + +The ``lsst.pipe.base.cmdLineTask.CmdLineTask`` command line task interface supports a +flag -d to import debug.py from your PYTHONPATH; see baseDebug for more about debug.py files. + +.. code-block:: none + + display + If True, display debugging plots + displayExposure + display the Exposure + spatialCells + displayPsfCandidates + show mosaic of candidates + showBadCandidates + Include bad candidates + displayPsfMosaic + show mosaic of reconstructed PSF(xy) + displayResiduals + show residuals + normalizeResiduals + Normalise residuals by object amplitude + + +Additionally you can enable any debug outputs that your chosen star selector and psf determiner support. + +To investigate the pipe_tasks_measurePsf_Debug, put something like + +.. code-block :: none + + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + + if name == "lsst.pipe.tasks.measurePsf" : + di.display = True + di.displayExposure = False # display the Exposure + spatialCells + di.displayPsfCandidates = True # show mosaic of candidates + di.displayPsfMosaic = True # show mosaic of reconstructed PSF(xy) + di.displayResiduals = True # show residuals + di.showBadCandidates = True # Include bad candidates + di.normalizeResiduals = False # Normalise residuals by object amplitude + + return di + + lsstDebug.Info = DebugInfo + +into your debug.py file and run measurePsfTask.py with the --debug flag. diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.repairTask.RepairTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.repairTask.RepairTask.rst index 32888f20d..2fa9e81dc 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.repairTask.RepairTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.repairTask.RepairTask.rst @@ -4,6 +4,14 @@ RepairTask ########## +.. _lsst.pipe.tasks.repair.RepairTask-summary: + +Processing summary +================== + +RepairTask repairs known defects in an exposure, and it also identifies and +repairs cosmic rays. + .. _lsst.pipe.tasks.repair.RepairTask-api: Python API summary @@ -24,3 +32,31 @@ Configuration fields ==================== .. lsst-task-config-fields:: lsst.pipe.tasks.repair.RepairTask + +.. _lsst.pipe.tasks.repair.RepairTask-debug: + +Debugging +========= + +The available debug variables in RepairTask are: + +display : A dictionary containing debug point names as keys with frame number as value. Valid keys are: +repair.before : display image before any repair is done +repair.after : display image after cosmic ray and defect correction +displayCR : If True, display the exposure on ds9's frame 1 and overlay bounding boxes around detects CRs. + +To investigate the pipe_tasks_repair_Debug, put something like + +.. code-block :: none + + import lsstDebug + def DebugInfo(name): + di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively + if name == "lsst.pipe.tasks.repair": + di.display = {'repair.before':2, 'repair.after':3} + di.displayCR = True + return di + +lsstDebug.Info = DebugInfo +into your debug.py file and run runRepair.py with the --debug flag. + diff --git a/python/lsst/pipe/tasks/characterizeImage.py b/python/lsst/pipe/tasks/characterizeImage.py index 8f09e3792..41576ccc2 100644 --- a/python/lsst/pipe/tasks/characterizeImage.py +++ b/python/lsst/pipe/tasks/characterizeImage.py @@ -41,7 +41,7 @@ class CharacterizeImageConfig(pexConfig.Config): - """!Config for CharacterizeImageTask""" + """Config for CharacterizeImageTask""" doMeasurePsf = pexConfig.Field( dtype=bool, default=True, @@ -169,131 +169,55 @@ def validate(self): "because flags determined by PSF measurement are used to identify " "sources used to measure aperture correction") -## \addtogroup LSST_task_documentation -## \{ -## \page CharacterizeImageTask -## \ref CharacterizeImageTask_ "CharacterizeImageTask" -## \copybrief CharacterizeImageTask -## \} - class CharacterizeImageTask(pipeBase.CmdLineTask): - """Measure bright sources and use this to estimate background and PSF of an exposure + """Measure bright sources and use this to estimate background and PSF of + an exposure. + + Given an exposure (typically, e.g., as output by IsrTask): + (1) Iterate over the following config.psfIteration times, or once if + config.doMeasurePsf is False: + - detect and measure bright sources + - do an initial repair of cosmic rays (no interpolation yet) + - measure and subtract background + - do an initial PSF measurement estimate + (2) Update or set final PSF + (3) Do final cosmic ray repair, including interpolation + (4) Perform final measurement with final PSF, including measuring and + applying aperture correction, if applicable Parameters ---------- - butler : + butler : `lsst.daf.persistence.Butler`, optional A butler object is passed to the refObjLoader constructor in case - it is needed to load catalogs. May be None if a catalog-based star selector is - not used, if the reference object loader constructor does not require a butler, - or if a reference object loader is passed directly via the refObjLoader argument. - refObjLoader : - An instance of LoadReferenceObjectsTasks that supplies an - external reference catalog to a catalog-based star selector. May be None if a - catalog star selector is not used or the loader can be constructed from the - butler argument. - schema : - initial schema (an lsst.afw.table.SourceTable), or None - kwargs : - other keyword arguments for lsst.pipe.base.CmdLineTask + it is needed to load catalogs. + May be None if a catalog-based star selector is not used, if the + reference object loader constructor does not require a butler, or if a + reference object loader is passed directly via the refObjLoader + argument. + refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`, optional + An instance of LoadReferenceObjectsTask that supplies an + external reference catalog to a catalog-based star selector. + May be None if a catalog star selector is not used or the loader + can be constructed from the butler argument. + schema : `lsst.afw.table.SourceTable`, optional + Initial schema, or None. + **kwargs + Other keyword arguments for `lsst.pipe.base.CmdLineTask` Notes ----- - Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask): - - detect and measure bright sources - - repair cosmic rays - - measure and subtract background - - measure PSF - - If you want this task to unpersist inputs or persist outputs, then call + If you want this task to load inputs or persist outputs, then call the `runDataRef` method (a thin wrapper around the `run` method). - If you already have the inputs unpersisted and do not want to persist the output - then it is more direct to call the `run` method: - - Configuration parameters - - See CharacterizeImageConfig - - Debug variables - - The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a flag - `--debug` to import `debug.py` from your `$PYTHONPATH`; see baseDebug for more about `debug.py`. - - CharacterizeImageTask has a debug dictionary with the following keys - - frame: - - `int`: if specified, the frame of first debug image displayed (defaults to 1) - - repair_iter: - - `bool`; if True display image after each repair in the measure PSF loop - - background_iter: - - `bool`; if True display image after each background subtraction in the measure PSF loop - - measure_iter: - - `bool`; if True display image and sources at the end of each iteration of the measure PSF loop - - See lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. - - psf: - - `bool`; if True display image and sources after PSF is measured; - - this will be identical to the final image displayed by measure_iter if measure_iter is true - - repair: - - `bool`; if True display image and sources after final repair - - measure: - - `bool`; if True display image and sources after final measurement - - Examples - -------- - For example, put something like: - - .. code-block:: none - - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively - if name == "lsst.pipe.tasks.characterizeImage": - di.display = dict( - repair = True, - ) - - return di - - lsstDebug.Info = DebugInfo - - into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. - Some subtasks may have their own debug variables; see individual Task documentation. + If you already have the inputs loaded and do not want to persist the + output, then it is more direct to call the `run` method. """ - - # Example description used to live here, removed 2-20-2017 by MSSG - ConfigClass = CharacterizeImageConfig _DefaultName = "characterizeImage" RunnerClass = pipeBase.ButlerInitializedTaskRunner def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): - """Construct a CharacterizeImageTask - - Parameters - ---------- - butler : - A butler object is passed to the refObjLoader constructor in case - it is needed to load catalogs. May be None if a catalog-based star selector is - not used, if the reference object loader constructor does not require a butler, - or if a reference object loader is passed directly via the refObjLoader argument. - refObjLoader : - An instance of LoadReferenceObjectsTasks that supplies an - external reference catalog to a catalog-based star selector. May be None if a - catalog star selector is not used or the loader can be constructed from the - butler argument. - schema : - initial schema (an lsst.afw.table.SourceTable), or None - kwargs : - other keyword arguments for lsst.pipe.base.CmdLineTask - """ pipeBase.CmdLineTask.__init__(self, **kwargs) if schema is None: schema = SourceTable.makeMinimalSchema() @@ -322,39 +246,51 @@ def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): @pipeBase.timeMethod def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True): - """Characterize a science image and, if wanted, persist the results + """Characterize a science image and optionally persist the results. - This simply unpacks the exposure and passes it to the characterize method to do the work. + This unpacks an exposure and passes it to the characterize method + to do the work. Parameters ---------- - dataRef: butler data reference for science exposure - exposure : - exposure to characterize (an lsst.afw.image.ExposureF or similar). - If None then unpersist from "postISRCCD". - The following changes are made, depending on the config: - - - set psf to the measured PSF - - set apCorrMap to the measured aperture correction - - subtract background - - interpolate over cosmic rays - - update detection and cosmic ray mask planes - - background : `lsst.afw.math.BackgroundList` - initial model of background already subtracted from exposure - (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, - which is typical for image characterization. + dataRef : `lsst.daf.persistence.ButlerDataRef` + Butler data reference for science exposure + exposure : `lsst.afw.image.Exposure`, optional + Exposure to characterize. + If None, then use the butler to get the "postISRCCD" dataset type. + background : `lsst.afw.math.BackgroundList`, optional + Initial model of background already subtracted from exposure + (an lsst.afw.math.BackgroundList). May be None if no background has + been subtracted, which is typical for image characterization. A refined background model is output. - doUnpersist : - if True the exposure is read from the repository - and the exposure and background arguments must be None; - if False the exposure must be provided. - True is intended for running as a command-line task, False for running as a subtask + doUnpersist : `bool` + If True, the exposure is read from the repository + and the exposure and background arguments must be None. + If False, the exposure must be provided. + True is intended for running as a command-line task, False for + running as a subtask. Returns ------- - charRes : - same data as the characterize method + charRes : `lsst.pipe.base.Struct` + Result struct from the final iteration of + detectMeasureAndEstimatePsf with components: + - ``exposure`` : characterized exposure; image is repaired by + interpolating over cosmic rays, mask is updated accordingly, + and the PSF model is set (an `lsst.afw.image.Exposure`) + - ``sourceCat`` : catalog of detected sources + (an `lsst.afw.table.SourceCatalog`) + - ``background`` : model of background subtracted from exposure + (an `lsst.afw.math.BackgroundList`) + - ``psfCellSet`` : spatial cells of PSF candidates + (an `lsst.afw.math.SpatialCellSet`) + + Raises + ------ + RuntimeError + Raised if doUnpersist True but exposure and background + are not both None; also raised if doUnpersist False but + exposure is not provided. """ self._frame = self._initialFrame # reset debug display frame self.log.info("Processing %s" % (dataRef.dataId)) @@ -384,43 +320,34 @@ def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True): @pipeBase.timeMethod def run(self, exposure, exposureIdInfo=None, background=None): - """Characterize a science image - - Peforms the following operations: - - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: - - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) - - interpolate over cosmic rays - - perform final measurement + """Characterize a science image. Parameters ---------- - exposure : - exposure to characterize (an lsst.afw.image.ExposureF or similar). - The following changes are made: - - update or set psf - - set apCorrMap - - update detection and cosmic ray mask planes - - subtract background and interpolate over cosmic rays - - exposureIdInfo : - ID info for exposure (an lsst.obs.base.ExposureIdInfo). - If not provided, returned SourceCatalog IDs will not be globally unique. - background : `lsst.afw.math.BackgroundList` - initial model of background already subtracted from exposure - (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, - which is typical for image characterization. + exposure : `lsst.afw.image.Exposure` + Exposure to characterize. + exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional + ID info for exposure. If not provided, returned SourceCatalog IDs + will not be globally unique. + background : `lsst.afw.math.BackgroundList`, optional + initial model of background already subtracted from exposure. + May be None if no background has been subtracted, which is typical + for image characterization. Returns ------- - dmeRes : `struct` - pipe_base Struct containing these fields, all from the final iteration - of detectMeasureAndEstimatePsf: - - ``exposure`` : characterized exposure; image is repaired by interpolating over cosmic rays, - mask is updated accordingly, and the PSF model is set - - ``sourceCat`` : detected sources (an lsst.afw.table.SourceCatalog) - - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - - ``psfCellSet`` : spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) - + dmeRes : `lsst.pipe.base.Struct` + Result struct from the final iteration of + detectMeasureAndEstimatePsf with components: + - ``exposure`` : characterized exposure; image is repaired by + interpolating over cosmic rays, mask is updated accordingly, + and the PSF model is set (an `lsst.afw.image.Exposure`) + - ``sourceCat`` : catalog of detected sources + (an `lsst.afw.table.SourceCatalog`) + - ``background`` : model of background subtracted from exposure + (an `lsst.afw.math.BackgroundList`) + - ``psfCellSet`` : spatial cells of PSF candidates + (an `lsst.afw.math.SpatialCellSet`) """ self._frame = self._initialFrame # reset debug display frame @@ -455,8 +382,8 @@ def run(self, exposure, exposureIdInfo=None, background=None): self.repair.run(exposure=dmeRes.exposure) self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) - # perform final measurement with final PSF, including measuring and applying aperture correction, - # if wanted + # perform final measurement with final PSF, including measuring and + # applying aperture correction, if wanted self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, exposureId=exposureIdInfo.expId) if self.config.doApCorr: @@ -471,53 +398,59 @@ def run(self, exposure, exposureIdInfo=None, background=None): @pipeBase.timeMethod def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): - """Perform one iteration of detect, measure and estimate PSF - - Performs the following operations: + """Perform one iteration of detect, measure, and estimate PSF. - - if config.doMeasurePsf or not exposure.hasPsf(): - - install a simple PSF model (replacing the existing one, if need be) - - interpolate over cosmic rays with keepCRs=True - - estimate background and subtract it from the exposure - - detect, deblend and measure sources, and subtract a refined background model; - - if config.doMeasurePsf: - - - measure PSF - - exposure : - exposure to characterize (an lsst.afw.image.ExposureF or similar) - The following changes are made: + The following changes are made to the input exposure: - update or set psf - update detection and cosmic ray mask planes - subtract background + Parameters + ---------- + exposure : `lsst.afw.image.Exposure` + Exposure to characterize. exposureIdInfo : `lsst.afw.math.BackgroundList` - ID info for exposure (an lsst.obs_base.ExposureIdInfo) + ID info for exposure. background : `lsst.afw.math.BackgroundList` - initial model of background already subtracted from exposure - (an lsst.afw.math.BackgroundList). + Initial model of background already subtracted from exposure. Returns ------- - pipe_base_struct : `struct` - ` pipe_base Struct containing these fields, all from the final iteration - of detect sources, measure sources and estimate PSF: - - - ``exposure`` : characterized exposure; image is repaired by interpolating over cosmic rays, - - mask is updated accordingly, and the PSF model is set: - - ``sourceCat`` : detected sources (an lsst.afw.table.SourceCatalog) - - ``background`` : model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - - ``psfCellSet`` : spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) + result : `lsst.pipe.base.Struct` + Result struct (from the final iteration of detect sources, measure + sources, and estimate PSF) with components: + - ``exposure`` : characterized exposure; image is repaired by + interpolating over cosmic rays, mask is updated accordingly, + and the PSF model is set (an `lsst.afw.image.Exposure`) + - ``sourceCat`` : catalog of detected sources + (an `lsst.afw.table.SourceCatalog`) + - ``background`` : model of background subtracted from exposure + (an `lsst.afw.math.BackgroundList`) + - ``psfCellSet`` : spatial cells of PSF candidates + (an `lsst.afw.math.SpatialCellSet`) + + Notes + ----- + ``detectMeasureAndEstimatePsf`` performs the following operations: + + if the exposure doesn't have a PSF and/or the user wants to make and + use a simple one: + - install a simple PSF model (replacing the existing one, if relevant) + - interpolate over cosmic rays with keepCRs=True + - estimate background and subtract it from the exposure + - detect, deblend and measure sources, and subtract a refined + background model + if ``config.doMeasurePsf``: + - measure PSF """ # install a simple PSF model, if needed or wanted if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): self.log.warn("Source catalog detected and measured with placeholder or default PSF") self.installSimplePsf.run(exposure=exposure) - # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) + # run repair, but do not interpolate over cosmic rays + # (do that later in `run` with the final PSF model) self.repair.run(exposure=exposure, keepCRs=True) self.display("repair_iter", exposure=exposure) @@ -557,20 +490,25 @@ def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): ) def getSchemaCatalogs(self): - """Return a dict of empty catalogs for each catalog dataset produced by this task. + """Return a dict of empty catalogs for each catalog dataset produced + by this task. """ sourceCat = SourceCatalog(self.schema) sourceCat.getTable().setMetadata(self.algMetadata) return {"icSrc": sourceCat} def display(self, itemName, exposure, sourceCat=None): - """Display exposure and sources on next frame, if display of itemName has been requested + """Display exposure and sources on next frame, if display of itemName + has been requested. Parameters ---------- - itemName : name of item in debugInfo - exposure : exposure to display - sourceCat : source catalog to display + itemName : `str` + Name of item in debugInfo. + exposure : `lsst.afw.image.Exposure` + Exposure to display. + sourceCat : `lsst.afw.table.SourceCatalog`, optional + Source catalog to display. """ val = getDebugFrame(self._display, itemName) if not val: diff --git a/python/lsst/pipe/tasks/measurePsf.py b/python/lsst/pipe/tasks/measurePsf.py index a3958e16b..d95ede688 100644 --- a/python/lsst/pipe/tasks/measurePsf.py +++ b/python/lsst/pipe/tasks/measurePsf.py @@ -47,102 +47,34 @@ class MeasurePsfConfig(pexConfig.Config): doc="Reserve sources from fitting" ) -## @addtogroup LSST_task_documentation -## @{ -## @page MeasurePsfTask -## @ref MeasurePsfTask_ "MeasurePsfTask" -## @copybrief MeasurePsfTask -## @} - class MeasurePsfTask(pipeBase.Task): - """A task that selects stars from a catalog of sources and uses those to measure the PSF. + """Select stars from a catalog of sources and use them to measure the PSF. Parameters ---------- - schema : `lsst.sfw.table.Schema` - An lsst::afw::table::Schema used to create the output lsst.afw.table.SourceCatalog - kwargs : - Keyword arguments passed to lsst.pipe.base.task.Task.__init__. + schema : `lsst.afw.table.Schema`, optional + Schema used to create the output source catalog. + **kwargs + Other keyword arguments for `lsst.pipe.base.Task`. Notes ----- - If schema is not None, 'calib_psf_candidate' and 'calib_psf_used' fields will be added to - identify which stars were employed in the PSF estimation. + If schema is not None, 'calib_psf_candidate' and 'calib_psf_used' fields + will be added to identify which stars were employed in the PSF estimation. - This task can add fields to the schema, so any code calling this task must ensure that - these fields are indeed present in the input table. + This task can add fields to the schema, so any code calling this task must + ensure that these fields are indeed present in the input table. The star selector is a subclass of - ``lsst.meas.algorithms.starSelector.BaseStarSelectorTask`` "lsst.meas.algorithms.BaseStarSelectorTask" + ``lsst.meas.algorithms.starSelector.BaseStarSelectorTask`` and the PSF determiner is a sublcass of - ``lsst.meas.algorithms.psfDeterminer.BasePsfDeterminerTask`` "lsst.meas.algorithms.BasePsfDeterminerTask" - - There is no establised set of configuration parameters for these algorithms, so once you start modifying - parameters (as we do in @ref pipe_tasks_measurePsf_Example) your code is no longer portable. - - @section pipe_tasks_measurePsf_Debug Debug variables - - The ``lsst.pipe.base.cmdLineTask.CmdLineTask`` command line task interface supports a - flag -d to import debug.py from your PYTHONPATH; see baseDebug for more about debug.py files. - - .. code-block:: none - - display - If True, display debugging plots - displayExposure - display the Exposure + spatialCells - displayPsfCandidates - show mosaic of candidates - showBadCandidates - Include bad candidates - displayPsfMosaic - show mosaic of reconstructed PSF(xy) - displayResiduals - show residuals - normalizeResiduals - Normalise residuals by object amplitude - - - Additionally you can enable any debug outputs that your chosen star selector and psf determiner support. - - A complete example of using MeasurePsfTask - - This code is in ``measurePsfTask.py`` in the examples directory, and can be run as e.g. - - .. code-block:: none - - examples/measurePsfTask.py --ds9 - - The example also runs SourceDetectionTask and SourceMeasurementTask; - see ``meas_algorithms_measurement_Example`` for more explanation. + ``lsst.meas.algorithms.psfDeterminer.BasePsfDeterminerTask``. - Import the tasks (there are some other standard imports; read the file to see them all): - - - To investigate the @ref pipe_tasks_measurePsf_Debug, put something like - - .. code-block :: none - - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively - - if name == "lsst.pipe.tasks.measurePsf" : - di.display = True - di.displayExposure = False # display the Exposure + spatialCells - di.displayPsfCandidates = True # show mosaic of candidates - di.displayPsfMosaic = True # show mosaic of reconstructed PSF(xy) - di.displayResiduals = True # show residuals - di.showBadCandidates = True # Include bad candidates - di.normalizeResiduals = False # Normalise residuals by object amplitude - - return di - - lsstDebug.Info = DebugInfo - - into your debug.py file and run measurePsfTask.py with the --debug flag. - """ + There is no establised set of configuration parameters for these + algorithms, so once you start modifying parameters, your code is no + longer portable. + """ ConfigClass = MeasurePsfConfig _DefaultName = "measurePsf" @@ -170,33 +102,34 @@ def __init__(self, schema=None, **kwargs): @pipeBase.timeMethod def run(self, exposure, sources, expId=0, matches=None): - """Measure the PSF + """Measure the PSF. Parameters ---------- - exposure : + exposure : `lsst.afw.image.Exposure` Exposure to process; measured PSF will be added. - sources : + sources : `lsst.afw.table.SourceCatalog` Measured sources on exposure; flag fields will be set marking - stars chosen by the star selector and the PSF determiner if a schema - was passed to the task constructor. - expId : + stars chosen by the star selector and the PSF determiner if a + schema was passed to the task constructor. + expId : `int` Exposure id used for generating random seed. - matches : - A list of lsst.afw.table.ReferenceMatch objects - (i.e. of lsst.afw.table.Match - with first being of type lsst.afw.table.SimpleRecord and @c second - type lsst.afw.table.SourceRecord --- the reference object and detected - object respectively) as returned by e.g. the AstrometryTask. + matches : `list` of `lsst.afw.table.ReferenceMatch` + A list of objects as returned by, e.g., ``AstrometryTask``. + (The list may be of type ``lsst.afw.table.Match`` + with the first member being of type ``lsst.afw.table.SimpleRecord`` + and the second being of type ``lsst.afw.table.SourceRecord`` for + the reference object and detected object, respectively.) Used by star selectors that choose to refer to an external catalog. Returns ------- - result : `pipe.base.Struct` - a pipe.base.Struct with fields: - - ``psf`` : The measured PSF (also set in the input exposure) - - ``cellSet`` : an lsst.afw.math.SpatialCellSet containing the PSF candidates - as returned by the psf determiner. + result : `lsst.pipe.base.Struct` + Result struct with components: + - ``psf`` : an ``lsst.afw.detection.Psf``, containing the measured + PSF (also set in the input exposure). + - ``cellSet`` : an ``lsst.afw.math.SpatialCellSet`` containing the + PSF candidates as returned by the psf determiner. """ self.log.info("Measuring PSF") @@ -269,7 +202,13 @@ def run(self, exposure, sources, expId=0, matches=None): @property def usesMatches(self): - """Return True if this task makes use of the "matches" argument to the run method""" + """Does this task makes use of the "matches" argument? + + Returns + ------- + result : `bool` + True if this task uses the "matches" argument to the run method. + """ return self.starSelector.usesMatches # diff --git a/python/lsst/pipe/tasks/repair.py b/python/lsst/pipe/tasks/repair.py index 4a86d59c8..562c775bf 100644 --- a/python/lsst/pipe/tasks/repair.py +++ b/python/lsst/pipe/tasks/repair.py @@ -56,52 +56,13 @@ def setDefaults(self): self.interp.fallbackValueType = "MEANCLIP" self.interp.negativeFallbackAllowed = True -## @addtogroup LSST_task_documentation -## @{ -## @page RepairTask -## @ref RepairTask_ "RepairTask" -## @copybrief RepairTask -## @} - class RepairTask(pipeBase.Task): - """This task operates on an lsst.afw.image.Exposure in place to interpolate over a set of - lsst.meas.algorithms.Defect objects. - It will also, optionally, find and interpolate any cosmic rays in the lsst.afw.image.Exposure. - - Notes - ----- - The available debug variables in RepairTask are: - - display : - A dictionary containing debug point names as keys with frame number as value. Valid keys are: - repair.before : - display image before any repair is done - repair.after : - display image after cosmic ray and defect correction - displayCR : - If True, display the exposure on ds9's frame 1 and overlay bounding boxes around detects CRs. - - Examples - -------- - To investigate the pipe_tasks_repair_Debug, put something like - - .. code-block :: none - - import lsstDebug - def DebugInfo(name): - di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively - if name == "lsst.pipe.tasks.repair": - di.display = {'repair.before':2, 'repair.after':3} - di.displayCR = True - return di - - lsstDebug.Info = DebugInfo - into your debug.py file and run runRepair.py with the --debug flag. - - - Conversion notes: - Display code should be updated once we settle on a standard way of controlling what is displayed. + """Repair image defects and find and repair cosmic rays in an exposure. + + This task operates on an exposure in place to interpolate over a list of + lsst.meas.algorithms.Defect objects. It will also, optionally, find and + interpolate any cosmic rays. """ ConfigClass = RepairConfig _DefaultName = "repair" @@ -113,34 +74,18 @@ def __init__(self, **kwargs): @pipeBase.timeMethod def run(self, exposure, defects=None, keepCRs=None): - """Repair an Exposure's defects and cosmic rays + """Repair an exposure's defects and cosmic rays. Parameters ---------- exposure : `lsst.afw.image.Exposure` - Exposure must have a valid Psf. - Modified in place. - defects :`lsst.meas.algorithms.DefectListT` - If None, do no - defect correction. - keepCRs : - don't interpolate over the CR pixels (defer to RepairConfig if None) - - Raises - ------ - AssertionError - with the following strings: - - Notes - ----- - - .. code-block:: none - - No exposure provided - The object provided as exposure evaluates to False - No PSF provided - The Exposure has no associated Psf - + Exposure with a valid Psf, which will be modified in place. + defects : `list` of `lsst.meas.algorithms.Defect`, optional + If None, do no defect correction. + keepCRs : `bool`, optional + If True, don't interpolate over the CR pixels (keep them). + If False, interpolate over the CR pixels (don't keep them). + If None, defer to the setting in RepairConfig.cosmicray. """ assert exposure, "No exposure provided" psf = exposure.getPsf() @@ -161,12 +106,16 @@ def run(self, exposure, defects=None, keepCRs=None): getDisplay(frame).mtv(exposure) def cosmicRay(self, exposure, keepCRs=None): - """Mask cosmic rays + """Mask cosmic rays. - exposure : + Parameters + ---------- + exposure : `lsst.afw.image.Exposure` Exposure to process - keepCRs : - Don't interpolate over the CR pixels (defer to pex_config if None) + keepCRs : `bool`, optional + If True, don't interpolate over the CR pixels (keep them). + If False, interpolate over the CR pixels (don't keep them). + If None, defer to the setting in RepairConfig.cosmicray. """ import lsstDebug display = lsstDebug.Info(__name__).display