Skip to content

Commit 3514787

Browse files
natelustnatelust
committed
Create framework for runing ci_hsc tests with gen3
1 parent 2c61df0 commit 3514787

30 files changed

+3538
-0
lines changed

‎.gitignore

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
pytest_session.txt
2+
.cache
3+
.coverage
4+
.sconf_temp
5+
.sconsign.dblite
6+
config.log
7+
tests/.tests
8+
tests/.cache
9+
ups/*.cfgc
10+
_build.*
11+
bin/*.py
12+
*.yaml
13+
brightObjectMasks
14+
ps1_pv3_3pi_20170110
15+
raw
16+
CALIB
17+
shared
18+
gen3.sqlite3

‎.travis.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
os: linux
2+
language: python
3+
matrix:
4+
include:
5+
- env: TEST=flake8
6+
python: '3.6'
7+
install:
8+
- pip install flake8
9+
script:
10+
- flake8
11+
- flake8 --extend-ignore=F821 SConstruct */SConscript
12+
- env: TEST=shellcheck
13+
services:
14+
- docker
15+
script: |
16+
set -e
17+
shopt -s globstar nullglob
18+
CHECK=( **/*.sh )
19+
[[ ${#CHECK[@]} -eq 0 ]] && exit
20+
docker run -v $(pwd):$(pwd) -w $(pwd) \
21+
koalaman/shellcheck-alpine:v0.4.6 -x "${CHECK[@]}"
22+
notifications:
23+
email: false

‎COPYRIGHT

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Copyright 2019 Association of Universities for Research in Astronomy

‎LICENSE

+674
Large diffs are not rendered by default.

‎README.rst

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
==========
2+
``ci_hsc_gen3``
3+
==========
4+
5+
``gen2_ci_hsc`` provides scripts which use the LSST stack to perform single frame and coadd processing based
6+
on engineering test data from Hyper Suprime-Cam.
7+
8+
Obtaining test data
9+
===================
10+
11+
The data used by ``gen2_ci_hsc`` is linked against data installed by ``testdata_ci_hsc``, please
12+
setup that package before running scons on this one.
13+
14+
Running the tests
15+
=================
16+
17+
Set up the package
18+
------------------
19+
20+
The package must be set up in the usual way before running::
21+
22+
$ cd ci_hsc_gen3
23+
$ setup -j -r .
24+
25+
Running the tests
26+
-----------------
27+
28+
Execute ``scons``. On a Mac running OSX 10.11 or greater, you must specify a
29+
Python interpreter followed by a full path to ``scons``::
30+
31+
$ python $(which scons)
32+
33+
On other systems, simply running ``scons`` should be sufficient.
34+
35+
Cleaning up
36+
-----------
37+
After each run of this test, and before the next the repository should be cleaned as follows (with appropriate
38+
modifications for MacOs::
39+
$ scons --clean

‎SConstruct

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import os
2+
from SCons.Script import SConscript
3+
from lsst.sconsUtils.utils import libraryLoaderEnvironment
4+
from lsst.utils import getPackageDir
5+
SConscript(os.path.join(".", "bin.src", "SConscript"))
6+
7+
env = Environment(ENV=os.environ)
8+
env["ENV"]["OMP_NUM_THREADS"] = "1" # Disable threading
9+
10+
location = getPackageDir("ci_hsc_gen3")
11+
12+
13+
def getExecutable(package, script, directory=None):
14+
"""
15+
Given the name of a package and a script or other executable which lies
16+
within the given subdirectory (defaults to "bin"), return an appropriate
17+
string which can be used to set up an appropriate environment and execute
18+
the command.
19+
This includes:
20+
* Specifying an explict list of paths to be searched by the dynamic linker;
21+
* Specifying a Python executable to be run (we assume the one on the
22+
default ${PATH} is appropriate);
23+
* Specifying the complete path to the script.
24+
"""
25+
if directory is None:
26+
directory = "bin"
27+
return "{} python {}".format(libraryLoaderEnvironment(),
28+
os.path.join(getPackageDir(package), directory, script))
29+
30+
31+
# Create butler
32+
butler = env.Command(["butler.yaml", "gen3.sqlite3"], "bin",
33+
["{} .".format(getExecutable("daf_butler", "makeButlerRepo.py"))])
34+
35+
# Use name butler to just run making a butler
36+
env.Alias("butler", butler)
37+
38+
# Run the linker
39+
links = env.Command(["CALIB", "raw", "brightObjectMasks", "ps1_pv3_3pi_20170110"], butler, ["bin/linker.sh"])
40+
41+
register = env.Command("register", links,
42+
["{} butler.yaml --nocalibs".format(getExecutable("ci_hsc_gen3", "gen3.py"))])
43+
44+
sql = env.Command("sql", register, ["bin/dbImport.sh"])
45+
46+
hsc = env.Command("shared/ci_hsc", sql, ["{} butler.yaml".format(getExecutable("ci_hsc_gen3", "gen3.py"))])
47+
48+
skymap = env.Command("skymap", hsc,
49+
["{} -C configs/skymap.py butler.yaml shared/ci_hsc"
50+
.format(getExecutable("pipe_tasks", "makeGen3Skymap.py"))])
51+
52+
externalData = env.Command("external", skymap,
53+
["{} butler.yaml".format(getExecutable("ci_hsc_gen3", "ingestExternalData.py"))])
54+
55+
raws = env.Command("raws", externalData,
56+
["{0} butler.yaml {1}/raw -C {1}/configs/ingestRaws.py"
57+
.format(getExecutable("ci_hsc_gen3", "ingestRaws.py"), location)])
58+
59+
# Use name ingest to run everything up to but not including running the
60+
# pipeline
61+
env.Alias("ingest", raws)
62+
63+
num_process = GetOption('num_jobs')
64+
65+
pipeline = env.Command("shared/ci_hsc_output", raws, ["bin/pipeline.sh {}".format(num_process)])
66+
67+
everything = [butler, links, register, sql, hsc, skymap, externalData, raws, pipeline]
68+
69+
env.Alias("all", everything)
70+
Default(everything)
71+
72+
env.Clean(everything, [y for x in everything for y in x])

‎bin.src/SConscript

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from lsst.sconsUtils import scripts
2+
scripts.BasicSConscript.shebang()

‎bin.src/gen3.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env python
2+
3+
import argparse
4+
import logging
5+
6+
import lsst.log
7+
from lsst.log import Log
8+
9+
from lsst.daf.butler import Butler
10+
from lsst.obs.subaru.gen3.hsc import HyperSuprimeCam
11+
12+
if __name__ == "__main__":
13+
parser = argparse.ArgumentParser(description="Creates various table linkages and registers data"
14+
" types in the registry")
15+
parser.add_argument("root", help="Path to butler to use")
16+
parser.add_argument("-v", "--verbose", action="store_const", dest="logLevel",
17+
default=Log.INFO, const=Log.DEBUG,
18+
help="Set the log level to DEBUG.")
19+
parser.add_argument("--nocalibs", action="store_true", help='Dont write calibs, only register datatypes')
20+
21+
args = parser.parse_args()
22+
log = Log.getLogger("lsst.daf.butler")
23+
log.setLevel(args.logLevel)
24+
25+
# Forward python logging to lsst logger
26+
lgr = logging.getLogger("lsst.daf.butler")
27+
lgr.setLevel(logging.INFO if args.logLevel == Log.INFO else logging.DEBUG)
28+
lgr.addHandler(lsst.log.LogHandler())
29+
30+
butler = Butler(args.root, run="shared/ci_hsc")
31+
if not args.nocalibs:
32+
HyperSuprimeCam().writeCuratedCalibrations(butler)

‎bin.src/ingestExternalData.py

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env python
2+
3+
import argparse
4+
import logging
5+
6+
from collections import namedtuple
7+
8+
import lsst.log
9+
from lsst.log import Log
10+
11+
from lsst.daf.butler import Butler, DatasetType
12+
13+
14+
ExternalProducts = namedtuple("ExternalProducts",
15+
["datatype", "dimensions", "storageClass", "dataId", "path"])
16+
17+
18+
def ingestExternalData(butler, products):
19+
"""Adds a list of products to a registry and ingests them into a
20+
Datastore.
21+
22+
butler: `lsst.daf.butler.Butler`
23+
Butler which to operate on
24+
products: `ExternalProducts`
25+
Named tuple describing datasets to add to the registry and datastore
26+
"""
27+
28+
for entry in products:
29+
dsType = DatasetType(entry.datatype, entry.dimensions, entry.storageClass)
30+
butler.registry.registerDatasetType(dsType)
31+
dsRef = butler.registry.addDataset(dsType, entry.dataId, butler.run)
32+
butler.datastore.ingest(entry.path, dsRef)
33+
34+
35+
if __name__ == "__main__":
36+
parser = argparse.ArgumentParser(description="Ingests externally created products into the butler"
37+
" registry")
38+
parser.add_argument("root", help="Path to butler to use")
39+
parser.add_argument("-v", "--verbose", action="store_const", dest="logLevel",
40+
default=Log.INFO, const=Log.DEBUG,
41+
help="Set the log level to DEBUG.")
42+
43+
args = parser.parse_args()
44+
log = Log.getLogger("lsst.daf.butler")
45+
log.setLevel(args.logLevel)
46+
47+
# Forward python logging to lsst logger
48+
lgr = logging.getLogger("lsst.daf.butler")
49+
lgr.setLevel(logging.INFO if args.logLevel == Log.INFO else logging.DEBUG)
50+
lgr.addHandler(lsst.log.LogHandler())
51+
52+
butler = Butler(args.root, run="shared/ci_hsc")
53+
54+
products = [ExternalProducts("brightObjectMask",
55+
("tract", "patch", "skymap", "abstract_filter"),
56+
"ObjectMaskCatalog",
57+
{"tract": 0,
58+
"patch": 69,
59+
"skymap": "ci_hsc",
60+
"abstract_filter": "i"},
61+
"brightObjectMasks/0/BrightObjectMask-0-5,4-HSC-I.reg"),
62+
ExternalProducts("brightObjectMask",
63+
("tract", "patch", "skymap", "abstract_filter"),
64+
"ObjectMaskCatalog",
65+
{"tract": 0,
66+
"patch": 69,
67+
"skymap": "ci_hsc",
68+
"abstract_filter": "r"},
69+
"brightObjectMasks/0/BrightObjectMask-0-5,4-HSC-I.reg"),
70+
ExternalProducts("ref_cat",
71+
("skypix",),
72+
"SimpleCatalog",
73+
{"skypix": 189584},
74+
"ps1_pv3_3pi_20170110/189584.fits"),
75+
ExternalProducts("ref_cat",
76+
("skypix",),
77+
"SimpleCatalog",
78+
{"skypix": 189648},
79+
"ps1_pv3_3pi_20170110/189648.fits"),
80+
]
81+
ingestExternalData(butler, products)

‎bin.src/ingestRaws.py

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/usr/bin/env python
2+
3+
import argparse
4+
import logging
5+
import os
6+
7+
import lsst.log
8+
from lsst.log import Log
9+
10+
from lsst.daf.butler import Butler
11+
from lsst.obs.base.gen3 import RawIngestTask, RawIngestConfig
12+
13+
import lsst.obs.subaru.gen3.hsc.instrument
14+
15+
if __name__ == "__main__":
16+
parser = argparse.ArgumentParser(description="Ingests raw frames into the butler registry")
17+
parser.add_argument("root", help="Path to butler to use")
18+
parser.add_argument("-v", "--verbose", action="store_const", dest="logLevel",
19+
default=Log.INFO, const=Log.DEBUG,
20+
help="Set the log level to DEBUG.")
21+
parser.add_argument("-C", "--config-file", help="Path to config file overload for RawIngestTask",
22+
default=None, dest="configFile")
23+
parser.add_argument("dir", help="Path to directory containing raws to ingest")
24+
25+
args = parser.parse_args()
26+
log = Log.getLogger("lsst.daf.butler")
27+
log.setLevel(args.logLevel)
28+
29+
# Forward python logging to lsst logger
30+
lgr = logging.getLogger("lsst.daf.butler")
31+
lgr.setLevel(logging.INFO if args.logLevel == Log.INFO else logging.DEBUG)
32+
lgr.addHandler(lsst.log.LogHandler())
33+
34+
butler = Butler(args.root, run="shared/ci_hsc")
35+
36+
config = RawIngestConfig()
37+
if args.configFile is not None:
38+
config.load(args.configFile)
39+
ingester = RawIngestTask(config=config, butler=butler)
40+
41+
files = [os.path.join(args.dir, f) for f in os.listdir(args.dir)
42+
if f.endswith("fits") or f.endswith("FITS")]
43+
ingester.run(files)

‎bin/dbImport.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/usr/bin/env sh
2+
sqlite3 gen3.sqlite3 < resources/calibs.sql

‎bin/linker.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env sh
2+
ln -s "$TESTDATA_CI_HSC_DIR"/CALIB .
3+
ln -s "$TESTDATA_CI_HSC_DIR"/raw .
4+
ln -s "$TESTDATA_CI_HSC_DIR"/brightObjectMasks .
5+
ln -s "$TESTDATA_CI_HSC_DIR"/ps1_pv3_3pi_20170110 .

‎bin/pipeline.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env sh
2+
COLLECTION=shared/ci_hsc_output
3+
4+
pipetask -d "patch.patch = 69" -j "$1" -b "$CI_HSC_GEN3_DIR"/butler.yaml -p lsst.meas.base -p lsst.ip.isr -p \
5+
lsst.pipe.tasks -i calib,shared/ci_hsc -o "$COLLECTION" run \
6+
--register-dataset-types \
7+
-t isrTask.IsrTask:isr -C isr:"$CI_HSC_GEN3_DIR"/configs/isr.py \
8+
-t characterizeImage.CharacterizeImageTask:cit -C cit:"$CI_HSC_GEN3_DIR"/configs/charImage.py \
9+
-t calibrate.CalibrateTask:ct -C ct:"$CI_HSC_GEN3_DIR"/configs/calibrate.py \
10+
-t makeCoaddTempExp.MakeWarpTask:mwt -C mwt:"$CI_HSC_GEN3_DIR"/configs/makeWarp.py \
11+
-t assembleCoadd.CompareWarpAssembleCoaddTask:cwact -C cwact:"$CI_HSC_GEN3_DIR"/configs/compareWarpAssembleCoadd.py \
12+
-t multiBand.DetectCoaddSourcesTask:dcst -C dcst:"$CI_HSC_GEN3_DIR"/configs/detectCoaddSources.py \
13+
-t mergeDetections.MergeDetectionsTask:mdt -C mdt:"$CI_HSC_GEN3_DIR"/configs/mergeCoaddDetections.py \
14+
-t deblendCoaddSourcesPipeline.DeblendCoaddSourcesSingleTask:dcsst -C dcsst:"$CI_HSC_GEN3_DIR"/configs/deblendCoaddSourcesSingle.py \
15+
-t multiBand.MeasureMergedCoaddSourcesTask:mmcst -C mmcst:"$CI_HSC_GEN3_DIR"/configs/measureCoaddSources.py \
16+
-t mergeMeasurements.MergeMeasurementsTask:mmt -C mmt:"$CI_HSC_GEN3_DIR"/configs/mergeCoaddMeasurements.py \
17+
-t forcedPhotCcd.ForcedPhotCcdTask:fpccdt -C fpccdt:"$CI_HSC_GEN3_DIR"/configs/forcedPhotCcd.py \
18+
-t forcedPhotCoadd.ForcedPhotCoaddTask:fpct -C fpct:"$CI_HSC_GEN3_DIR"/configs/forcedPhotCoadd.py \

‎configs/calibrate.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
from lsst.utils import getPackageDir
4+
5+
subaruConfig = os.path.join(getPackageDir("obs_subaru"), "config", "calibrate.py")
6+
if os.path.exists(subaruConfig):
7+
config.load(subaruConfig)
8+
hscConfig = os.path.join(getPackageDir("obs_subaru"), "config", "hsc", "calibrate.py")
9+
if os.path.exists(hscConfig):
10+
config.load(hscConfig)

‎configs/charImage.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
from lsst.utils import getPackageDir
4+
5+
subaruConfig = os.path.join(getPackageDir("obs_subaru"), "config", "charImage.py")
6+
if os.path.exists(subaruConfig):
7+
config.load(subaruConfig)
8+
hscConfig = os.path.join(getPackageDir("obs_subaru"), "config", "hsc", "charImage.py")
9+
if os.path.exists(hscConfig):
10+
config.load(hscConfig)

‎configs/compareWarpAssembleCoadd.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
from lsst.utils import getPackageDir
4+
5+
subaruConfig = os.path.join(getPackageDir("obs_subaru"), "config", "compareWarpAssembleCoadd.py")
6+
if os.path.exists(subaruConfig):
7+
config.load(subaruConfig)
8+
hscConfig = os.path.join(getPackageDir("obs_subaru"), "config", "hsc", "compareWarpAssembleCoadd.py")
9+
if os.path.exists(hscConfig):
10+
config.load(hscConfig)

‎configs/deblendCoaddSourcesSingle.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
from lsst.utils import getPackageDir
4+
5+
subaruConfig = os.path.join(getPackageDir("obs_subaru"), "config", "singleBandDeblend.py")
6+
if os.path.exists(subaruConfig):
7+
config.singleBandDeblend.load(subaruConfig)
8+
hscConfig = os.path.join(getPackageDir("obs_subaru"), "config", "hsc", "singleBandDeblend.py")
9+
if os.path.exists(hscConfig):
10+
config.singleBandDeblend.load(hscConfig)

0 commit comments

Comments
 (0)