Coverage for tests/test_pipelines.py: 99%
153 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-08-29 10:10 +0000
1# This file is part of drp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for tracking edits to drp_pipe pipelines.
23"""
25import os
26import tempfile
27import unittest
28from typing import Any
30from lsst.daf.butler import Butler, Config
31from lsst.daf.butler.tests import DatastoreMock
32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester
35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")
36TEST_DIR = os.path.abspath(os.path.dirname(__file__))
38# mostly common inputs
39COMMON_INPUTS = {
40 "bfKernel",
41 "bias",
42 "camera",
43 "crosstalk",
44 "dark",
45 "flat",
46 "fringe",
47 "isrOverscanCorrected",
48 "raw",
49 "skyMap",
50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)
51 "ptc",
52 "linearizer",
53 "bfk",
54 "pretrainedModelPackage",
55 # Optional preloaded solar system objects.
56 # Generated by generateEphemerides in base-v2 but not required in general.
57 "preloaded_ss_object_visit",
58 # Solar system state required to generateEphemerides
59 "mpcorb",
60 "de440s",
61 "sb441_n16",
62 "obscodes",
63 "linux_p1550p2650",
64 "pck00010",
65 "earth_latest_high_prec",
66 "earth_620120_250826",
67 "earth_2025_250826_2125_predict",
68 "naif0012"
69}
71# HSC common inputs, in addition to COMMON_INPUTS
72HSC_INPUTS = {
73 "brightObjectMask",
74 "brighterFatterKernel",
75 "defects",
76 "gaia_dr2_20200414",
77 "gaia_dr3_20230707",
78 "linearizer",
79 "ps1_pv3_3pi_20170110",
80 "sky",
81 "transmission_atmosphere",
82 "transmission_filter",
83 "transmission_optics",
84 "transmission_sensor",
85 "yBackground",
86}
88# LATISS common inputs, in addition to COMMON_INPUTS
89LATISS_INPUTS = {
90 "atlas_refcat2_20220201",
91 "defects",
92 "gaia_dr3_20230707",
93 "sky",
94 "transmission_atmosphere_fgcm",
95 "gain_correction",
96}
98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS
99LSSTCAM_IMSIM_INPUTS = {
100 "bfk",
101 "cal_ref_cat_2_2",
102 "truth_summary",
103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"
104}
106# LSSTCam common inputs, in addition to COMMON_INPUTS
107LSSTCAM_INPUTS = {
108 "bfk",
109 "electroBfDistortionMatrix",
110 "linearizer",
111 "defects",
112 "cti",
113 "the_monster_20250219",
114 "gaia_dr3_20230707",
115 "sky",
116 "astrometry_camera",
117 "gain_correction",
118}
120# LSSTComCam common inputs, in addition to COMMON_INPUTS
121LSSTCOMCAM_INPUTS = {
122 "bfk",
123 "ptc",
124 "linearizer",
125 "defects",
126 "cti",
127 "the_monster_20250219",
128 "illuminationCorrection",
129 "gain_correction",
130}
132# LSSTComCamSim common inputs, in addition to COMMON_INPUTS
133LSSTCOMCAMSIM_INPUTS = {
134 "uw_stars_20240524",
135 "ptc",
136 "bfk",
137 "gain_correction",
138}
140# a selection of mostly common outputs
141COMMON_OUTPUTS = {
142 "ccdVisitTable",
143 "compare_warp_artifact_mask",
144 "deepCoaddVisits",
145 "deepCoadd",
146 "deepCoadd_calexp",
147 "deepCoadd_calexp_background",
148 "deepCoadd_det",
149 "deepCoadd_directWarp",
150 "deepCoadd_directWarp_maskedFraction",
151 "deepCoadd_forced_src",
152 "deepCoadd_inputMap",
153 "deepCoadd_meas",
154 "deepCoadd_mergeDet",
155 "deepCoadd_nImage",
156 "deepCoadd_obj",
157 "deepCoadd_psfMatchedWarp",
158 "deepCoadd_ref",
159 "deepCoadd_scarletModelData",
160 "finalized_src_table",
161 "finalVisitSummary",
162 "initial_astrometry_match_detector",
163 "initial_photometry_match_detector",
164 "initial_psf_stars_footprints_detector",
165 "initial_psf_stars_detector",
166 "calexp",
167 "calexpBackground",
168 "src",
169 "objectTable",
170 "objectTable_tract",
171 "postISRCCD",
172 "sourceTable_visit",
173 "visitSummary",
174 "visitTable",
175 "isolated_star_presource_associations",
176 "isolated_star_presources",
177}
179# Translated COMMON_OUTPUTS
180COMMON_V2_OUTPUTS = {
181 'visit_detector_table',
182 'compare_warp_artifact_mask',
183 'deep_coadd_visit_selection',
184 'deep_coadd_predetection',
185 'deep_coadd',
186 'deep_coadd_background',
187 'object_detection',
188 'direct_warp',
189 'direct_warp_masked_fraction',
190 'object_forced_measurement',
191 'deep_coadd_input_map',
192 'object_unforced_measurement',
193 'object_detection_merged',
194 'deep_coadd_n_image',
195 'object_unstandardized',
196 'psf_matched_warp',
197 'object_ref_measurement',
198 'object_scarlet_models',
199 'refit_psf_star',
200 'visit_summary',
201 'initial_astrometry_match_detector',
202 'initial_photometry_match_detector',
203 'single_visit_psf_star_footprints',
204 'single_visit_psf_star',
205 'preliminary_visit_image',
206 'preliminary_visit_image_background',
207 'single_visit_star_footprints',
208 'object_patch',
209 'object_all',
210 'post_isr_image',
211 'recalibrated_star',
212 'preliminary_visit_summary',
213 'visit_table',
214 'isolated_star_association',
215 'isolated_star'
216}
218# HSC common outputs, in addition to COMMON_OUTPUTS
219HSC_OUTPUTS = {
220 "calexp_skyCorr_visit_mosaic",
221 "calexpBackground_skyCorr_visit_mosaic",
222 "preSourceTable",
223 "preSourceTable_visit",
224 "skyCorr",
225}
227# LATISS common outputs, in addition to COMMON_OUTPUTS
228LATISS_OUTPUTS = {
229 "diaObjectTable_tract",
230 "diaSourceTable",
231 "diaSourceTable_tract",
232 "fgcm_Cycle0_FitParameters",
233 "fgcm_Cycle1_FitParameters",
234 "fgcm_Cycle2_FitParameters",
235 "fgcm_Cycle3_FitParameters",
236 "fgcm_Cycle4_FitParameters",
237 "fgcm_Cycle0_FlaggedStars",
238 "fgcm_Cycle1_FlaggedStars",
239 "fgcm_Cycle2_FlaggedStars",
240 "fgcm_Cycle3_FlaggedStars",
241 "fgcm_Cycle4_FlaggedStars",
242 "fgcm_reference_stars",
243 "fgcm_star_ids",
244 "fgcm_star_observations",
245 "fgcm_standard_star",
246 "forcedSourceOnDiaObjectTable",
247 "forcedSourceOnDiaObjectTable_tract",
248 "forcedSourceTable",
249 "forcedSourceTable_tract",
250 "goodSeeingCoadd",
251 "goodSeeingCoadd_nImage",
252 "goodSeeingDiff_assocDiaSrcTable",
253 "goodSeeingDiff_diaObjTable",
254 "goodSeeingDiff_diaSrc",
255 "goodSeeingDiff_diaSrcTable",
256 "goodSeeingDiff_differenceExp",
257 "goodSeeingDiff_differenceTempExp",
258 "goodSeeingDiff_fullDiaObjTable",
259 "goodSeeingDiff_matchedExp",
260 "goodSeeingDiff_templateExp",
261 "goodSeeingVisits",
262 "mergedForcedSource",
263 "mergedForcedSourceOnDiaObject",
264 "transmission_atmosphere_fgcm",
265}
267# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS
268LSSTCAM_IMSIM_OUTPUTS = {
269 "diaObjectTable_tract",
270 "diaSourceTable",
271 "diaSourceTable_tract",
272 "forcedSourceOnDiaObjectTable",
273 "forcedSourceOnDiaObjectTable_tract",
274 "forcedSourceTable",
275 "forcedSourceTable_tract",
276 "goodSeeingCoadd",
277 "goodSeeingCoadd_nImage",
278 "goodSeeingDiff_assocDiaSrcTable",
279 "goodSeeingDiff_diaObjTable",
280 "goodSeeingDiff_diaSrc",
281 "goodSeeingDiff_diaSrcTable",
282 "goodSeeingDiff_differenceExp",
283 "goodSeeingDiff_differenceTempExp",
284 "goodSeeingDiff_fullDiaObjTable",
285 "goodSeeingDiff_matchedExp",
286 "goodSeeingDiff_templateExp",
287 "goodSeeingVisits",
288 "mergedForcedSource",
289 "mergedForcedSourceOnDiaObject",
290}
292# v2 outputs common to all "quickLook" pipelines, which only include
293QUICKLOOK_V2_OUTPUTS = {
294 "preliminary_visit_image", "preliminary_visit_summary"
295}
297# Outputs common to all "quickLook" pipelines, which only include
298QUICKLOOK_OUTPUTS = {
299 "calexp", "visitSummary"
300}
302QUICKLOOK_BIAS_OUTPUTS = {
303 "post_isr_image",
304}
306QUICKLOOK_DARK_OUTPUTS = {
307 "post_isr_image",
308}
310QUICKLOOK_FLAT_OUTPUTS = {
311 "post_isr_image",
312}
314# All refcats used by any pipelines, for inclusion as the initial_dataset_types
315# argument to PipelineStepTester; the intent is just to allow the 'skypix'
316# dimension placeholder used in these connections to be resolved.
317REFCATS = [
318 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),
319 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),
320 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),
321 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),
322 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),
323 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),
324 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),
325]
328class PipelineTestCase(unittest.TestCase):
329 def setUp(self):
330 self.root = makeTestTempDir(TEST_DIR)
331 self.maxDiff = None
333 def tearDown(self):
334 removeTestTempDir(self.root)
336 def makeButler(self, **kwargs: Any) -> Butler:
337 """Return new Butler instance on each call."""
338 config = Config()
340 # make separate temporary directory for registry of this instance
341 tmpdir = tempfile.mkdtemp(dir=self.root)
342 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
343 config = Butler.makeRepo(self.root, config)
344 butler = Butler.from_config(config, **kwargs)
345 self.enterContext(butler)
346 DatastoreMock.apply(butler)
347 return butler
349 def test_decam_isrForCrosstalkSources(self):
350 butler = self.makeButler(writeable=True)
351 tester = PipelineStepTester(
352 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),
353 ["#step0"],
354 initial_dataset_types=REFCATS,
355 expected_inputs={
356 "camera",
357 "raw",
358 },
359 expected_outputs={"overscanRaw"},
360 )
361 tester.run(butler, self)
363 def test_decam_drp_merian(self):
364 butler = self.makeButler(writeable=True)
365 tester = PipelineStepTester(
366 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
367 [
368 "#step1",
369 "#step2a",
370 "#step2b",
371 "#step2d",
372 "#step2e",
373 "#step3",
374 ],
375 initial_dataset_types=REFCATS,
376 expected_inputs=COMMON_INPUTS
377 | {
378 "defects",
379 "gaia_dr2_20200414",
380 "gaia_dr3_20230707",
381 "linearizer",
382 "overscanRaw",
383 "ps1_pv3_3pi_20170110",
384 "sky",
385 },
386 expected_outputs=COMMON_OUTPUTS
387 | {
388 "goodSeeingCoadd",
389 "goodSeeingCoadd_nImage",
390 "goodSeeingVisits",
391 "preSourceTable",
392 "preSourceTable_visit",
393 },
394 )
395 tester.run(butler, self)
397 def test_hsc_drp_ci_hsc(self):
398 butler = self.makeButler(writeable=True)
399 tester = PipelineStepTester(
400 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),
401 [""],
402 initial_dataset_types=REFCATS,
403 expected_inputs=COMMON_INPUTS
404 | HSC_INPUTS
405 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},
406 expected_outputs=COMMON_OUTPUTS
407 | HSC_OUTPUTS
408 | {
409 "diaObjectTable_tract",
410 "diaSourceTable",
411 "diaSourceTable_tract",
412 "forcedSourceOnDiaObjectTable",
413 "forcedSourceOnDiaObjectTable_tract",
414 "forcedSourceTable",
415 "forcedSourceTable_tract",
416 "goodSeeingCoadd",
417 "goodSeeingCoadd_nImage",
418 "goodSeeingDiff_assocDiaSrcTable",
419 "goodSeeingDiff_diaObjTable",
420 "goodSeeingDiff_diaSrc",
421 "goodSeeingDiff_diaSrcTable",
422 "goodSeeingDiff_differenceTempExp",
423 "goodSeeingDiff_fullDiaObjTable",
424 "goodSeeingDiff_matchedExp",
425 "goodSeeingDiff_templateExp",
426 "goodSeeingVisits",
427 "mergedForcedSource",
428 "mergedForcedSourceOnDiaObject",
429 },
430 )
431 tester.run(butler, self)
433 def test_hsc_drp_prod(self):
434 butler = self.makeButler(writeable=True)
435 tester = PipelineStepTester(
436 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),
437 [
438 "#step1",
439 "#step2a",
440 "#step2b",
441 "#step2c",
442 "#step2d",
443 "#step2e",
444 "#step3",
445 "#step4",
446 "#step7",
447 ],
448 initial_dataset_types=REFCATS,
449 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
450 expected_outputs=COMMON_OUTPUTS
451 | HSC_OUTPUTS
452 | {
453 "fgcm_Cycle6_AtmosphereParameters",
454 "fgcm_Cycle0_FitParameters",
455 "fgcm_Cycle1_FitParameters",
456 "fgcm_Cycle2_FitParameters",
457 "fgcm_Cycle3_FitParameters",
458 "fgcm_Cycle4_FitParameters",
459 "fgcm_Cycle5_FitParameters",
460 "fgcm_Cycle6_FitParameters",
461 "fgcm_Cycle0_FlaggedStars",
462 "fgcm_Cycle1_FlaggedStars",
463 "fgcm_Cycle2_FlaggedStars",
464 "fgcm_Cycle3_FlaggedStars",
465 "fgcm_Cycle4_FlaggedStars",
466 "fgcm_Cycle5_FlaggedStars",
467 "fgcm_Cycle6_FlaggedStars",
468 "fgcm_reference_stars",
469 "fgcm_Cycle6_StandardStars",
470 "fgcm_star_ids",
471 "fgcm_star_observations",
472 "fgcm_Cycle6_Zeropoints",
473 "transmission_atmosphere_fgcm",
474 "fgcm_standard_star",
475 },
476 )
477 tester.run(butler, self)
479 def test_hsc_drp_rc2(self):
480 butler = self.makeButler(writeable=True)
481 tester = PipelineStepTester(
482 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),
483 [
484 "#step1",
485 "#step2a",
486 "#step2b",
487 "#step2cde",
488 "#step3",
489 "#step4",
490 "#step5",
491 "#step6",
492 "#step7",
493 ],
494 initial_dataset_types=REFCATS,
495 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
496 expected_outputs=COMMON_OUTPUTS
497 | HSC_OUTPUTS
498 | {
499 "diaObjectTable_tract",
500 "diaSourceTable",
501 "diaSourceTable_tract",
502 "fgcm_Cycle4_AtmosphereParameters",
503 "fgcm_Cycle0_FitParameters",
504 "fgcm_Cycle1_FitParameters",
505 "fgcm_Cycle2_FitParameters",
506 "fgcm_Cycle3_FitParameters",
507 "fgcm_Cycle4_FitParameters",
508 "fgcm_Cycle0_FlaggedStars",
509 "fgcm_Cycle1_FlaggedStars",
510 "fgcm_Cycle2_FlaggedStars",
511 "fgcm_Cycle3_FlaggedStars",
512 "fgcm_Cycle4_FlaggedStars",
513 "fgcm_reference_stars",
514 "fgcm_Cycle4_StandardStars",
515 "fgcm_star_ids",
516 "fgcm_star_observations",
517 "fgcm_Cycle4_Zeropoints",
518 "fgcm_standard_star",
519 "forcedSourceOnDiaObjectTable",
520 "forcedSourceOnDiaObjectTable_tract",
521 "forcedSourceTable",
522 "forcedSourceTable_tract",
523 "goodSeeingCoadd",
524 "goodSeeingCoadd_nImage",
525 "goodSeeingDiff_assocDiaSrcTable",
526 "goodSeeingDiff_diaObjTable",
527 "goodSeeingDiff_diaSrc",
528 "goodSeeingDiff_diaSrcTable",
529 "goodSeeingDiff_differenceTempExp",
530 "goodSeeingDiff_fullDiaObjTable",
531 "goodSeeingDiff_matchedExp",
532 "goodSeeingDiff_templateExp",
533 "goodSeeingVisits",
534 "mergedForcedSource",
535 "mergedForcedSourceOnDiaObject",
536 "transmission_atmosphere_fgcm",
537 },
538 )
539 tester.run(butler, self)
541 def test_hsc_drp_rc2_subset(self):
542 butler = self.makeButler(writeable=True)
543 tester = PipelineStepTester(
544 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),
545 [
546 "#nightlyStep1",
547 "#nightlyStep2a",
548 "#nightlyStep2b",
549 "#nightlyStep2c",
550 "#nightlyStep2d",
551 "#nightlyStep3",
552 "#nightlyStep4",
553 ],
554 initial_dataset_types=REFCATS,
555 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
556 expected_outputs=COMMON_OUTPUTS
557 | HSC_OUTPUTS
558 | {
559 "fgcm_Cycle4_AtmosphereParameters",
560 "fgcm_Cycle0_FitParameters",
561 "fgcm_Cycle1_FitParameters",
562 "fgcm_Cycle2_FitParameters",
563 "fgcm_Cycle3_FitParameters",
564 "fgcm_Cycle4_FitParameters",
565 "fgcm_Cycle0_FlaggedStars",
566 "fgcm_Cycle1_FlaggedStars",
567 "fgcm_Cycle2_FlaggedStars",
568 "fgcm_Cycle3_FlaggedStars",
569 "fgcm_Cycle4_FlaggedStars",
570 "fgcm_reference_stars",
571 "fgcm_Cycle4_StandardStars",
572 "fgcm_star_ids",
573 "fgcm_star_observations",
574 "fgcm_Cycle4_Zeropoints",
575 "transmission_atmosphere_fgcm",
576 "fgcm_standard_star",
577 },
578 )
579 tester.run(butler, self)
581 def test_latiss_drp(self):
582 butler = self.makeButler(writeable=True)
583 tester = PipelineStepTester(
584 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),
585 [
586 "#step1",
587 "#step2a",
588 "#step2bcde",
589 "#step3a",
590 "#step3b",
591 "#step3c",
592 "#step4",
593 "#step5",
594 "#step6",
595 ],
596 initial_dataset_types=REFCATS,
597 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},
598 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,
599 )
600 tester.run(butler, self)
602 def test_lsstcam_quickLook(self):
603 butler = self.makeButler(writeable=True)
604 tester = PipelineStepTester(
605 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),
606 [
607 "#step1a-single-visit-detectors",
608 "#step1b-single-visit-visits",
609 "#step1d-single-visit-global",
610 ],
611 initial_dataset_types=REFCATS,
612 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,
613 expected_outputs=QUICKLOOK_V2_OUTPUTS,
614 )
615 tester.run(butler, self)
617 def test_lsstcam_quickLook_bias(self):
618 butler = self.makeButler(writeable=True)
619 tester = PipelineStepTester(
620 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),
621 [
622 "#verifyBiasIsr",
623 ],
624 initial_dataset_types=[],
625 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,
626 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,
627 )
628 tester.run(butler, self)
630 def test_lsstcam_quickLook_dark(self):
631 butler = self.makeButler(writeable=True)
632 tester = PipelineStepTester(
633 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),
634 [
635 "#verifyDarkIsr",
636 ],
637 initial_dataset_types=[],
638 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,
639 expected_outputs=QUICKLOOK_DARK_OUTPUTS,
640 )
641 tester.run(butler, self)
643 def test_lsstcam_quickLook_flat(self):
644 butler = self.makeButler(writeable=True)
645 tester = PipelineStepTester(
646 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),
647 [
648 "#verifyFlatIsr",
649 ],
650 initial_dataset_types=[],
651 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,
652 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,
653 )
654 tester.run(butler, self)
656 def test_lsstcam_nightly_validation(self):
657 butler = self.makeButler(writeable=True)
658 tester = PipelineStepTester(
659 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),
660 [
661 "#step1a-single-visit-detectors",
662 "#step1b-single-visit-visits",
663 "#step1c-single-visit-tracts",
664 "#step1d-single-visit-global",
665 "#stage3-coadd"
667 ],
668 initial_dataset_types=REFCATS,
669 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,
670 # Check for some basic stage 1 outputs
671 expected_outputs={"preliminary_visit_image",
672 "preliminary_visit_summary",
673 "preliminary_visit_table",
674 "preliminary_visit_detector_table",
675 "single_visit_star_association_metrics",
676 "deep_coadd"}
677 )
678 tester.run(butler, self)
680 def test_lsstcam_drp(self):
681 butler = self.makeButler(writeable=True)
682 tester = PipelineStepTester(
683 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),
684 [
685 "#stage1-single-visit",
686 "#stage2-recalibrate",
687 "#stage3-coadd",
688 "#stage4-measure-variability",
689 ],
690 initial_dataset_types=REFCATS,
691 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},
692 expected_outputs=COMMON_V2_OUTPUTS | {"source"}
693 )
694 tester.run(butler, self)
696 def test_lsstcam_drp_compat(self):
697 butler = self.makeButler(writeable=True)
698 tester = PipelineStepTester(
699 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),
700 [
701 "#stage1-single-visit",
702 "#stage2-recalibrate",
703 "#stage3-coadd",
704 "#stage4-measure-variability",
705 ],
706 initial_dataset_types=REFCATS,
707 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},
708 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},
709 )
710 tester.run(butler, self)
712 def test_lsstcam_drp_future(self):
713 butler = self.makeButler(writeable=True)
714 tester = PipelineStepTester(
715 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-future.yaml"),
716 [
717 "#stage1-single-visit",
718 "#stage2-recalibrate",
719 "#stage3-coadd",
720 "#stage4-measure-variability",
721 ],
722 initial_dataset_types=REFCATS,
723 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},
724 expected_outputs=(COMMON_V2_OUTPUTS | {"source"}) - {"deep_coadd_background"},
725 )
726 tester.run(butler, self)
728 def test_lsstcam_drp_fl(self):
729 butler = self.makeButler(writeable=True)
730 tester = PipelineStepTester(
731 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),
732 [
733 "#stage1-single-visit",
734 "#stage2-recalibrate",
735 "#stage3-coadd",
736 "#stage4-measure-variability",
737 ],
738 initial_dataset_types=REFCATS,
739 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},
740 # Check for some basic outputs
741 expected_outputs={"preliminary_visit_image",
742 "preliminary_visit_summary",
743 "deep_coadd"}
744 )
745 tester.run(butler, self)
747 def test_lsstcam_nv_fl(self):
748 butler = self.makeButler(writeable=True)
749 tester = PipelineStepTester(
750 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),
751 [
752 "#stage1-single-visit",
753 "#stage2-recalibrate",
754 "#stage3-coadd",
755 ],
756 initial_dataset_types=REFCATS,
757 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},
758 # Check for some basic outputs
759 expected_outputs={"preliminary_visit_image",
760 "preliminary_visit_summary",
761 "deep_coadd_predetection"}
762 )
763 tester.run(butler, self)
765 def test_lsstcam_imsim_drp_ci_imsim(self):
766 butler = self.makeButler(writeable=True)
767 tester = PipelineStepTester(
768 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),
769 [""],
770 initial_dataset_types=REFCATS,
771 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
772 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
773 )
774 tester.run(butler, self)
776 def test_lsstcam_imsim_drp_test_med_1(self):
777 butler = self.makeButler(writeable=True)
778 tester = PipelineStepTester(
779 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),
780 [f"#step{N}" for N in range(1, 8)],
781 initial_dataset_types=REFCATS,
782 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
783 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
784 )
785 tester.run(butler, self)
787 def test_lsstcam_imsim_drp(self):
788 butler = self.makeButler(writeable=True)
789 tester = PipelineStepTester(
790 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),
791 [f"#step{N}" for N in range(1, 8)],
792 initial_dataset_types=REFCATS,
793 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
794 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
795 )
796 tester.run(butler, self)
798 def test_lsstcam_imsim_nightly_validation(self):
799 butler = self.makeButler(writeable=True)
800 tester = PipelineStepTester(
801 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),
802 [
803 "#step1",
804 "#step2",
805 "#step3",
806 "#step7",
807 ],
808 initial_dataset_types=REFCATS,
809 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
810 expected_outputs=COMMON_OUTPUTS,
811 )
812 tester.run(butler, self)
814 def test_lsstcomcamsim_drp(self):
815 butler = self.makeButler(writeable=True)
816 tester = PipelineStepTester(
817 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),
818 [
819 "#step1",
820 "#step2a",
821 "#step2b",
822 "#step2c",
823 "#step2d",
824 "#step2e",
825 "#step3",
826 "#step4",
827 "#step5",
828 "#step6",
829 "#step7",
830 ],
831 initial_dataset_types=REFCATS,
832 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},
833 expected_outputs=COMMON_OUTPUTS,
834 )
835 tester.run(butler, self)
837 def test_lsstcomcamsim_nightly_validation(self):
838 butler = self.makeButler(writeable=True)
839 tester = PipelineStepTester(
840 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),
841 [
842 "#step1",
843 "#step2a",
844 "#nightlyRollup",
845 "#step2b",
846 "#step2d",
847 "#step2e",
848 "#step3",
849 "#step7",
850 ],
851 initial_dataset_types=REFCATS,
852 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,
853 expected_outputs=COMMON_OUTPUTS,
854 )
855 tester.run(butler, self)
857 def test_lsstcomcamsim_quickLook(self):
858 butler = self.makeButler(writeable=True)
859 tester = PipelineStepTester(
860 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),
861 [
862 "#step1",
863 "#step2a",
864 "#nightlyRollup",
865 ],
866 initial_dataset_types=REFCATS,
867 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,
868 expected_outputs=QUICKLOOK_OUTPUTS,
869 )
870 tester.run(butler, self)
872 def test_lsstcomcam_drp(self):
873 butler = self.makeButler(writeable=True)
874 tester = PipelineStepTester(
875 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),
876 [
877 "#step1",
878 "#step2a",
879 "#step2b",
880 "#step2c",
881 "#step2d",
882 "#step2e",
883 "#step3a",
884 "#step3b",
885 "#step4",
886 "#step5",
887 "#step6",
888 "#step7",
889 ],
890 initial_dataset_types=REFCATS,
891 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},
892 expected_outputs=COMMON_OUTPUTS,
893 )
894 tester.run(butler, self)
896 def test_lsstcomcam_nightly_validation(self):
897 butler = self.makeButler(writeable=True)
898 tester = PipelineStepTester(
899 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),
900 [
901 "#step1",
902 "#step2a",
903 "#nightlyRollup",
904 "#step2b",
905 "#step2d",
906 "#step2e",
907 "#step3a",
908 "#step3b",
909 "#step7",
910 ],
911 initial_dataset_types=REFCATS,
912 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,
913 expected_outputs=COMMON_OUTPUTS,
914 )
915 tester.run(butler, self)
917 def test_lsstcomcam_quickLook(self):
918 butler = self.makeButler(writeable=True)
919 tester = PipelineStepTester(
920 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),
921 [
922 "#step1",
923 "#step2a",
924 "#nightlyRollup",
925 ],
926 initial_dataset_types=REFCATS,
927 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,
928 expected_outputs=QUICKLOOK_OUTPUTS,
929 )
930 tester.run(butler, self)
933if __name__ == "__main__": 933 ↛ 934line 933 didn't jump to line 934 because the condition on line 933 was never true
934 unittest.main()