Coverage for tests/test_pipelines.py: 99%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95 "gain_correction",

96}

98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

99LSSTCAM_IMSIM_INPUTS = {

100 "bfk",

101 "cal_ref_cat_2_2",

102 "truth_summary",

103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

104}

105

106# LSSTCam common inputs, in addition to COMMON_INPUTS

107LSSTCAM_INPUTS = {

108 "bfk",

109 "electroBfDistortionMatrix",

110 "linearizer",

111 "defects",

112 "cti",

113 "the_monster_20250219",

114 "gaia_dr3_20230707",

115 "sky",

116 "astrometry_camera",

117 "gain_correction",

118}

119

120# LSSTComCam common inputs, in addition to COMMON_INPUTS

121LSSTCOMCAM_INPUTS = {

122 "bfk",

123 "ptc",

124 "linearizer",

125 "defects",

126 "cti",

127 "the_monster_20250219",

128 "illuminationCorrection",

129 "gain_correction",

130}

131

132# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

133LSSTCOMCAMSIM_INPUTS = {

134 "uw_stars_20240524",

135 "ptc",

136 "bfk",

137 "gain_correction",

138}

139

140# a selection of mostly common outputs

141COMMON_OUTPUTS = {

142 "ccdVisitTable",

143 "compare_warp_artifact_mask",

144 "deepCoaddVisits",

145 "deepCoadd",

146 "deepCoadd_calexp",

147 "deepCoadd_calexp_background",

148 "deepCoadd_det",

149 "deepCoadd_directWarp",

150 "deepCoadd_directWarp_maskedFraction",

151 "deepCoadd_forced_src",

152 "deepCoadd_inputMap",

153 "deepCoadd_meas",

154 "deepCoadd_mergeDet",

155 "deepCoadd_nImage",

156 "deepCoadd_obj",

157 "deepCoadd_psfMatchedWarp",

158 "deepCoadd_ref",

159 "deepCoadd_scarletModelData",

160 "finalized_src_table",

161 "finalVisitSummary",

162 "initial_astrometry_match_detector",

163 "initial_photometry_match_detector",

164 "initial_psf_stars_footprints_detector",

165 "initial_psf_stars_detector",

166 "calexp",

167 "calexpBackground",

168 "src",

169 "objectTable",

170 "objectTable_tract",

171 "postISRCCD",

172 "sourceTable_visit",

173 "visitSummary",

174 "visitTable",

175 "isolated_star_presource_associations",

176 "isolated_star_presources",

177}

178

179# Translated COMMON_OUTPUTS

180COMMON_V2_OUTPUTS = {

181 'visit_detector_table',

182 'compare_warp_artifact_mask',

183 'deep_coadd_visit_selection',

184 'deep_coadd_predetection',

185 'deep_coadd',

186 'deep_coadd_background',

187 'object_detection',

188 'direct_warp',

189 'direct_warp_masked_fraction',

190 'object_forced_measurement',

191 'deep_coadd_input_map',

192 'object_unforced_measurement',

193 'object_detection_merged',

194 'deep_coadd_n_image',

195 'object_unstandardized',

196 'psf_matched_warp',

197 'object_ref_measurement',

198 'object_scarlet_models',

199 'refit_psf_star',

200 'visit_summary',

201 'initial_astrometry_match_detector',

202 'initial_photometry_match_detector',

203 'single_visit_psf_star_footprints',

204 'single_visit_psf_star',

205 'preliminary_visit_image',

206 'preliminary_visit_image_background',

207 'single_visit_star_footprints',

208 'object_patch',

209 'object_all',

210 'post_isr_image',

211 'recalibrated_star',

212 'preliminary_visit_summary',

213 'visit_table',

214 'isolated_star_association',

215 'isolated_star'

216}

217

218# HSC common outputs, in addition to COMMON_OUTPUTS

219HSC_OUTPUTS = {

220 "calexp_skyCorr_visit_mosaic",

221 "calexpBackground_skyCorr_visit_mosaic",

222 "preSourceTable",

223 "preSourceTable_visit",

224 "skyCorr",

225}

226

227# LATISS common outputs, in addition to COMMON_OUTPUTS

228LATISS_OUTPUTS = {

229 "diaObjectTable_tract",

230 "diaSourceTable",

231 "diaSourceTable_tract",

232 "fgcm_Cycle0_FitParameters",

233 "fgcm_Cycle1_FitParameters",

234 "fgcm_Cycle2_FitParameters",

235 "fgcm_Cycle3_FitParameters",

236 "fgcm_Cycle4_FitParameters",

237 "fgcm_Cycle0_FlaggedStars",

238 "fgcm_Cycle1_FlaggedStars",

239 "fgcm_Cycle2_FlaggedStars",

240 "fgcm_Cycle3_FlaggedStars",

241 "fgcm_Cycle4_FlaggedStars",

242 "fgcm_reference_stars",

243 "fgcm_star_ids",

244 "fgcm_star_observations",

245 "fgcm_standard_star",

246 "forcedSourceOnDiaObjectTable",

247 "forcedSourceOnDiaObjectTable_tract",

248 "forcedSourceTable",

249 "forcedSourceTable_tract",

250 "goodSeeingCoadd",

251 "goodSeeingCoadd_nImage",

252 "goodSeeingDiff_assocDiaSrcTable",

253 "goodSeeingDiff_diaObjTable",

254 "goodSeeingDiff_diaSrc",

255 "goodSeeingDiff_diaSrcTable",

256 "goodSeeingDiff_differenceExp",

257 "goodSeeingDiff_differenceTempExp",

258 "goodSeeingDiff_fullDiaObjTable",

259 "goodSeeingDiff_matchedExp",

260 "goodSeeingDiff_templateExp",

261 "goodSeeingVisits",

262 "mergedForcedSource",

263 "mergedForcedSourceOnDiaObject",

264 "transmission_atmosphere_fgcm",

265}

266

267# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

268LSSTCAM_IMSIM_OUTPUTS = {

269 "diaObjectTable_tract",

270 "diaSourceTable",

271 "diaSourceTable_tract",

272 "forcedSourceOnDiaObjectTable",

273 "forcedSourceOnDiaObjectTable_tract",

274 "forcedSourceTable",

275 "forcedSourceTable_tract",

276 "goodSeeingCoadd",

277 "goodSeeingCoadd_nImage",

278 "goodSeeingDiff_assocDiaSrcTable",

279 "goodSeeingDiff_diaObjTable",

280 "goodSeeingDiff_diaSrc",

281 "goodSeeingDiff_diaSrcTable",

282 "goodSeeingDiff_differenceExp",

283 "goodSeeingDiff_differenceTempExp",

284 "goodSeeingDiff_fullDiaObjTable",

285 "goodSeeingDiff_matchedExp",

286 "goodSeeingDiff_templateExp",

287 "goodSeeingVisits",

288 "mergedForcedSource",

289 "mergedForcedSourceOnDiaObject",

290}

291

292# v2 outputs common to all "quickLook" pipelines, which only include

293QUICKLOOK_V2_OUTPUTS = {

294 "preliminary_visit_image", "preliminary_visit_summary"

295}

296

297# Outputs common to all "quickLook" pipelines, which only include

298QUICKLOOK_OUTPUTS = {

299 "calexp", "visitSummary"

300}

301

302QUICKLOOK_BIAS_OUTPUTS = {

303 "post_isr_image",

304}

305

306QUICKLOOK_DARK_OUTPUTS = {

307 "post_isr_image",

308}

309

310QUICKLOOK_FLAT_OUTPUTS = {

311 "post_isr_image",

312}

313

314# All refcats used by any pipelines, for inclusion as the initial_dataset_types

315# argument to PipelineStepTester; the intent is just to allow the 'skypix'

316# dimension placeholder used in these connections to be resolved.

317REFCATS = [

318 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

319 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

320 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

321 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

322 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

323 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

324 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

325]

326

327

328class PipelineTestCase(unittest.TestCase):

329 def setUp(self):

330 self.root = makeTestTempDir(TEST_DIR)

331 self.maxDiff = None

332

333 def tearDown(self):

334 removeTestTempDir(self.root)

335

336 def makeButler(self, **kwargs: Any) -> Butler:

337 """Return new Butler instance on each call."""

338 config = Config()

339

340 # make separate temporary directory for registry of this instance

341 tmpdir = tempfile.mkdtemp(dir=self.root)

342 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

343 config = Butler.makeRepo(self.root, config)

344 butler = Butler.from_config(config, **kwargs)

345 self.enterContext(butler)

346 DatastoreMock.apply(butler)

347 return butler

348

349 def test_decam_isrForCrosstalkSources(self):

350 butler = self.makeButler(writeable=True)

351 tester = PipelineStepTester(

352 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

353 ["#step0"],

354 initial_dataset_types=REFCATS,

355 expected_inputs={

356 "camera",

357 "raw",

358 },

359 expected_outputs={"overscanRaw"},

360 )

361 tester.run(butler, self)

362

363 def test_decam_drp_merian(self):

364 butler = self.makeButler(writeable=True)

365 tester = PipelineStepTester(

366 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

367 [

368 "#step1",

369 "#step2a",

370 "#step2b",

371 "#step2d",

372 "#step2e",

373 "#step3",

374 ],

375 initial_dataset_types=REFCATS,

376 expected_inputs=COMMON_INPUTS

377 | {

378 "defects",

379 "gaia_dr2_20200414",

380 "gaia_dr3_20230707",

381 "linearizer",

382 "overscanRaw",

383 "ps1_pv3_3pi_20170110",

384 },

385 expected_outputs=COMMON_OUTPUTS

386 | {

387 "goodSeeingCoadd",

388 "goodSeeingCoadd_nImage",

389 "goodSeeingVisits",

390 "preSourceTable",

391 "preSourceTable_visit",

392 },

393 )

394 tester.run(butler, self)

395

396 def test_hsc_drp_ci_hsc(self):

397 butler = self.makeButler(writeable=True)

398 tester = PipelineStepTester(

399 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

400 [""],

401 initial_dataset_types=REFCATS,

402 expected_inputs=COMMON_INPUTS

403 | HSC_INPUTS

404 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

405 expected_outputs=COMMON_OUTPUTS

406 | HSC_OUTPUTS

407 | {

408 "diaObjectTable_tract",

409 "diaSourceTable",

410 "diaSourceTable_tract",

411 "forcedSourceOnDiaObjectTable",

412 "forcedSourceOnDiaObjectTable_tract",

413 "forcedSourceTable",

414 "forcedSourceTable_tract",

415 "goodSeeingCoadd",

416 "goodSeeingCoadd_nImage",

417 "goodSeeingDiff_assocDiaSrcTable",

418 "goodSeeingDiff_diaObjTable",

419 "goodSeeingDiff_diaSrc",

420 "goodSeeingDiff_diaSrcTable",

421 "goodSeeingDiff_differenceTempExp",

422 "goodSeeingDiff_fullDiaObjTable",

423 "goodSeeingDiff_matchedExp",

424 "goodSeeingDiff_templateExp",

425 "goodSeeingVisits",

426 "mergedForcedSource",

427 "mergedForcedSourceOnDiaObject",

428 },

429 )

430 tester.run(butler, self)

431

432 def test_hsc_drp_prod(self):

433 butler = self.makeButler(writeable=True)

434 tester = PipelineStepTester(

435 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

436 [

437 "#step1",

438 "#step2a",

439 "#step2b",

440 "#step2c",

441 "#step2d",

442 "#step2e",

443 "#step3",

444 "#step4",

445 "#step7",

446 ],

447 initial_dataset_types=REFCATS,

448 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

449 expected_outputs=COMMON_OUTPUTS

450 | HSC_OUTPUTS

451 | {

452 "fgcm_Cycle6_AtmosphereParameters",

453 "fgcm_Cycle0_FitParameters",

454 "fgcm_Cycle1_FitParameters",

455 "fgcm_Cycle2_FitParameters",

456 "fgcm_Cycle3_FitParameters",

457 "fgcm_Cycle4_FitParameters",

458 "fgcm_Cycle5_FitParameters",

459 "fgcm_Cycle6_FitParameters",

460 "fgcm_Cycle0_FlaggedStars",

461 "fgcm_Cycle1_FlaggedStars",

462 "fgcm_Cycle2_FlaggedStars",

463 "fgcm_Cycle3_FlaggedStars",

464 "fgcm_Cycle4_FlaggedStars",

465 "fgcm_Cycle5_FlaggedStars",

466 "fgcm_Cycle6_FlaggedStars",

467 "fgcm_reference_stars",

468 "fgcm_Cycle6_StandardStars",

469 "fgcm_star_ids",

470 "fgcm_star_observations",

471 "fgcm_Cycle6_Zeropoints",

472 "transmission_atmosphere_fgcm",

473 "fgcm_standard_star",

474 },

475 )

476 tester.run(butler, self)

477

478 def test_hsc_drp_rc2(self):

479 butler = self.makeButler(writeable=True)

480 tester = PipelineStepTester(

481 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

482 [

483 "#step1",

484 "#step2a",

485 "#step2b",

486 "#step2cde",

487 "#step3",

488 "#step4",

489 "#step5",

490 "#step6",

491 "#step7",

492 ],

493 initial_dataset_types=REFCATS,

494 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

495 expected_outputs=COMMON_OUTPUTS

496 | HSC_OUTPUTS

497 | {

498 "diaObjectTable_tract",

499 "diaSourceTable",

500 "diaSourceTable_tract",

501 "fgcm_Cycle4_AtmosphereParameters",

502 "fgcm_Cycle0_FitParameters",

503 "fgcm_Cycle1_FitParameters",

504 "fgcm_Cycle2_FitParameters",

505 "fgcm_Cycle3_FitParameters",

506 "fgcm_Cycle4_FitParameters",

507 "fgcm_Cycle0_FlaggedStars",

508 "fgcm_Cycle1_FlaggedStars",

509 "fgcm_Cycle2_FlaggedStars",

510 "fgcm_Cycle3_FlaggedStars",

511 "fgcm_Cycle4_FlaggedStars",

512 "fgcm_reference_stars",

513 "fgcm_Cycle4_StandardStars",

514 "fgcm_star_ids",

515 "fgcm_star_observations",

516 "fgcm_Cycle4_Zeropoints",

517 "fgcm_standard_star",

518 "forcedSourceOnDiaObjectTable",

519 "forcedSourceOnDiaObjectTable_tract",

520 "forcedSourceTable",

521 "forcedSourceTable_tract",

522 "goodSeeingCoadd",

523 "goodSeeingCoadd_nImage",

524 "goodSeeingDiff_assocDiaSrcTable",

525 "goodSeeingDiff_diaObjTable",

526 "goodSeeingDiff_diaSrc",

527 "goodSeeingDiff_diaSrcTable",

528 "goodSeeingDiff_differenceTempExp",

529 "goodSeeingDiff_fullDiaObjTable",

530 "goodSeeingDiff_matchedExp",

531 "goodSeeingDiff_templateExp",

532 "goodSeeingVisits",

533 "mergedForcedSource",

534 "mergedForcedSourceOnDiaObject",

535 "transmission_atmosphere_fgcm",

536 },

537 )

538 tester.run(butler, self)

539

540 def test_hsc_drp_rc2_subset(self):

541 butler = self.makeButler(writeable=True)

542 tester = PipelineStepTester(

543 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

544 [

545 "#nightlyStep1",

546 "#nightlyStep2a",

547 "#nightlyStep2b",

548 "#nightlyStep2c",

549 "#nightlyStep2d",

550 "#nightlyStep3",

551 "#nightlyStep4",

552 ],

553 initial_dataset_types=REFCATS,

554 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

555 expected_outputs=COMMON_OUTPUTS

556 | HSC_OUTPUTS

557 | {

558 "fgcm_Cycle4_AtmosphereParameters",

559 "fgcm_Cycle0_FitParameters",

560 "fgcm_Cycle1_FitParameters",

561 "fgcm_Cycle2_FitParameters",

562 "fgcm_Cycle3_FitParameters",

563 "fgcm_Cycle4_FitParameters",

564 "fgcm_Cycle0_FlaggedStars",

565 "fgcm_Cycle1_FlaggedStars",

566 "fgcm_Cycle2_FlaggedStars",

567 "fgcm_Cycle3_FlaggedStars",

568 "fgcm_Cycle4_FlaggedStars",

569 "fgcm_reference_stars",

570 "fgcm_Cycle4_StandardStars",

571 "fgcm_star_ids",

572 "fgcm_star_observations",

573 "fgcm_Cycle4_Zeropoints",

574 "transmission_atmosphere_fgcm",

575 "fgcm_standard_star",

576 },

577 )

578 tester.run(butler, self)

579

580 def test_latiss_drp(self):

581 butler = self.makeButler(writeable=True)

582 tester = PipelineStepTester(

583 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

584 [

585 "#step1",

586 "#step2a",

587 "#step2bcde",

588 "#step3a",

589 "#step3b",

590 "#step3c",

591 "#step4",

592 "#step5",

593 "#step6",

594 ],

595 initial_dataset_types=REFCATS,

596 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

597 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

598 )

599 tester.run(butler, self)

600

601 def test_lsstcam_quickLook(self):

602 butler = self.makeButler(writeable=True)

603 tester = PipelineStepTester(

604 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

605 [

606 "#step1a-single-visit-detectors",

607 "#step1b-single-visit-visits",

608 "#step1d-single-visit-global",

609 ],

610 initial_dataset_types=REFCATS,

611 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

612 expected_outputs=QUICKLOOK_V2_OUTPUTS,

613 )

614 tester.run(butler, self)

615

616 def test_lsstcam_quickLook_bias(self):

617 butler = self.makeButler(writeable=True)

618 tester = PipelineStepTester(

619 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

620 [

621 "#verifyBiasIsr",

622 ],

623 initial_dataset_types=[],

624 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

625 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

626 )

627 tester.run(butler, self)

628

629 def test_lsstcam_quickLook_dark(self):

630 butler = self.makeButler(writeable=True)

631 tester = PipelineStepTester(

632 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

633 [

634 "#verifyDarkIsr",

635 ],

636 initial_dataset_types=[],

637 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

638 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

639 )

640 tester.run(butler, self)

641

642 def test_lsstcam_quickLook_flat(self):

643 butler = self.makeButler(writeable=True)

644 tester = PipelineStepTester(

645 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

646 [

647 "#verifyFlatIsr",

648 ],

649 initial_dataset_types=[],

650 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

651 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

652 )

653 tester.run(butler, self)

654

655 def test_lsstcam_nightly_validation(self):

656 butler = self.makeButler(writeable=True)

657 tester = PipelineStepTester(

658 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

659 [

660 "#step1a-single-visit-detectors",

661 "#step1b-single-visit-visits",

662 "#step1c-single-visit-tracts",

663 "#step1d-single-visit-global",

664 "#stage3-coadd"

665

666 ],

667 initial_dataset_types=REFCATS,

668 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

669 # Check for some basic stage 1 outputs

670 expected_outputs={"preliminary_visit_image",

671 "preliminary_visit_summary",

672 "preliminary_visit_table",

673 "preliminary_visit_detector_table",

674 "single_visit_star_association_metrics",

675 "deep_coadd"}

676 )

677 tester.run(butler, self)

678

679 def test_lsstcam_drp(self):

680 butler = self.makeButler(writeable=True)

681 tester = PipelineStepTester(

682 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

683 [

684 "#stage1-single-visit",

685 "#stage2-recalibrate",

686 "#stage3-coadd",

687 "#stage4-measure-variability",

688 ],

689 initial_dataset_types=REFCATS,

690 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

691 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

692 )

693 tester.run(butler, self)

694

695 def test_lsstcam_drp_compat(self):

696 butler = self.makeButler(writeable=True)

697 tester = PipelineStepTester(

698 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

699 [

700 "#stage1-single-visit",

701 "#stage2-recalibrate",

702 "#stage3-coadd",

703 "#stage4-measure-variability",

704 ],

705 initial_dataset_types=REFCATS,

706 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

707 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

708 )

709 tester.run(butler, self)

710

711 def test_lsstcam_drp_future(self):

712 butler = self.makeButler(writeable=True)

713 tester = PipelineStepTester(

714 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-future.yaml"),

715 [

716 "#stage1-single-visit",

717 "#stage2-recalibrate",

718 "#stage3-coadd",

719 "#stage4-measure-variability",

720 ],

721 initial_dataset_types=REFCATS,

722 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

723 expected_outputs=(COMMON_V2_OUTPUTS | {"source"}) - {"deep_coadd_background"},

724 )

725 tester.run(butler, self)

726

727 def test_lsstcam_drp_fl(self):

728 butler = self.makeButler(writeable=True)

729 tester = PipelineStepTester(

730 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

731 [

732 "#stage1-single-visit",

733 "#stage2-recalibrate",

734 "#stage3-coadd",

735 "#stage4-measure-variability",

736 ],

737 initial_dataset_types=REFCATS,

738 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

739 # Check for some basic outputs

740 expected_outputs={"preliminary_visit_image",

741 "preliminary_visit_summary",

742 "deep_coadd"}

743 )

744 tester.run(butler, self)

745

746 def test_lsstcam_nv_fl(self):

747 butler = self.makeButler(writeable=True)

748 tester = PipelineStepTester(

749 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

750 [

751 "#stage1-single-visit",

752 "#stage2-recalibrate",

753 "#stage3-coadd",

754 ],

755 initial_dataset_types=REFCATS,

756 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

757 # Check for some basic outputs

758 expected_outputs={"preliminary_visit_image",

759 "preliminary_visit_summary",

760 "deep_coadd_predetection"}

761 )

762 tester.run(butler, self)

763

764 def test_lsstcam_imsim_drp_ci_imsim(self):

765 butler = self.makeButler(writeable=True)

766 tester = PipelineStepTester(

767 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

768 [""],

769 initial_dataset_types=REFCATS,

770 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

771 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

772 )

773 tester.run(butler, self)

774

775 def test_lsstcam_imsim_drp_test_med_1(self):

776 butler = self.makeButler(writeable=True)

777 tester = PipelineStepTester(

778 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

779 [f"#step{N}" for N in range(1, 8)],

780 initial_dataset_types=REFCATS,

781 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

782 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

783 )

784 tester.run(butler, self)

785

786 def test_lsstcam_imsim_drp(self):

787 butler = self.makeButler(writeable=True)

788 tester = PipelineStepTester(

789 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

790 [f"#step{N}" for N in range(1, 8)],

791 initial_dataset_types=REFCATS,

792 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

793 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

794 )

795 tester.run(butler, self)

796

797 def test_lsstcam_imsim_nightly_validation(self):

798 butler = self.makeButler(writeable=True)

799 tester = PipelineStepTester(

800 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

801 [

802 "#step1",

803 "#step2",

804 "#step3",

805 "#step7",

806 ],

807 initial_dataset_types=REFCATS,

808 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

809 expected_outputs=COMMON_OUTPUTS,

810 )

811 tester.run(butler, self)

812

813 def test_lsstcomcamsim_drp(self):

814 butler = self.makeButler(writeable=True)

815 tester = PipelineStepTester(

816 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

817 [

818 "#step1",

819 "#step2a",

820 "#step2b",

821 "#step2c",

822 "#step2d",

823 "#step2e",

824 "#step3",

825 "#step4",

826 "#step5",

827 "#step6",

828 "#step7",

829 ],

830 initial_dataset_types=REFCATS,

831 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

832 expected_outputs=COMMON_OUTPUTS,

833 )

834 tester.run(butler, self)

835

836 def test_lsstcomcamsim_nightly_validation(self):

837 butler = self.makeButler(writeable=True)

838 tester = PipelineStepTester(

839 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

840 [

841 "#step1",

842 "#step2a",

843 "#nightlyRollup",

844 "#step2b",

845 "#step2d",

846 "#step2e",

847 "#step3",

848 "#step7",

849 ],

850 initial_dataset_types=REFCATS,

851 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

852 expected_outputs=COMMON_OUTPUTS,

853 )

854 tester.run(butler, self)

855

856 def test_lsstcomcamsim_quickLook(self):

857 butler = self.makeButler(writeable=True)

858 tester = PipelineStepTester(

859 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

860 [

861 "#step1",

862 "#step2a",

863 "#nightlyRollup",

864 ],

865 initial_dataset_types=REFCATS,

866 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

867 expected_outputs=QUICKLOOK_OUTPUTS,

868 )

869 tester.run(butler, self)

870

871 def test_lsstcomcam_drp(self):

872 butler = self.makeButler(writeable=True)

873 tester = PipelineStepTester(

874 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

875 [

876 "#step1",

877 "#step2a",

878 "#step2b",

879 "#step2c",

880 "#step2d",

881 "#step2e",

882 "#step3a",

883 "#step3b",

884 "#step4",

885 "#step5",

886 "#step6",

887 "#step7",

888 ],

889 initial_dataset_types=REFCATS,

890 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

891 expected_outputs=COMMON_OUTPUTS,

892 )

893 tester.run(butler, self)

894

895 def test_lsstcomcam_nightly_validation(self):

896 butler = self.makeButler(writeable=True)

897 tester = PipelineStepTester(

898 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

899 [

900 "#step1",

901 "#step2a",

902 "#nightlyRollup",

903 "#step2b",

904 "#step2d",

905 "#step2e",

906 "#step3a",

907 "#step3b",

908 "#step7",

909 ],

910 initial_dataset_types=REFCATS,

911 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

912 expected_outputs=COMMON_OUTPUTS,

913 )

914 tester.run(butler, self)

915

916 def test_lsstcomcam_quickLook(self):

917 butler = self.makeButler(writeable=True)

918 tester = PipelineStepTester(

919 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

920 [

921 "#step1",

922 "#step2a",

923 "#nightlyRollup",

924 ],

925 initial_dataset_types=REFCATS,

926 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

927 expected_outputs=QUICKLOOK_OUTPUTS,

928 )

929 tester.run(butler, self)

930

931

932if __name__ == "__main__": 932 ↛ 933line 932 didn't jump to line 933 because the condition on line 932 was never true

933 unittest.main()