Coverage for python/lsst/pipe/tasks/multiBand.py: 20%

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DetectCoaddSourcesConfig", "DetectCoaddSourcesTask", 

23 "MeasureMergedCoaddSourcesConfig", "MeasureMergedCoaddSourcesTask", 

24 "DEEP_COADD_BACKGROUND_DOCSTRING", 

25 ] 

26 

27import dataclasses 

28 

29import astropy.units 

30import numpy as np 

31 

32from lsst.geom import Extent2I 

33from lsst.pipe.base import ( 

34 AnnotatedPartialOutputsError, 

35 Struct, 

36 PipelineTask, 

37 PipelineTaskConfig, 

38 PipelineTaskConnections 

39) 

40import lsst.pipe.base.connectionTypes as cT 

41from lsst.pex.config import Field, ChoiceField, ConfigurableField 

42from lsst.cell_coadds import MultipleCellCoadd 

43from lsst.images import Mask, get_legacy_deep_coadd_mask_planes 

44from lsst.images.cells import CellCoadd 

45from lsst.images.fields import field_from_legacy_background 

46from lsst.meas.algorithms import ( 

47 DynamicDetectionTask, 

48 ExceedsMaxVarianceScaleError, 

49 InsufficientSourcesError, 

50 PsfGenerationError, 

51 ReferenceObjectLoader, 

52 ScaleVarianceTask, 

53 SetPrimaryFlagsTask, 

54 TooManyMaskedPixelsError, 

55 ZeroFootprintError, 

56) 

57from lsst.meas.base import ( 

58 SingleFrameMeasurementTask, 

59 ApplyApCorrTask, 

60 CatalogCalculationTask, 

61 SkyMapIdGeneratorConfig, 

62) 

63from lsst.meas.extensions.scarlet.io import updateCatalogFootprints 

64from lsst.meas.astrom import DirectMatchTask, denormalizeMatches 

65from lsst.pipe.tasks.propagateSourceFlags import PropagateSourceFlagsTask 

66import lsst.afw.image as afwImage 

67import lsst.afw.math as afwMath 

68import lsst.afw.table as afwTable 

69from lsst.daf.base import PropertyList 

70from lsst.skymap import BaseSkyMap 

71 

72# NOTE: these imports are a convenience so multiband users only have to import this file. 

73from .mergeDetections import MergeDetectionsConfig, MergeDetectionsTask # noqa: F401 

74from .mergeMeasurements import MergeMeasurementsConfig, MergeMeasurementsTask # noqa: F401 

75from .multiBandUtils import CullPeaksConfig # noqa: F401 

76from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiConfig # noqa: F401 

77from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiTask # noqa: F401 

78 

79 

80""" 

81New set types: 

82* deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 

83* deepCoadd_mergeDet: merged detections (tract, patch) 

84* deepCoadd_meas: measurements of merged detections (tract, patch, filter) 

85* deepCoadd_ref: reference sources (tract, patch) 

86All of these have associated *_schema catalogs that require no data ID and hold no records. 

87 

88In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 

89the mergeDet, meas, and ref dataset Footprints: 

90* deepCoadd_peak_schema 

91""" 

92 

93 

94############################################################################################################## 

95 

96# The default for DetectCoaddSourcesConfig.backgroundDescription: 

97DEEP_COADD_BACKGROUND_DOCSTRING = ( 

98 "Background subtracted from the image when generating the Object catalog. " 

99 "This intentionally oversubtracts the background to reduce blending and ensure " 

100 "scattered light is subtracted. " 

101 "Restoring this background does not restore all original backgrounds, " 

102 "as the coadd was built from background-subtracted visit images; in most " 

103 "cases this background term is actually quite small " 

104) 

105 

106 

107class DetectCoaddSourcesConnections(PipelineTaskConnections, 

108 dimensions=("tract", "patch", "band", "skymap"), 

109 defaultTemplates={"inputCoaddName": "deep", "outputCoaddName": "deep"}): 

110 detectionSchema = cT.InitOutput( 

111 doc="Schema of the detection catalog", 

112 name="{outputCoaddName}Coadd_det_schema", 

113 storageClass="SourceCatalog", 

114 ) 

115 exposure = cT.Input( 

116 doc="Exposure on which detections are to be performed (if useCellCoadd=False). ", 

117 name="{inputCoaddName}Coadd", 

118 storageClass="ExposureF", 

119 dimensions=("tract", "patch", "band", "skymap") 

120 ) 

121 exposure_cells = cT.Input( 

122 doc="Exposure on which detections are to be performed (if useCellCoadd=True). ", 

123 name="{inputCoaddName}CoaddCell", 

124 storageClass="MultipleCellCoadd", 

125 dimensions=("tract", "patch", "band", "skymap"), 

126 ) 

127 skyMap = cT.Input( 

128 doc="Description of the skymap's tracts and patches.", 

129 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

130 storageClass="SkyMap", 

131 dimensions=("skymap",), 

132 ) 

133 outputBackgrounds = cT.Output( 

134 doc="Output Backgrounds used in detection", 

135 name="{outputCoaddName}Coadd_calexp_background", 

136 storageClass="Background", 

137 dimensions=("tract", "patch", "band", "skymap") 

138 ) 

139 outputSources = cT.Output( 

140 doc="Detected sources catalog", 

141 name="{outputCoaddName}Coadd_det", 

142 storageClass="SourceCatalog", 

143 dimensions=("tract", "patch", "band", "skymap") 

144 ) 

145 outputExposure = cT.Output( 

146 doc="Exposure post detection", 

147 name="{outputCoaddName}Coadd_calexp", 

148 storageClass="ExposureF", 

149 dimensions=("tract", "patch", "band", "skymap") 

150 ) 

151 

152 def __init__(self, *, config=None): 

153 super().__init__(config=config) 

154 assert isinstance(config, DetectCoaddSourcesConfig) 

155 

156 if config.imageType == "future": 

157 if config.useCellCoadds: 

158 self.exposure_cells = dataclasses.replace(self.exposure_cells, storageClass="CellCoadd") 

159 else: 

160 self.exposure = dataclasses.replace(self.exposure, storageClass="CellCoadd") 

161 self.outputExposure = dataclasses.replace(self.outputExposure, storageClass="CellCoadd") 

162 del self.outputBackgrounds 

163 if config.useCellCoadds: 

164 del self.exposure 

165 else: 

166 del self.exposure_cells 

167 if not self.config.forceExactBinning: 

168 del self.skyMap 

169 if self.config.writeOnlyBackgrounds: 

170 del self.outputExposure 

171 del self.outputSources 

172 del self.detectionSchema 

173 

174 

175class DetectCoaddSourcesConfig(PipelineTaskConfig, pipelineConnections=DetectCoaddSourcesConnections): 

176 """Configuration parameters for the DetectCoaddSourcesTask 

177 """ 

178 

179 doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?") 

180 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling") 

181 detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection") 

182 coaddName = Field(dtype=str, default="deep", doc="Name of coadd") 

183 useCellCoadds = Field(dtype=bool, default=False, doc="Whether to use cell coadds?") 

184 hasFakes = Field( 

185 dtype=bool, 

186 default=False, 

187 doc="Should be set to True if fake sources have been inserted into the input data.", 

188 ) 

189 idGenerator = SkyMapIdGeneratorConfig.make_field() 

190 forceExactBinning = Field( 

191 dtype=bool, 

192 default=False, 

193 doc=( 

194 "Check that the background bin size evenly divides the patch inner region, and " 

195 "crop the outer region to an integer number of bins." 

196 ) 

197 ) 

198 writeOnlyBackgrounds = Field(dtype=bool, default=False, doc="If true, only save the background models.") 

199 writeEmptyBackgrounds = Field( 

200 dtype=bool, 

201 default=True, 

202 doc=( 

203 "If true, save a placeholder background with NaNs in all bins (but the right geometry) when " 

204 "there are no pixels to compute a background from. This can be useful if a later task combines " 

205 "backgrounds from multiple patches as input." 

206 ) 

207 ) 

208 imageType = ChoiceField( 

209 "Which image type to use for the input and output coadd. " 

210 "This option only directly affects connection storage classes and hence 'runQuantum'; the 'run' " 

211 "method behavior is determined by which type is actually passed in.", 

212 allowed={ 

213 "legacy": ( 

214 "Read a lsst.cell_coadds.MultipleCellCoadd (if useCellCoadd) or " 

215 "lsst.afw.image.Exposure (if not useCellCoadd) and write an lsst.afw.image.Exposure." 

216 ), 

217 "future": ( 

218 "Read and write lsst.images.cells.CellCoadd (useCellCoadd just " 

219 "sets which of 'exposure' or 'exposure_cells' will be used for inputs). " 

220 "The 'outputBackground' is deleted, writeEmptyBackgrounds is ignored, and " 

221 "writeOnlyBackgrounds=True is invalid." 

222 ), 

223 }, 

224 dtype=str, 

225 optional=False, 

226 default="legacy", 

227 ) 

228 backgroundName = Field( 

229 "Name of the subtracted background, to be stored with the image when the input and " 

230 "output images are lsst.images.cells.CellCoadd.", 

231 dtype=str, 

232 default="object", 

233 ) 

234 backgroundDescription = Field( 

235 "Description of the subtracted background, to be stored with the image when the input and " 

236 "output images are lsst.images.cells.CellCoadd.", 

237 dtype=str, 

238 default=DEEP_COADD_BACKGROUND_DOCSTRING, 

239 ) 

240 

241 def setDefaults(self): 

242 super().setDefaults() 

243 self.detection.thresholdType = "pixel_stdev" 

244 self.detection.isotropicGrow = True 

245 # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic 

246 self.detection.reEstimateBackground = False 

247 self.detection.background.useApprox = False 

248 self.detection.background.binSize = 4096 

249 self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER' 

250 self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender 

251 # Include band in packed data IDs that go into object IDs (None -> "as 

252 # many bands as are defined", rather than the default of zero). 

253 self.idGenerator.packer.n_bands = None 

254 

255 def validate(self): 

256 super().validate() 

257 if self.imageType == "future": 

258 if self.doScaleVariance: 

259 raise ValueError("doScaleVariance=True is not compatible with imageType='future'") 

260 if self.forceExactBinning: 

261 raise ValueError("forceExactBinning=True is not compatible with imageType='future'") 

262 if self.writeOnlyBackgrounds: 

263 raise ValueError("writeOnlyBackgrounds=True is not compatible with imageType='future'") 

264 

265 

266class DetectCoaddSourcesTask(PipelineTask): 

267 """Detect sources on a single filter coadd. 

268 

269 Coadding individual visits requires each exposure to be warped. This 

270 introduces covariance in the noise properties across pixels. Before 

271 detection, we correct the coadd variance by scaling the variance plane in 

272 the coadd to match the observed variance. This is an approximate 

273 approach -- strictly, we should propagate the full covariance matrix -- 

274 but it is simple and works well in practice. 

275 

276 After scaling the variance plane, we detect sources and generate footprints 

277 by delegating to the @ref SourceDetectionTask_ "detection" subtask. 

278 

279 DetectCoaddSourcesTask is meant to be run after assembling a coadded image 

280 in a given band. The purpose of the task is to update the background, 

281 detect all sources in a single band and generate a set of parent 

282 footprints. Subsequent tasks in the multi-band processing procedure will 

283 merge sources across bands and, eventually, perform forced photometry. 

284 

285 Parameters 

286 ---------- 

287 schema : `lsst.afw.table.Schema`, optional 

288 Initial schema for the output catalog, modified-in place to include all 

289 fields set by this task. If None, the source minimal schema will be used. 

290 **kwargs 

291 Additional keyword arguments. 

292 """ 

293 

294 _DefaultName = "detectCoaddSources" 

295 ConfigClass = DetectCoaddSourcesConfig 

296 

297 def __init__(self, schema=None, **kwargs): 

298 # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree 

299 # call structure has been reviewed carefully to be sure super will work as intended. 

300 super().__init__(**kwargs) 

301 if schema is None: 

302 schema = afwTable.SourceTable.makeMinimalSchema() 

303 self.schema = schema 

304 self.makeSubtask("detection", schema=self.schema) 

305 if self.config.doScaleVariance: 

306 self.makeSubtask("scaleVariance") 

307 

308 self.detectionSchema = afwTable.SourceCatalog(self.schema) 

309 

310 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

311 inputs = butlerQC.get(inputRefs) 

312 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

313 

314 if self.config.useCellCoadds: 

315 multiple_cell_coadd = inputs.pop("exposure_cells") 

316 match self.config.imageType: 

317 case "legacy": 

318 exposure = multiple_cell_coadd.stitch().asExposure() 

319 case "future": 

320 exposure = multiple_cell_coadd # conversion deferred to run(). 

321 case _: 

322 raise AssertionError(f"Invalid choice {self.config.imageType!r} for imageType.") 

323 else: 

324 exposure = inputs.pop("exposure") 

325 

326 skyMap = inputs.pop("skyMap", None) 

327 if skyMap is not None: 

328 patchInfo = skyMap[butlerQC.quantum.dataId["tract"]][butlerQC.quantum.dataId["patch"]] 

329 else: 

330 patchInfo = None 

331 

332 assert not inputs, "runQuantum got more inputs than expected." 

333 try: 

334 outputs = self.run( 

335 exposure=exposure, 

336 idFactory=idGenerator.make_table_id_factory(), 

337 expId=idGenerator.catalog_id, 

338 patchInfo=patchInfo, 

339 ) 

340 except ( 

341 TooManyMaskedPixelsError, 

342 ExceedsMaxVarianceScaleError, 

343 InsufficientSourcesError, 

344 PsfGenerationError, 

345 ZeroFootprintError, 

346 ) as e: 

347 if self.config.writeEmptyBackgrounds: 

348 butlerQC.put(self._makeEmptyBackground(exposure, patchInfo), outputRefs.outputBackgrounds) 

349 # Detection failed, so clear any leftover detected mask planes. 

350 if isinstance(exposure, CellCoadd): 

351 # If we passed a CellCoadd in, it won't be modified until 'run' 

352 # is about to exit, and hence it can't have picked up a 

353 # DETECTED_NEGATIVE plane, which can only be temporary here. 

354 # But it might have a preexisting DETECTED plane (i.e. a union 

355 # of the DETECTED plane from the warps) that we'd still want to 

356 # clear. 

357 exposure.mask.clear("DETECTED") 

358 else: 

359 for maskName in ["DETECTED", "DETECTED_NEGATIVE"]: 

360 if maskName in exposure.mask.getMaskPlaneDict().keys(): 

361 detectedMask = exposure.mask.getMaskPlane(maskName) 

362 exposure.mask.clearMaskPlane(detectedMask) 

363 error = AnnotatedPartialOutputsError.annotate( 

364 e, 

365 self, 

366 exposure, 

367 log=self.log, 

368 ) 

369 butlerQC.put(exposure, outputRefs.outputExposure) 

370 raise error from e 

371 

372 butlerQC.put(outputs, outputRefs) 

373 

374 def run(self, exposure, idFactory, expId, patchInfo=None): 

375 """Run detection on an exposure. 

376 

377 First scale the variance plane to match the observed variance 

378 using ``ScaleVarianceTask``. Then invoke the ``SourceDetectionTask_`` "detection" subtask to 

379 detect sources. 

380 

381 Parameters 

382 ---------- 

383 exposure : `lsst.afw.image.Exposure` or `lsst.images.cells.CellCoadd`. 

384 Exposure on which to detect (may be background-subtracted and scaled, 

385 depending on configuration). 

386 idFactory : `lsst.afw.table.IdFactory` 

387 IdFactory to set source identifiers. 

388 expId : `int` 

389 Exposure identifier (integer) for RNG seed. 

390 patchInfo : `lsst.skymap.PatchInfo`, optional 

391 Description of the patch geometry. Only needed if 

392 `~DetectCoaddSourceConfig.forceExactBinning` is `True`. 

393 

394 Returns 

395 ------- 

396 result : `lsst.pipe.base.Struct` 

397 Results as a struct with attributes: 

398 

399 ``outputSources`` 

400 Catalog of detections (`lsst.afw.table.SourceCatalog`). 

401 ``outputBackgrounds`` 

402 List of backgrounds (`list`). 

403 ``outputExposure`` 

404 The background-subtracted coadd image, with its mask plane 

405 updated to include detections. This will have the same type 

406 as ``exposure``. 

407 """ 

408 cell_coadd = None 

409 if isinstance(exposure, CellCoadd): 

410 cell_coadd = exposure 

411 exposure = cell_coadd.to_legacy() 

412 if self.config.forceExactBinning: 

413 if cell_coadd is not None: 

414 raise ValueError("forceExactBinning=True is not compatible with CellCoadd inputs") 

415 exposure = self._cropToExactBinning(exposure, patchInfo) 

416 if self.config.doScaleVariance: 

417 if cell_coadd is not None: 

418 raise ValueError("doScaleVariance=True is not compatible with CellCoadd inputs") 

419 varScale = self.scaleVariance.run(exposure.maskedImage) 

420 exposure.getMetadata().add("VARIANCE_SCALE", varScale) 

421 backgrounds = afwMath.BackgroundList() 

422 table = afwTable.SourceTable.make(self.schema, idFactory) 

423 detections = self.detection.run(table, exposure, expId=expId) 

424 sources = detections.sources 

425 if hasattr(detections, "background") and detections.background: 

426 for bg in detections.background: 

427 backgrounds.append(bg) 

428 if len(backgrounds) == 0: 

429 # Persist a constant background with value of NaN to get around 

430 # inability to persist empty BackgroundList. 

431 emptyBg = self._makeEmptyBackground(exposure, patchInfo) 

432 backgrounds.append(emptyBg) 

433 if cell_coadd is not None: 

434 cell_coadd.image.array[...] = exposure.image.array 

435 cell_coadd.mask = Mask.from_legacy( 

436 exposure.mask, plane_map=get_legacy_deep_coadd_mask_planes() 

437 ).view(sky_projection=cell_coadd.sky_projection) 

438 if backgrounds: 

439 cell_coadd.backgrounds.add( 

440 self.config.backgroundName, 

441 field_from_legacy_background(backgrounds, unit=astropy.units.nJy), 

442 self.config.backgroundDescription, 

443 is_subtracted=True, 

444 ) 

445 exposure = cell_coadd 

446 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure) 

447 

448 def _cropToExactBinning(self, exposure, patchInfo): 

449 """Crop a coadd `~lsst.afw.image.Exposure` instance to ensure exact 

450 background binning. 

451 

452 Parameters 

453 ---------- 

454 exposure : `lsst.afw.image.Exposure` 

455 Exposure to crop, assumed to cover the patch outer bounding box. 

456 patchInfo : `lsst.skymap.PatchInfo` 

457 Description of the patch geometry. 

458 

459 Returns 

460 ------- 

461 cropped : `lsst.afw.image.Exposure` 

462 View of ``exposure`` with background bins that evenly divide both 

463 the full cropped image and the patch inner region. The bounding 

464 box is guaranteed to contain the patch inner bounding box and be 

465 contained by the patch outer bounding box. 

466 

467 Raises 

468 ------ 

469 ValueError 

470 Raised if the patch inner region width or height is not a multiple 

471 of the background bin size. 

472 """ 

473 bbox = patchInfo.getInnerBBox() 

474 if bbox.width % self.detection.background.binSizeX: 

475 raise ValueError( 

476 f"Patch inner width {bbox.width} does not evenly " 

477 f"divide bin width {self.detection.background.binSizeX}." 

478 ) 

479 if bbox.height % self.detection.background.binSizeY: 

480 raise ValueError( 

481 f"Patch inner height {bbox.height} does not evenly " 

482 f"divide bin height {self.detection.background.binSizeY}." 

483 ) 

484 outer_bbox = patchInfo.getOuterBBox() 

485 n_bins_grow_x = (bbox.x.begin - outer_bbox.x.begin) // self.detection.background.binSizeX 

486 n_bins_grow_y = (bbox.y.begin - outer_bbox.y.begin) // self.detection.background.binSizeY 

487 bbox.grow( 

488 Extent2I( 

489 n_bins_grow_x*self.detection.background.binSizeX, 

490 n_bins_grow_y*self.detection.background.binSizeY, 

491 ) 

492 ) 

493 assert outer_bbox.contains(bbox) 

494 assert bbox.contains(patchInfo.getInnerBBox()) 

495 assert bbox.width % self.detection.background.binSizeX == 0 

496 assert bbox.height % self.detection.background.binSizeY == 0 

497 return exposure[bbox] 

498 

499 def _makeEmptyBackground(self, exposure, patchInfo=None): 

500 """Construct an empty `lsst.afw.math.BackgroundList` with NaN values. 

501 

502 Parameters 

503 ---------- 

504 exposure : `lsst.afw.image.Exposure` 

505 Exposure that the background should correspond to. 

506 patchInfo : `lsst.skymap.PatchInfo`, optional 

507 Description of the patch geometry. Only needed if 

508 `~DetectCoaddSourceConfig.forceExactBinning` is `True`. 

509 

510 Returns 

511 ------- 

512 background : `lsst.afw.math.BackgroundList` 

513 A background object with a single layer and the same bin geometry 

514 that a background for that exposure would have had if it had enough 

515 usable pixels. This object cannot actually be used for background 

516 subtraction. 

517 """ 

518 # Create a backgroundList with one entry whose "stats image" is NaNs 

519 # and has all pixels set as NO_DATA. 

520 if self.config.forceExactBinning: 

521 exposure = self._cropToExactBinning(exposure, patchInfo).clone() 

522 

523 bgLevel = np.nan 

524 bgStats = afwImage.MaskedImageF(1, 1) 

525 bgStats.set(bgLevel, 0, bgLevel) 

526 bg = afwMath.BackgroundMI(exposure.getBBox(), bgStats) 

527 bgData = (bg, afwMath.Interpolate.LINEAR, afwMath.REDUCE_INTERP_ORDER, 

528 afwMath.ApproximateControl.UNKNOWN, 0, 0, False) 

529 background = afwMath.BackgroundList() 

530 background.append(bgData) 

531 for bg, *_ in background: 

532 stats = bg.getStatsImage() 

533 stats.mask.array[:, :] = stats.mask.getPlaneBitMask("NO_DATA") 

534 stats.variance.array[:, :] = 0.0 

535 return background 

536 

537 

538class MeasureMergedCoaddSourcesConnections( 

539 PipelineTaskConnections, 

540 dimensions=("tract", "patch", "band", "skymap"), 

541 defaultTemplates={ 

542 "inputCoaddName": "deep", 

543 "outputCoaddName": "deep", 

544 "deblendedCatalog": "deblendedFlux", 

545 }, 

546 deprecatedTemplates={ 

547 # TODO[DM-47797]: remove this deprecated connection template. 

548 "deblendedCatalog": "Support for old deblender outputs will be removed after v29." 

549 }, 

550): 

551 inputSchema = cT.InitInput( 

552 doc="Input schema for measure merged task produced by a deblender or detection task", 

553 name="{inputCoaddName}Coadd_deblendedFlux_schema", 

554 storageClass="SourceCatalog" 

555 ) 

556 outputSchema = cT.InitOutput( 

557 doc="Output schema after all new fields are added by task", 

558 name="{inputCoaddName}Coadd_meas_schema", 

559 storageClass="SourceCatalog" 

560 ) 

561 # TODO[DM-47797]: remove this deprecated connection. 

562 refCat = cT.PrerequisiteInput( 

563 doc="Reference catalog used to match measured sources against known sources", 

564 name="ref_cat", 

565 storageClass="SimpleCatalog", 

566 dimensions=("skypix",), 

567 deferLoad=True, 

568 multiple=True, 

569 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

570 ) 

571 exposure = cT.Input( 

572 doc="Input non-cell-based coadd image", 

573 name="{inputCoaddName}Coadd_calexp", 

574 storageClass="ExposureF", 

575 dimensions=("tract", "patch", "band", "skymap") 

576 ) 

577 exposure_cells = cT.Input( 

578 doc="Input cell-based coadd image", 

579 name="{inputCoaddName}CoaddCell", 

580 storageClass="MultipleCellCoadd", 

581 dimensions=("tract", "patch", "band", "skymap"), 

582 ) 

583 background = cT.Input( 

584 doc="Background to subtract from cell-based coadd image", 

585 name="{inputCoaddName}Coadd_calexp_background", 

586 storageClass="Background", 

587 dimensions=("tract", "patch", "band", "skymap") 

588 ) 

589 skyMap = cT.Input( 

590 doc="SkyMap to use in processing", 

591 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

592 storageClass="SkyMap", 

593 dimensions=("skymap",), 

594 ) 

595 # TODO[DM-47424]: remove this deprecated connection. 

596 visitCatalogs = cT.Input( 

597 doc="Deprecated and unused.", 

598 name="src", 

599 dimensions=("instrument", "visit", "detector"), 

600 storageClass="SourceCatalog", 

601 multiple=True, 

602 deprecated="Deprecated and unused. Will be removed after v29.", 

603 ) 

604 sourceTableHandles = cT.Input( 

605 doc=("Source tables that are derived from the ``CalibrateTask`` sources. " 

606 "These tables contain astrometry and photometry flags, and optionally " 

607 "PSF flags."), 

608 name="sourceTable_visit", 

609 storageClass="ArrowAstropy", 

610 dimensions=("instrument", "visit"), 

611 multiple=True, 

612 deferLoad=True, 

613 ) 

614 finalizedSourceTableHandles = cT.Input( 

615 doc=("Finalized source tables from ``FinalizeCalibrationTask``. These " 

616 "tables contain PSF flags from the finalized PSF estimation."), 

617 name="finalized_src_table", 

618 storageClass="ArrowAstropy", 

619 dimensions=("instrument", "visit"), 

620 multiple=True, 

621 deferLoad=True, 

622 ) 

623 finalVisitSummaryHandles = cT.Input( 

624 doc="Final visit summary table", 

625 name="finalVisitSummary", 

626 storageClass="ExposureCatalog", 

627 dimensions=("instrument", "visit"), 

628 multiple=True, 

629 deferLoad=True, 

630 ) 

631 # TODO[DM-47797]: remove this deprecated connection. 

632 inputCatalog = cT.Input( 

633 doc=("Name of the input catalog to use." 

634 "If the single band deblender was used this should be 'deblendedFlux." 

635 "If the multi-band deblender was used this should be 'deblendedModel, " 

636 "or deblendedFlux if the multiband deblender was configured to output " 

637 "deblended flux catalogs. If no deblending was performed this should " 

638 "be 'mergeDet'"), 

639 name="{inputCoaddName}Coadd_{deblendedCatalog}", 

640 storageClass="SourceCatalog", 

641 deprecated="Support for old deblender outputs will be removed after v29.", 

642 dimensions=("tract", "patch", "band", "skymap"), 

643 ) 

644 scarletCatalog = cT.Input( 

645 doc="Catalogs produced by multiband deblending", 

646 name="{inputCoaddName}Coadd_deblendedCatalog", 

647 storageClass="SourceCatalog", 

648 dimensions=("tract", "patch", "skymap"), 

649 ) 

650 scarletModels = cT.Input( 

651 doc="Multiband scarlet models produced by the deblender", 

652 name="{inputCoaddName}Coadd_scarletModelData", 

653 storageClass="LsstScarletModelData", 

654 dimensions=("tract", "patch", "skymap"), 

655 ) 

656 outputSources = cT.Output( 

657 doc="Source catalog containing all the measurement information generated in this task", 

658 name="{outputCoaddName}Coadd_meas", 

659 dimensions=("tract", "patch", "band", "skymap"), 

660 storageClass="SourceCatalog", 

661 ) 

662 # TODO[DM-47797]: remove this deprecated connection. 

663 matchResult = cT.Output( 

664 doc="Match catalog produced by configured matcher, optional on doMatchSources", 

665 name="{outputCoaddName}Coadd_measMatch", 

666 dimensions=("tract", "patch", "band", "skymap"), 

667 storageClass="Catalog", 

668 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

669 ) 

670 # TODO[DM-47797]: remove this deprecated connection. 

671 denormMatches = cT.Output( 

672 doc="Denormalized Match catalog produced by configured matcher, optional on " 

673 "doWriteMatchesDenormalized", 

674 name="{outputCoaddName}Coadd_measMatchFull", 

675 dimensions=("tract", "patch", "band", "skymap"), 

676 storageClass="Catalog", 

677 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

678 ) 

679 

680 def __init__(self, *, config=None): 

681 super().__init__(config=config) 

682 del self.visitCatalogs 

683 if not config.doPropagateFlags: 

684 del self.sourceTableHandles 

685 del self.finalizedSourceTableHandles 

686 else: 

687 # Check for types of flags required. 

688 if not config.propagateFlags.source_flags: 

689 del self.sourceTableHandles 

690 if not config.propagateFlags.finalized_source_flags: 

691 del self.finalizedSourceTableHandles 

692 # TODO[DM-47797]: only the 'if' block contents here should survive. 

693 if config.inputCatalog == "deblendedCatalog": 

694 del self.inputCatalog 

695 if not config.doAddFootprints: 

696 del self.scarletModels 

697 else: 

698 del self.deblendedCatalog 

699 del self.scarletModels 

700 

701 # TODO[DM-47797]: delete the conditionals below. 

702 if not config.doMatchSources: 

703 del self.refCat 

704 del self.matchResult 

705 

706 if not config.doWriteMatchesDenormalized: 

707 del self.denormMatches 

708 

709 if self.config.imageType == "future": 

710 self.exposure = dataclasses.replace(self.exposure, storageClass="CellCoadd") 

711 del self.exposure_cells 

712 del self.background 

713 elif self.config.useCellCoadds: 

714 del self.exposure 

715 else: 

716 del self.exposure_cells 

717 del self.background 

718 

719 

720class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig, 

721 pipelineConnections=MeasureMergedCoaddSourcesConnections): 

722 """Configuration parameters for the MeasureMergedCoaddSourcesTask 

723 """ 

724 inputCatalog = ChoiceField( 

725 dtype=str, 

726 default="deblendedCatalog", 

727 allowed={ 

728 "deblendedCatalog": "Output catalog from ScarletDeblendTask", 

729 "deblendedFlux": "Output catalog from SourceDeblendTask", 

730 "mergeDet": "The merged detections before deblending." 

731 }, 

732 doc="The name of the input catalog.", 

733 # TODO[DM-47797]: remove this config option and anything using it. 

734 deprecated="Support for old deblender outputs will be removed after v29.", 

735 ) 

736 doAddFootprints = Field(dtype=bool, 

737 default=True, 

738 doc="Whether or not to add footprints to the input catalog from scarlet models. " 

739 "This should be true whenever using the multi-band deblender, " 

740 "otherwise this should be False.") 

741 doConserveFlux = Field(dtype=bool, default=True, 

742 doc="Whether to use the deblender models as templates to re-distribute the flux " 

743 "from the 'exposure' (True), or to perform measurements on the deblender " 

744 "model footprints.") 

745 doStripFootprints = Field(dtype=bool, default=True, 

746 doc="Whether to strip footprints from the output catalog before " 

747 "saving to disk. " 

748 "This is usually done when using scarlet models to save disk space.") 

749 useCellCoadds = Field(dtype=bool, default=False, doc="Whether to use cell coadds?") 

750 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement") 

751 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch") 

752 doPropagateFlags = Field( 

753 dtype=bool, default=True, 

754 doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 

755 ) 

756 propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd") 

757 doMatchSources = Field( 

758 dtype=bool, 

759 default=False, 

760 doc="Match sources to reference catalog?", 

761 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

762 ) 

763 match = ConfigurableField( 

764 target=DirectMatchTask, 

765 doc="Matching to reference catalog", 

766 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

767 ) 

768 doWriteMatchesDenormalized = Field( 

769 dtype=bool, 

770 default=False, 

771 doc=("Write reference matches in denormalized format? " 

772 "This format uses more disk space, but is more convenient to read."), 

773 deprecated="Reference matching in measureCoaddSources will be removed after v29.", 

774 ) 

775 coaddName = Field(dtype=str, default="deep", doc="Name of coadd") 

776 psfCache = Field(dtype=int, default=100, doc="Size of psfCache") 

777 checkUnitsParseStrict = Field( 

778 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

779 dtype=str, 

780 default="raise", 

781 ) 

782 doApCorr = Field( 

783 dtype=bool, 

784 default=True, 

785 doc="Apply aperture corrections" 

786 ) 

787 applyApCorr = ConfigurableField( 

788 target=ApplyApCorrTask, 

789 doc="Subtask to apply aperture corrections" 

790 ) 

791 doRunCatalogCalculation = Field( 

792 dtype=bool, 

793 default=True, 

794 doc='Run catalogCalculation task' 

795 ) 

796 catalogCalculation = ConfigurableField( 

797 target=CatalogCalculationTask, 

798 doc="Subtask to run catalogCalculation plugins on catalog" 

799 ) 

800 

801 hasFakes = Field( 

802 dtype=bool, 

803 default=False, 

804 doc="Should be set to True if fake sources have been inserted into the input data." 

805 ) 

806 idGenerator = SkyMapIdGeneratorConfig.make_field() 

807 imageType = ChoiceField( 

808 "Which image type to expect for the input coadd. " 

809 "This option only directly affects connection storage classes and hence 'runQuantum'; the 'run' " 

810 "method behavior is determined by which type is actually passed in.", 

811 allowed={ 

812 "legacy": ( 

813 "Read a lsst.cell_coadds.MultipleCellCoadd via 'exposure_cells` and restore 'background' " 

814 "(if useCellCoadd) or lsst.afw.image.Exposure via `exposure` (if not useCellCoadd)." 

815 ), 

816 "future": ( 

817 "Read lsst.images.cells.CellCoadd via the 'exposure' connection. useCellCoadd is ignored." 

818 ), 

819 }, 

820 dtype=str, 

821 optional=False, 

822 default="legacy", 

823 ) 

824 

825 @property 

826 def refObjLoader(self): 

827 return self.match.refObjLoader 

828 

829 def setDefaults(self): 

830 super().setDefaults() 

831 self.measurement.plugins.names |= ['base_InputCount', 

832 'base_Variance', 

833 'base_LocalPhotoCalib', 

834 'base_LocalWcs'] 

835 

836 # TODO: Remove STREAK in DM-44658, streak masking to happen only in 

837 # ip_diffim; if we can propagate the streak mask from diffim, we can 

838 # still set flags with it here. 

839 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE', 

840 'INEXACT_PSF'] 

841 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE', 

842 'INEXACT_PSF'] 

843 

844 def validate(self): 

845 super().validate() 

846 

847 if not self.doMatchSources and self.doWriteMatchesDenormalized: 

848 raise ValueError("Cannot set doWriteMatchesDenormalized if doMatchSources is False.") 

849 

850 

851class MeasureMergedCoaddSourcesTask(PipelineTask): 

852 """Deblend sources from main catalog in each coadd seperately and measure. 

853 

854 Use peaks and footprints from a master catalog to perform deblending and 

855 measurement in each coadd. 

856 

857 Given a master input catalog of sources (peaks and footprints) or deblender 

858 outputs(including a HeavyFootprint in each band), measure each source on 

859 the coadd. Repeating this procedure with the same master catalog across 

860 multiple coadds will generate a consistent set of child sources. 

861 

862 The deblender retains all peaks and deblends any missing peaks (dropouts in 

863 that band) as PSFs. Source properties are measured and the @c is-primary 

864 flag (indicating sources with no children) is set. Visit flags are 

865 propagated to the coadd sources. 

866 

867 Optionally, we can match the coadd sources to an external reference 

868 catalog. 

869 

870 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we 

871 have a set of per-band catalogs. The next stage in the multi-band 

872 processing procedure will merge these measurements into a suitable catalog 

873 for driving forced photometry. 

874 

875 Parameters 

876 ---------- 

877 schema : ``lsst.afw.table.Schema`, optional 

878 The schema of the merged detection catalog used as input to this one. 

879 peakSchema : ``lsst.afw.table.Schema`, optional 

880 The schema of the PeakRecords in the Footprints in the merged detection catalog. 

881 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

882 An instance of ReferenceObjectLoader that supplies an external reference 

883 catalog. May be None if the loader can be constructed from the butler argument or all steps 

884 requiring a reference catalog are disabled. 

885 initInputs : `dict`, optional 

886 Dictionary that can contain a key ``inputSchema`` containing the 

887 input schema. If present will override the value of ``schema``. 

888 **kwargs