Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 27%

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import copy 

37 

38import numpy as np 

39 

40import lsst.pex.config as pexConfig 

41import lsst.pipe.base as pipeBase 

42from lsst.pipe.base import connectionTypes 

43import lsst.afw.table as afwTable 

44 

45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

46from .utilities import extractReferenceMags 

47from .utilities import makeZptSchema, makeZptCat 

48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

49from .sedterms import SedboundarytermDict, SedtermDict 

50from .focalPlaneProjector import FocalPlaneProjector 

51 

52import fgcm 

53 

54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask'] 

55 

56MULTIPLE_CYCLES_MAX = 10 

57 

58 

59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument",), 

61 defaultTemplates={"previousCycleNumber": "-1", 

62 "cycleNumber": "0"}): 

63 camera = connectionTypes.PrerequisiteInput( 

64 doc="Camera instrument", 

65 name="camera", 

66 storageClass="Camera", 

67 dimensions=("instrument",), 

68 isCalibration=True, 

69 ) 

70 

71 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

72 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

73 "chromatic corrections."), 

74 name="fgcmLookUpTable", 

75 storageClass="Catalog", 

76 dimensions=("instrument",), 

77 deferLoad=True, 

78 ) 

79 

80 fgcmVisitCatalog = connectionTypes.Input( 

81 doc="Catalog of visit information for fgcm", 

82 name="fgcmVisitCatalog", 

83 storageClass="Catalog", 

84 dimensions=("instrument",), 

85 deferLoad=True, 

86 ) 

87 

88 fgcmStarObservationsParquet = connectionTypes.Input( 

89 doc=("Catalog of star observations for fgcm, in parquet format. " 

90 "Used if useParquetCatalogFormat is True."), 

91 name="fgcm_star_observations", 

92 storageClass="ArrowAstropy", 

93 dimensions=("instrument",), 

94 deferLoad=True, 

95 ) 

96 

97 fgcmStarIdsParquet = connectionTypes.Input( 

98 doc=("Catalog of fgcm calibration star IDs, in parquet format. " 

99 "Used if useParquetCatalogFormat is True."), 

100 name="fgcm_star_ids", 

101 storageClass="ArrowAstropy", 

102 dimensions=("instrument",), 

103 deferLoad=True, 

104 ) 

105 

106 fgcmReferenceStarsParquet = connectionTypes.Input( 

107 doc=("Catalog of fgcm-matched reference stars, in parquet format. " 

108 "Used if useParquetCatalogFormat is True."), 

109 name="fgcm_reference_stars", 

110 storageClass="ArrowAstropy", 

111 dimensions=("instrument",), 

112 deferLoad=True, 

113 ) 

114 

115 fgcmStarObservations = connectionTypes.Input( 

116 doc=("Catalog of star observations for fgcm; old format. " 

117 "Used if useParquetCatalogFormat is False."), 

118 name="fgcmStarObservations", 

119 storageClass="Catalog", 

120 dimensions=("instrument",), 

121 deferLoad=True, 

122 ) 

123 

124 fgcmStarIds = connectionTypes.Input( 

125 doc=("Catalog of fgcm calibration star IDs. " 

126 "Used if useParquetCatalogFormat is False."), 

127 name="fgcmStarIds", 

128 storageClass="Catalog", 

129 dimensions=("instrument",), 

130 deferLoad=True, 

131 ) 

132 

133 fgcmStarIndices = connectionTypes.Input( 

134 doc=("Catalog of fgcm calibration star indices; old format." 

135 "Used if useParquetCatalogFormat is False."), 

136 name="fgcmStarIndices", 

137 storageClass="Catalog", 

138 dimensions=("instrument",), 

139 deferLoad=True, 

140 ) 

141 

142 fgcmReferenceStars = connectionTypes.Input( 

143 doc=("Catalog of fgcm-matched reference stars; old format." 

144 "Used if useParquetCatalogFormat is False."), 

145 name="fgcmReferenceStars", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 deferLoad=True, 

149 ) 

150 

151 def __init__(self, *, config=None): 

152 super().__init__(config=config) 

153 

154 # The connection parameters ``cycleNumber`` and ``previousCycleNumber`` 

155 # always comes in as strings, and doing a round-trip via 

156 # integer-to-string ensures that they actually are integers. 

157 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

158 raise ValueError("cycleNumber must be of integer format") 

159 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

160 raise ValueError("previousCycleNumber must be of integer format") 

161 # We additionally confirm that there is a delta of 1 between them. 

162 # Note that this math cannot be done in the parameters because they are 

163 # strings. 

164 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

165 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

166 

167 instDims = ("instrument",) 

168 bandDims = ("instrument", "band") 

169 filterDims = ("instrument", "physical_filter") 

170 filterDetectorDims = ("instrument", "physical_filter", "detector") 

171 

172 inputAndOutputConnections = [ 

173 ("FitParameters", "Catalog", "Catalog of fgcm fit parameters.", instDims), 

174 ("FlaggedStars", "Catalog", "Catalog of flagged stars for fgcm calibration.", instDims), 

175 ] 

176 multicycleOutputConnections = [ 

177 ("OutputConfig", "Config", "Configuration for next fgcm fit cycle.", instDims), 

178 ] 

179 optionalZpOutputConnections = [ 

180 ("Zeropoints", "Catalog", "Catalog of fgcm zeropoint data.", instDims), 

181 ("AtmosphereParameters", "Catalog", "Catalog of atmospheric fit parameters.", instDims), 

182 ] 

183 optionalStarOutputConnections = [ 

184 ("StandardStars", "SimpleCatalog", "Catalog of standard star magnitudes.", instDims), 

185 ] 

186 

187 epochs = [f"epoch{i}" for i in range(len(config.epochMjds))] 

188 

189 # All names will be preceeded with ``fgcm_CycleN_``. 

190 plotConnections = [ 

191 ("Zeropoints_Plot", "Plot", "Plot of fgcm zeropoints.", instDims), 

192 ("ExpgrayDeep_Plot", 

193 "Plot", 

194 "Plot of gray term per exposure per time for deep fields.", 

195 instDims), 

196 ("NightlyAlpha_Plot", "Plot", "Plot of nightly AOD alpha term.", instDims), 

197 ("NightlyTau_Plot", "Plot", "Plot of nightly aerosol optical depth (tau).", instDims), 

198 ("NightlyPwv_Plot", "Plot", "Plot of nightly water vapor.", instDims), 

199 ("NightlyO3_Plot", "Plot", "Plot of nightly ozone.", instDims), 

200 ("FilterOffsets_Plot", "Plot", "Plot of in-band filter offsets.", instDims), 

201 ("AbsThroughputs_Plot", "Plot", "Plot of absolute throughput fractions.", instDims), 

202 ("QESysWashesInitial_Plot", "Plot", "Plot of initial system QE with mirror washes.", instDims), 

203 ("QESysWashesFinal_Plot", "Plot", "Plot of final system QE with mirror washes.", instDims), 

204 ("RpwvVsRpwvInput_Plot", 

205 "Plot", 

206 "Plot of change in per-visit ``retrieved`` PWV from previous fit cycle.", 

207 instDims), 

208 ("RpwvVsRpwvSmooth_Plot", 

209 "Plot", 

210 "Plot of per-visit ``retrieved`` PWV vs. smoothed PWV.", 

211 instDims), 

212 ("ModelPwvVsRpwv_Plot", 

213 "Plot", 

214 "Plot of model PWV vs. per-visit ``retrieved`` PWV.", 

215 instDims), 

216 ("ChisqFit_Plot", 

217 "Plot", 

218 "Plot of chisq as a function of iteration.", 

219 instDims), 

220 ] 

221 

222 plotConnections.extend( 

223 [ 

224 ("Apercorr_Plot", "Plot", "Plot of fgcm aperture corrections.", bandDims), 

225 ("EpsilonGlobal_Plot", 

226 "Plot", 

227 "Plot of global background over/undersubtraction.", 

228 bandDims), 

229 ("EpsilonMap_Plot", 

230 "Plot", 

231 "Map of spatially varying background over/undersubtraction.", 

232 bandDims), 

233 ("ExpgrayInitial_Plot", 

234 "Plot", 

235 "Histogram of initial gray term per exposure.", 

236 bandDims), 

237 ("CompareRedblueExpgray_Plot", 

238 "Plot", 

239 "Plot of red/blue split gray term per exposure", 

240 bandDims), 

241 ("Expgray_Plot", "Plot", "Histogram of gray term per exposure.", bandDims), 

242 ("ExpgrayAirmass_Plot", 

243 "Plot", 

244 "Plot of exposure gray term as a function of airmass.", 

245 bandDims), 

246 ("ExpgrayCompareMjdRedblue_Plot", 

247 "Plot", 

248 "Plot of red/blue split gray term per exposure as a function of time.", 

249 bandDims), 

250 ("ExpgrayUT_Plot", 

251 "Plot", 

252 "Plot of grey term per exposure as a function of time of night.", 

253 bandDims), 

254 ("ExpgrayCompareBands_Plot", 

255 "Plot", 

256 "Plot of gray term per exposure between bands nearby in time.", 

257 bandDims), 

258 ("ExpgrayReference_Plot", 

259 "Plot", 

260 "Histogram of gray term per exposure compared to reference mags.", 

261 bandDims), 

262 ("QESysRefstarsStdInitial_Plot", 

263 "Plot", 

264 "Plot of reference mag - calibrated (standard) mag vs. time (before fit).", 

265 bandDims), 

266 ("QESysRefstarsStdFinal_Plot", 

267 "Plot", 

268 "Plot of reference mag - calibrated (standard) mag vs. time (after fit).", 

269 bandDims), 

270 ("QESysRefstarsObsInitial_Plot", 

271 "Plot", 

272 "Plot of reference mag - observed (instrumental) mag vs. time (before fit).", 

273 bandDims), 

274 ("QESysRefstarsObsFinal_Plot", 

275 "Plot", 

276 "Plot of reference mag - observed (instrumental) mag vs. time (after fit).", 

277 bandDims), 

278 ("ModelMagerrInitial_Plot", 

279 "Plot", 

280 "Plots for magnitude error model, initial estimate.", 

281 bandDims), 

282 ("ModelMagerrPostfit_Plot", 

283 "Plot", 

284 "Plots for magnitude error model, after fitting.", 

285 bandDims), 

286 ("SigmaFgcmAllStars_Plot", 

287 "Plot", 

288 "Histograms for intrinsic scatter for all bright stars.", 

289 bandDims), 

290 ("SigmaFgcmReservedStars_Plot", 

291 "Plot", 

292 "Histograms for intrinsic scatter for reserved bright stars.", 

293 bandDims), 

294 ("SigmaFgcmReservedStarsCrunched_Plot", 

295 "Plot", 

296 "Histograms for intrinsic scatter for reserved bright stars (after gray correction).", 

297 bandDims), 

298 ("SigmaFgcmReservedStarsCrunchedNonPhotom_Plot", 

299 "Plot", 

300 "Histograms for intrinsic scatter for reserved bright stars (including non-photometric).", 

301 bandDims), 

302 ("SigmaFgcmPullsAllStars_Plot", 

303 "Plot", 

304 "Histograms for pulls for all bright stars.", 

305 bandDims), 

306 ("SigmaFgcmPullsReservedStars_Plot", 

307 "Plot", 

308 "Histograms for pulls for reserved bright stars.", 

309 bandDims), 

310 ("SigmaFgcmPullsReservedStarsCrunched_Plot", 

311 "Plot", 

312 "Histograms for pulls for reserved bright stars (after gray correction).", 

313 bandDims), 

314 ("SigmaFgcmPullsReservedStarsCrunchedNonPhotom_Plot", 

315 "Plot", 

316 "Histograms for pulls for reserved bright stars (including non-photometric).", 

317 bandDims), 

318 ("SigmaCal_Plot", 

319 "Plot", 

320 "Plot showing scatter as a function of systematic error floor.", 

321 bandDims), 

322 ("SigmaRef_Plot", 

323 "Plot", 

324 "Histograms of scatter with respect to reference stars.", 

325 bandDims), 

326 ("RefResidVsColorAll_Plot", 

327 "Plot", 

328 "Plot of reference star residuals vs. color (all stars).", 

329 bandDims), 

330 ("RefResidVsColorCut_Plot", 

331 "Plot", 

332 "Plot of reference star residuals vs. color (reference star color cuts).", 

333 bandDims), 

334 ("DensityAllStarMap_Plot", 

335 "Plot", 

336 "Density map of all stars input to calibration.", 

337 bandDims), 

338 ("DensityGoodStarMap_Plot", 

339 "Plot", 

340 "Density map of good stars constrainted after calibration.", 

341 bandDims), 

342 ] 

343 ) 

344 

345 plotConnections.extend( 

346 [ 

347 ("I1R1_Plot", "Plot", "Plot of fgcm R1 vs. I1.", filterDims), 

348 ("I1_Plot", "Plot", "Focal plane map of fgcm I1.", filterDims), 

349 ("R1_Plot", "Plot", "Focal plane map of fgcm R1.", filterDims), 

350 ("R1mI1Matchscale_Plot", "Plot", "Focal plane map of fgcm R1 - I1.", filterDims), 

351 ("R1mI1_Plot", "Plot", "Focal plane map of fgcm R1 - I1 (rescaled).", filterDims), 

352 ("R1mI1_vs_mjd_Plot", "Plot", "R1 - I1 residuals vs. mjd.", filterDims), 

353 ("CompareRedblueMirrorchrom_Plot", 

354 "Plot", 

355 "Comparison of mirror chromaticity residuals for red/blue stars.", 

356 filterDims), 

357 ("CcdChromaticity_Plot", 

358 "Plot", 

359 "Focal plane map of fgcm ccd chromaticity.", 

360 filterDims), 

361 ("EpsilonDetector_Plot", 

362 "Plot", 

363 "Focal plane map of background over/undersubtraction.", 

364 filterDims), 

365 ("EpsilonDetectorMatchscale_Plot", 

366 "Plot", 

367 "Focal plane map of background over/undersubtraction.", 

368 filterDims), 

369 ] 

370 ) 

371 for epoch in epochs: 

372 plotConnections.extend( 

373 [ 

374 ( 

375 f"Superstar_{epoch}_Plot", 

376 "Plot", 

377 "Plot of illumination Correction.", 

378 filterDims, 

379 ), 

380 ], 

381 ) 

382 if self.config.superStarPlotCcdResiduals: 

383 plotConnections.extend( 

384 [ 

385 ( 

386 f"SuperstarResidual_{epoch}_Plot", 

387 "Plot", 

388 "Binned illumination correction residuals.", 

389 filterDetectorDims, 

390 ), 

391 ( 

392 f"SuperstarResidualStd_{epoch}_Plot", 

393 "Plot", 

394 "Binned illumination correction residual stdev.", 

395 filterDetectorDims, 

396 ), 

397 ], 

398 ) 

399 

400 if config.doMultipleCycles: 

401 # Multiple cycle run. 

402 

403 # All but the final cycle get appended here. 

404 for cycle in range(config.multipleCyclesFinalCycleNumber): 

405 outputConnections = copy.copy(inputAndOutputConnections) 

406 outputConnections.extend(multicycleOutputConnections) 

407 if config.outputZeropointsBeforeFinalCycle: 

408 outputConnections.extend(optionalZpOutputConnections) 

409 if config.outputStandardsBeforeFinalCycle: 

410 outputConnections.extend(optionalStarOutputConnections) 

411 

412 if config.doPlots: 

413 # We will make the plots if doPlots is True and either 

414 # it is the next-to-last cycle or the 

415 # doPlotsBeforeFinalCycles configuration is set which 

416 # means we want plots for all cycles. 

417 # The final cycle doPlots is configured below. 

418 if cycle == (config.multipleCyclesFinalCycleNumber - 1) \ 

419 or config.doPlotsBeforeFinalCycles: 

420 outputConnections.extend(plotConnections) 

421 

422 for (name, storageClass, doc, dims) in outputConnections: 

423 connectionName = f"fgcm_Cycle{cycle}_{name}" 

424 storageName = connectionName 

425 outConnection = connectionTypes.Output( 

426 name=storageName, 

427 storageClass=storageClass, 

428 doc=doc, 

429 dimensions=dims, 

430 multiple=(len(dims) > 1), 

431 ) 

432 setattr(self, connectionName, outConnection) 

433 

434 # The final cycle has everything. 

435 outputConnections = copy.copy(inputAndOutputConnections) 

436 outputConnections.extend(multicycleOutputConnections) 

437 outputConnections.extend(optionalZpOutputConnections) 

438 outputConnections.extend(optionalStarOutputConnections) 

439 if config.doPlots: 

440 outputConnections.extend(plotConnections) 

441 for (name, storageClass, doc, dims) in outputConnections: 

442 connectionName = f"fgcm_Cycle{config.multipleCyclesFinalCycleNumber}_{name}" 

443 storageName = connectionName 

444 outConnection = connectionTypes.Output( 

445 name=storageName, 

446 storageClass=storageClass, 

447 doc=doc, 

448 dimensions=dims, 

449 multiple=(len(dims) > 1), 

450 ) 

451 setattr(self, connectionName, outConnection) 

452 else: 

453 # Single cycle run. 

454 if config.cycleNumber > 0: 

455 inputConnections = copy.copy(inputAndOutputConnections) 

456 else: 

457 inputConnections = [] 

458 outputConnections = copy.copy(inputAndOutputConnections) 

459 # The following configurations are also useful for runs 

460 # where fit cycles are run one-at-a-time since it is 

461 # not typical to look at these outputs for intermediate 

462 # steps. 

463 if config.isFinalCycle or config.outputZeropointsBeforeFinalCycle: 

464 outputConnections.extend(optionalZpOutputConnections) 

465 if config.isFinalCycle or config.outputStandardsBeforeFinalCycle: 

466 outputConnections.extend(optionalStarOutputConnections) 

467 

468 if config.doPlots: 

469 outputConnections.extend(plotConnections) 

470 

471 for (name, storageClass, doc, dims) in inputConnections: 

472 connectionName = f"fgcm{name}Input" 

473 storageName = f"fgcm_Cycle{config.cycleNumber - 1}_{name}" 

474 inConnection = connectionTypes.PrerequisiteInput( 

475 name=storageName, 

476 storageClass=storageClass, 

477 doc=doc, 

478 dimensions=dims, 

479 ) 

480 setattr(self, connectionName, inConnection) 

481 

482 for (name, storageClass, doc, dims) in outputConnections: 

483 connectionName = f"fgcm{name}" 

484 storageName = f"fgcm_Cycle{config.cycleNumber}_{name}" 

485 # Plots have unique names as well. 

486 if storageClass == "Plot": 

487 connectionName = storageName 

488 outConnection = connectionTypes.Output( 

489 name=storageName, 

490 storageClass=storageClass, 

491 doc=doc, 

492 dimensions=dims, 

493 multiple=(len(dims) > 1), 

494 ) 

495 setattr(self, connectionName, outConnection) 

496 

497 if not config.doReferenceCalibration: 

498 self.inputs.remove("fgcmReferenceStars") 

499 self.inputs.remove("fgcmReferenceStarsParquet") 

500 

501 if config.useParquetCatalogFormat: 

502 self.inputs.remove("fgcmStarObservations") 

503 self.inputs.remove("fgcmStarIds") 

504 self.inputs.remove("fgcmStarIndices") 

505 if config.doReferenceCalibration: 

506 self.inputs.remove("fgcmReferenceStars") 

507 else: 

508 self.inputs.remove("fgcmStarObservationsParquet") 

509 self.inputs.remove("fgcmStarIdsParquet") 

510 if config.doReferenceCalibration: 

511 self.inputs.remove("fgcmReferenceStarsParquet") 

512 

513 

514class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

515 pipelineConnections=FgcmFitCycleConnections): 

516 """Config for FgcmFitCycle""" 

517 

518 doMultipleCycles = pexConfig.Field( 

519 doc="Run multiple fit cycles in one task", 

520 dtype=bool, 

521 default=False, 

522 ) 

523 useParquetCatalogFormat = pexConfig.Field( 

524 doc="Use parquet catalog format?", 

525 dtype=bool, 

526 default=True, 

527 ) 

528 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

529 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

530 "is 0, with limited parameters fit. The next cycle is 1 with " 

531 "full parameter fit. The final cycle is a clean-up with no " 

532 "parameters fit. There will be a total of " 

533 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

534 "cycle number cannot be less than 2."), 

535 dtype=int, 

536 default=5, 

537 min=2, 

538 max=MULTIPLE_CYCLES_MAX, 

539 inclusiveMax=True, 

540 ) 

541 bands = pexConfig.ListField( 

542 doc="Bands to run calibration", 

543 dtype=str, 

544 default=[], 

545 ) 

546 fitBands = pexConfig.ListField( 

547 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

548 "the atmosphere constrained from the 'fitBands' on the same night. " 

549 "Must be a subset of `config.bands`"), 

550 dtype=str, 

551 default=[], 

552 ) 

553 requiredBands = pexConfig.ListField( 

554 doc=("Bands that are required for a star to be considered a calibration star. " 

555 "Must be a subset of `config.bands`"), 

556 dtype=str, 

557 default=[], 

558 ) 

559 physicalFilterMap = pexConfig.DictField( 

560 doc="Mapping from 'physicalFilter' to band.", 

561 keytype=str, 

562 itemtype=str, 

563 default={}, 

564 ) 

565 doReferenceCalibration = pexConfig.Field( 

566 doc="Use reference catalog as additional constraint on calibration", 

567 dtype=bool, 

568 default=True, 

569 ) 

570 refStarSnMin = pexConfig.Field( 

571 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

572 dtype=float, 

573 default=50.0, 

574 ) 

575 refStarOutlierNSig = pexConfig.Field( 

576 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

577 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

578 dtype=float, 

579 default=4.0, 

580 ) 

581 applyRefStarColorCuts = pexConfig.Field( 

582 doc=("Apply color cuts defined in ``starColorCuts`` to reference stars? " 

583 "These cuts are in addition to any cuts defined in ``refStarColorCuts``"), 

584 dtype=bool, 

585 default=True, 

586 ) 

587 refStarMaxFracUse = pexConfig.Field( 

588 doc=("Maximum fraction of reference stars to use in the fit. Remainder will " 

589 "be used only for validation."), 

590 dtype=float, 

591 default=0.5, 

592 ) 

593 useExposureReferenceOffset = pexConfig.Field( 

594 doc=("Use per-exposure (visit) offsets between calibrated stars and reference stars " 

595 "for final zeropoints? This may help uniformity for disjoint surveys."), 

596 dtype=bool, 

597 default=False, 

598 ) 

599 nStarPerRun = pexConfig.Field( 

600 doc="Number of stars to run in each chunk. Larger values tend to be faster for large " 

601 "datasets, at the expense of some memory overhead.", 

602 dtype=int, 

603 default=200000, 

604 ) 

605 nStarPerGrayRun = pexConfig.Field( 

606 doc="Number of stars to run in each chunk (including gray correction). Increasing this " 

607 "value may increase the peak memory use significantly.", 

608 dtype=int, 

609 default=50000, 

610 ) 

611 nObsPerRun = pexConfig.Field( 

612 doc="Number of observations to run in each chunk. Larger values tend to be faster for large " 

613 "datasets, at the expense of some memory overhead.", 

614 dtype=int, 

615 default=500000, 

616 ) 

617 nObsPerGrayRun = pexConfig.Field( 

618 doc="Number of observations to run in each chunk (including gray correction). Increasing this " 

619 "value may increase the peak memory use significantly.", 

620 dtype=int, 

621 default=100000, 

622 ) 

623 nExpPerRun = pexConfig.Field( 

624 doc="Number of exposures to run in each chunk", 

625 dtype=int, 

626 default=1000, 

627 ) 

628 reserveFraction = pexConfig.Field( 

629 doc="Fraction of stars to reserve for testing", 

630 dtype=float, 

631 default=0.1, 

632 ) 

633 freezeStdAtmosphere = pexConfig.Field( 

634 doc="Freeze atmosphere parameters to standard (for testing)", 

635 dtype=bool, 

636 default=False, 

637 ) 

638 precomputeSuperStarInitialCycle = pexConfig.Field( 

639 doc="Precompute superstar flat for initial cycle", 

640 dtype=bool, 

641 default=False, 

642 ) 

643 superStarSubCcdDict = pexConfig.DictField( 

644 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

645 "Must have one entry per band."), 

646 keytype=str, 

647 itemtype=bool, 

648 default={}, 

649 ) 

650 superStarSubCcdChebyshevOrder = pexConfig.Field( 

651 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

652 "Global default is first-order polynomials, and should be overridden " 

653 "on a camera-by-camera basis depending on the ISR."), 

654 dtype=int, 

655 default=1, 

656 ) 

657 superStarSubCcdTriangular = pexConfig.Field( 

658 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

659 "suppress high-order cross terms?"), 

660 dtype=bool, 

661 default=False, 

662 ) 

663 superStarSigmaClip = pexConfig.Field( 

664 doc="Number of sigma to clip outliers when selecting for superstar flats", 

665 dtype=float, 

666 default=5.0, 

667 ) 

668 superStarPlotCcdResiduals = pexConfig.Field( 

669 doc="If plotting is enabled, should per-detector residuals be plotted? " 

670 "This may produce a lot of output, and should be used only for " 

671 "debugging purposes.", 

672 dtype=bool, 

673 default=False, 

674 ) 

675 superStarForceZeroMean = pexConfig.Field( 

676 doc="When computing the super-star flat, force the focal-plane mean to " 

677 "zero (per band)? This should only be used when computing stand-alone " 

678 "illumination corrections.", 

679 dtype=bool, 

680 default=False, 

681 ) 

682 focalPlaneSigmaClip = pexConfig.Field( 

683 doc="Number of sigma to clip outliers per focal-plane.", 

684 dtype=float, 

685 default=4.0, 

686 ) 

687 ccdGraySubCcdDict = pexConfig.DictField( 

688 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

689 "('ccd gray') on a sub-ccd scale. This will only be used as a fallback " 

690 "if ``ccdGrayFocalPlaneDict[band]`` is set to True."), 

691 keytype=str, 

692 itemtype=bool, 

693 default={}, 

694 ) 

695 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

696 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

697 dtype=int, 

698 default=1, 

699 ) 

700 ccdGraySubCcdTriangular = pexConfig.Field( 

701 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

702 "suppress high-order cross terms?"), 

703 dtype=bool, 

704 default=True, 

705 ) 

706 ccdGrayFocalPlaneDict = pexConfig.DictField( 

707 doc=("Per-band specification on whether to compute focal-plane residual " 

708 "('ccd gray') corrections. The focal-plane residuals will be used " 

709 "instead of per-CCD residuals except when there are insufficient " 

710 "good CCDs (see ``ccdGrayFocalPlaneFitMinCcd``). Therefore, this " 

711 "is not recommended for large focal planes with non-photometric " 

712 "exposures."), 

713 keytype=str, 

714 itemtype=bool, 

715 default={}, 

716 ) 

717 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

718 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

719 "gray corrections. If there are fewer good CCDs then the gray " 

720 "correction is computed per-ccd."), 

721 dtype=int, 

722 default=1, 

723 ) 

724 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

725 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

726 dtype=int, 

727 default=3, 

728 ) 

729 ccdGrayFocalPlaneMaxStars = pexConfig.Field( 

730 doc="Maximum number of stars to use for focal plane fit. Required to keep " 

731 "matrix memory usage from running away. If there are more stars than " 

732 "this then they will be down-sampled.", 

733 dtype=int, 

734 default=50_000, 

735 ) 

736 cycleNumber = pexConfig.Field( 

737 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

738 "and stage of outlier rejection. See cookbook for details."), 

739 dtype=int, 

740 default=None, 

741 ) 

742 isFinalCycle = pexConfig.Field( 

743 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

744 "selection of stars and photometric exposures, and will output zeropoints " 

745 "and standard stars for use in fgcmOutputProducts"), 

746 dtype=bool, 

747 default=False, 

748 ) 

749 maxIterBeforeFinalCycle = pexConfig.Field( 

750 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

751 "will always be 0 in the final cycle for cleanup and final selection."), 

752 dtype=int, 

753 default=50, 

754 ) 

755 deltaMagBkgOffsetPercentile = pexConfig.Field( 

756 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

757 "offset from local background subtraction."), 

758 dtype=float, 

759 default=0.25, 

760 ) 

761 deltaMagBkgPerCcd = pexConfig.Field( 

762 doc=("Compute net offset from local background subtraction per-ccd? " 

763 "Otherwise, use computation per visit."), 

764 dtype=bool, 

765 default=False, 

766 ) 

767 utBoundary = pexConfig.Field( 

768 doc="Boundary (in UTC) from day-to-day", 

769 dtype=float, 

770 default=None, 

771 ) 

772 washMjds = pexConfig.ListField( 

773 doc="Mirror wash MJDs", 

774 dtype=float, 

775 default=(0.0,), 

776 ) 

777 epochMjds = pexConfig.ListField( 

778 doc="Epoch boundaries in MJD", 

779 dtype=float, 

780 default=(0.0,), 

781 ) 

782 minObsPerBand = pexConfig.Field( 

783 doc="Minimum good observations per band", 

784 dtype=int, 

785 default=2, 

786 ) 

787 # TODO: When DM-16511 is done, it will be possible to get the 

788 # telescope latitude directly from the camera. 

789 latitude = pexConfig.Field( 

790 doc="Observatory latitude", 

791 dtype=float, 

792 default=None, 

793 ) 

794 mirrorArea = pexConfig.Field( 

795 doc="Mirror area (square meters) of telescope. If not set, will " 

796 "try to estimate from camera.telescopeDiameter.", 

797 dtype=float, 

798 default=None, 

799 optional=True, 

800 ) 

801 cameraGain = pexConfig.Field( 

802 doc="Average camera gain. If not set, will use the median of the " 

803 "camera model/detector/amplifier gains.", 

804 dtype=float, 

805 default=None, 

806 optional=True, 

807 ) 

808 defaultCameraOrientation = pexConfig.Field( 

809 doc="Default camera orientation for QA plots.", 

810 dtype=float, 

811 default=None, 

812 ) 

813 brightObsGrayMax = pexConfig.Field( 

814 doc="Maximum gray extinction to be considered bright observation", 

815 dtype=float, 

816 default=0.15, 

817 ) 

818 minStarPerCcd = pexConfig.Field( 

819 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

820 "CCDs with fewer stars will have their calibration estimated from other " 

821 "CCDs in the same visit, with zeropoint error increased accordingly."), 

822 dtype=int, 

823 default=5, 

824 ) 

825 minCcdPerExp = pexConfig.Field( 

826 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

827 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

828 dtype=int, 

829 default=5, 

830 ) 

831 maxCcdGrayErr = pexConfig.Field( 

832 doc="Maximum error on CCD gray offset to be considered photometric", 

833 dtype=float, 

834 default=0.05, 

835 ) 

836 minStarPerExp = pexConfig.Field( 

837 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

838 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

839 dtype=int, 

840 default=600, 

841 ) 

842 minExpPerNight = pexConfig.Field( 

843 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

844 dtype=int, 

845 default=10, 

846 ) 

847 expGrayInitialCut = pexConfig.Field( 

848 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

849 "observations."), 

850 dtype=float, 

851 default=-0.25, 

852 ) 

853 expFwhmCutDict = pexConfig.DictField( 

854 doc=("Per-band specification on maximum exposure FWHM (arcseconds) that will " 

855 "be considered for the model fit. Exposures with median FWHM larger " 

856 "than this threshold will get zeropoints based on matching to good " 

857 "stars."), 

858 keytype=str, 

859 itemtype=float, 

860 default={}, 

861 ) 

862 expGrayPhotometricCutDict = pexConfig.DictField( 

863 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

864 "('gray term') for a visit to be considered photometric. Must have one " 

865 "entry per band. Broad-band filters should be -0.05."), 

866 keytype=str, 

867 itemtype=float, 

868 default={}, 

869 ) 

870 expGrayHighCutDict = pexConfig.DictField( 

871 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

872 "('gray term') for a visit to be considered photometric. Must have one " 

873 "entry per band. Broad-band filters should be 0.2."), 

874 keytype=str, 

875 itemtype=float, 

876 default={}, 

877 ) 

878 expGrayRecoverCut = pexConfig.Field( 

879 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

880 "Visits with more gray extinction will only get CCD zeropoints if there are " 

881 "sufficient star observations (minStarPerCcd) on that CCD."), 

882 dtype=float, 

883 default=-1.0, 

884 ) 

885 expVarGrayPhotometricCutDict = pexConfig.DictField( 

886 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

887 "photometric. Must have one entry per band. Broad-band filters should be " 

888 "0.0005."), 

889 keytype=str, 

890 itemtype=float, 

891 default={}, 

892 ) 

893 expGrayErrRecoverCut = pexConfig.Field( 

894 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

895 "Visits with more gray variance will only get CCD zeropoints if there are " 

896 "sufficient star observations (minStarPerCcd) on that CCD."), 

897 dtype=float, 

898 default=0.05, 

899 ) 

900 aperCorrUsePsfFwhm = pexConfig.Field( 

901 doc="Use PSF FWHM for aperture corrections? Otherwise delta_aper statistic is used.", 

902 dtype=bool, 

903 default=False, 

904 ) 

905 aperCorrPerCcd = pexConfig.Field( 

906 doc="Use aperture corrections per-ccd (detector) instead of per-visit?", 

907 dtype=bool, 

908 default=False, 

909 ) 

910 aperCorrFitNBins = pexConfig.Field( 

911 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

912 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

913 "used if available."), 

914 dtype=int, 

915 default=10, 

916 ) 

917 aperCorrInputSlopeDict = pexConfig.DictField( 

918 doc=("Per-band specification of aperture correction input slope parameters. These " 

919 "are used on the first fit iteration, and aperture correction parameters will " 

920 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

921 "to set this when there is insufficient data to fit the parameters (e.g. " 

922 "tract mode)."),