Coverage for python/lsst/images/_observation_summary

1# This file is part of lsst-images.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# Use of this source code is governed by a 3-clause BSD-style

10# license that can be found in the LICENSE file.

11from __future__ import annotations

13__all__ = ("ObservationSummaryStats",)

15import dataclasses

16import math

17from typing import TYPE_CHECKING, Any, ClassVar, Self, final, get_origin

19import pydantic

21from lsst.images.describe import (

22 DescribableMixin,

23 DescribeOptions,

24 FieldRole,

25 Report,

26 ReportField,

27 ReportValueGroup,

28)

29from lsst.images.serialization import ArchiveTree, InputArchive, InvalidParameterError, OutputArchive

31if TYPE_CHECKING:

32 try:

33 from lsst.afw.image import ExposureSummaryStats as LegacyExposureSummaryStats

34 except ImportError:

35 type LegacyExposureSummaryStats = Any # type: ignore[no-redef]

38def _default_corners() -> tuple[float, float, float, float]:

39 return (math.nan, math.nan, math.nan, math.nan)

42_CORNER_NAMES = frozenset({"raCorners", "decCorners"})

43"""Statistics a report drops when ``"corners"`` is excluded, because a

44container is tabulating the same corners from its sky projection.

45"""

48def _is_empty(value: Any) -> bool:

49 """Return whether a summary-statistic value is unset.

51 A value counts as unset if it is NaN, or an empty sequence, or a sequence

52 whose entries are all unset. Such fields carry no information and can be

53 dropped when converting to or from the legacy representation, allowing the

54 two representations to define different sets of fields as long as the

55 fields they do not share are empty.

56 """

57 if isinstance(value, (list, tuple)):

58 return all(_is_empty(item) for item in value)

59 return isinstance(value, float) and math.isnan(value)

62@final

63class ObservationSummaryStats(ArchiveTree, DescribableMixin):

64 """Various statistics obtained from a single observation."""

66 SCHEMA_NAME: ClassVar[str] = "observation_summary_stats"

67 SCHEMA_VERSION: ClassVar[str] = "1.0.0"

68 MIN_READ_VERSION: ClassVar[int] = 1

69 PUBLIC_TYPE: ClassVar[type] # Assigned after class construction.

71 psfSigma: float = pydantic.Field(math.nan, description="PSF determinant radius (pixels).")

73 psfArea: float = pydantic.Field(math.nan, description="PSF effective area (pixels**2).")

75 psfIxx: float = pydantic.Field(math.nan, description="PSF shape Ixx (pixels**2).")

77 psfIyy: float = pydantic.Field(math.nan, description="PSF shape Iyy (pixels**2).")

79 psfIxy: float = pydantic.Field(math.nan, description="PSF shape Ixy (pixels**2).")

81 ra: float = pydantic.Field(math.nan, description="Bounding box center Right Ascension (degrees).")

83 dec: float = pydantic.Field(math.nan, description="Bounding box center Declination (degrees).")

85 pixelScale: float = pydantic.Field(math.nan, description="Measured detector pixel scale (arcsec/pixel).")

87 zenithDistance: float = pydantic.Field(

88 math.nan, description="Bounding box center zenith distance (degrees)."

89 )

91 expTime: float = pydantic.Field(math.nan, description="Exposure time of the exposure (seconds).")

93 zeroPoint: float = pydantic.Field(math.nan, description="Mean zeropoint in detector (mag).")

95 skyBg: float = pydantic.Field(math.nan, description="Average sky background (ADU).")

97 skyNoise: float = pydantic.Field(math.nan, description="Average sky noise (ADU).")

99 meanVar: float = pydantic.Field(math.nan, description="Mean variance of the weight plane (ADU**2).")

100

101 raCorners: tuple[float, float, float, float] = pydantic.Field(

102 default_factory=_default_corners, description="Right Ascension of bounding box corners (degrees)."

103 )

104

105 decCorners: tuple[float, float, float, float] = pydantic.Field(

106 default_factory=_default_corners, description="Declination of bounding box corners (degrees)."

107 )

108

109 psfAdaptiveThresholdValue: float = pydantic.Field(

110 math.nan,

111 description="Threshold value used in the adaptive threshold detection pass for PSF modelling.",

112 )

113

114 psfAdaptiveIncludeThresholdMultiplier: float = pydantic.Field(

115 math.nan,

116 description="Threshold multiplier used in the adaptive threshold detection pass for PSF modelling.",

117 )

118

119 nShapeletsStar: int = pydantic.Field(

120 0,

121 description="Number of sources used in the shapelet decomposition.",

122 )

123

124 shapeletsOnlyIqScore: float = pydantic.Field(

125 math.nan,

126 description=(

127 "The dimensionless image quality score as determined from the shapelets decomposition "

128 "that includes power only from the non-atmospheric decomposition coefficients. The "

129 "score spans the range [0.0, 1.0] with lower values indicating better image quality."

130 ),

131 )

132

133 shapeletsIqScore: float = pydantic.Field(

134 math.nan,

135 description=(

136 "The dimensionless image quality score as determined from the shapelets decomposition "

137 "that includes power from the median centroid offset between those used in the decomposition "

138 "and those of the centroid slot in addition to non-atmospheric decomposition coefficients. "

139 "The score spans the range [0.0, 1.0] with lower values indicating better image quality."

140 ),

141 )

142

143 shapeletsCoeffs: tuple[float, ...] = pydantic.Field(

144 default_factory=tuple,

145 description="Coefficients from the PSF star shapelet decomposition.",

146 )

147

148 centroidDiffShapeletsVsSlotMedian: float = pydantic.Field(

149 math.nan,

150 description=(

151 "Median centroid difference (sqrt((slot_x - shapelet_x)**2 + (slot_y - shapelet_y)**2)) for "

152 "sources used in the shapelet decomposition (pixels)."

153 ),

154 )

155

156 shapeletsStarEMedian: float = pydantic.Field(

157 math.nan,

158 description=(

159 "Median ellipticity (sqrt(starE1**2.0 + starE2**2.0)) of the sources used in the "

160 "shapelet decomposition."

161 ),

162 )

163

164 shapeletsStarUnNormalizedEMedian: float = pydantic.Field(

165 math.nan,

166 description=(

167 "Median un-normalized ellipticity (sqrt((starXX - starYY)**2.0 + (2.0*starXY)**2.0)) "

168 "of the sources used in the shapelet decomposition (pixel**2)."

169 ),

170 )

171

172 refCatSourceDensity: float = pydantic.Field(

173 math.nan,

174 description=(

175 "Source density for the detector region as computed from the loaded reference catalog "

176 "(number per degrees**2)."

177 ),

178 )

179

180 astromOffsetMean: float = pydantic.Field(math.nan, description="Astrometry match offset mean.")

181

182 astromOffsetStd: float = pydantic.Field(math.nan, description="Astrometry match offset stddev.")

183

184 nPsfStar: int = pydantic.Field(0, description="Number of stars used for psf model.")

185

186 psfStarDeltaE1Median: float = pydantic.Field(

187 math.nan, description="Psf stars median E1 residual (starE1 - psfE1)."

188 )

189

190 psfStarDeltaE2Median: float = pydantic.Field(

191 math.nan, description="Psf stars median E2 residual (starE2 - psfE2)."

192 )

193

194 psfStarDeltaE1Scatter: float = pydantic.Field(

195 math.nan, description="Psf stars MAD E1 scatter (starE1 - psfE1)."

196 )

197

198 psfStarDeltaE2Scatter: float = pydantic.Field(

199 math.nan, description="Psf stars MAD E2 scatter (starE2 - psfE2)."

200 )

201

202 psfStarDeltaSizeMedian: float = pydantic.Field(

203 math.nan, description="Psf stars median size residual (starSize - psfSize)."

204 )

205

206 psfStarDeltaSizeScatter: float = pydantic.Field(

207 math.nan, description="Psf stars MAD size scatter (starSize - psfSize)."

208 )

209

210 psfStarScaledDeltaSizeScatter: float = pydantic.Field(

211 math.nan, description="Psf stars MAD size scatter scaled by psfSize**2."

212 )

213

214 psfTraceRadiusDelta: float = pydantic.Field(

215 math.nan,

216 description=(

217 "Delta (max - min) of the model psf trace radius values evaluated on a grid of "

218 "unmasked pixels (pixels)."

219 ),

220 )

221

222 psfApFluxDelta: float = pydantic.Field(

223 math.nan,

224 description=(

225 "Delta (max - min) of the model psf aperture flux (with aperture radius of max(2, 3*psfSigma)) "

226 "values evaluated on a grid of unmasked pixels."

227 ),

228 )

229

230 psfApCorrSigmaScaledDelta: float = pydantic.Field(

231 math.nan,

232 description=(

233 "Delta (max - min) of the psf flux aperture correction factors scaled (divided) by the "

234 "psfSigma evaluated on a grid of unmasked pixels."

235 ),

236 )

237

238 maxDistToNearestPsf: float = pydantic.Field(

239 math.nan,

240 description="Maximum distance of an unmasked pixel to its nearest model psf star (pixels).",

241 )

242

243 starEMedian: float = pydantic.Field(

244 math.nan,

245 description=(

246 "Median ellipticity (sqrt(starE1**2.0 + starE2**2.0)) of the stars used in the PSF model."

247 ),

248 )

249

250 starUnNormalizedEMedian: float = pydantic.Field(

251 math.nan,

252 description=(

253 "Median un-normalized ellipticity (sqrt((starXX - starYY)**2.0 + "

254 "(2.0*starXY)**2.0)) of the stars used in the PSF model."

255 ),

256 )

257

258 starComa1Median: float = pydantic.Field(

259 math.nan,

260 description=(

261 "Coma-like higher-order moment combination: median M30 + M12 of the stars used in the PSF model."

262 ),

263 )

264

265 starComa2Median: float = pydantic.Field(

266 math.nan,

267 description=(

268 "Coma-like higher-order moment combination: median M21 + M03 of the stars used in the PSF model."

269 ),

270 )

271

272 starTrefoil1Median: float = pydantic.Field(

273 math.nan,

274 description=(

275 "Trefoil-like higher-order moment combination: median M30 - 3*M12 "

276 "of the stars used in the PSF model."

277 ),

278 )

279

280 starTrefoil2Median: float = pydantic.Field(

281 math.nan,

282 description=(

283 "Trefoil-like higher-order moment combination: median 3*M21 - M03 "

284 "of the stars used in the PSF model."

285 ),

286 )

287

288 starKurtosisMedian: float = pydantic.Field(

289 math.nan,

290 description=(

291 "Kurtosis-like higher-order moment combination: median M40 + 2*M22 + M04 "

292 "of the stars used in the PSF model."

293 ),

294 )

295

296 starE41Median: float = pydantic.Field(

297 math.nan,

298 description=(

299 "Fourth-order ellipticity-like higher-order moment combination: median M40 - M04 "

300 "of the stars used in the PSF model."

301 ),

302 )

303

304 starE42Median: float = pydantic.Field(

305 math.nan,

306 description=(

307 "Fourth-order ellipticity-like higher-order moment combination: median 2*(M31 + M13) "

308 "of the stars used in the PSF model."

309 ),

310 )

311

312 effTime: float = pydantic.Field(

313 math.nan,

314 description="Effective exposure time calculated from psfSigma, skyBg, and zeroPoint (seconds).",

315 )

316

317 effTimePsfSigmaScale: float = pydantic.Field(

318 math.nan, description="PSF scaling of the effective exposure time."

319 )

320

321 effTimeSkyBgScale: float = pydantic.Field(

322 math.nan, description="Sky background scaling of the effective exposure time."

323 )

324

325 effTimeZeroPointScale: float = pydantic.Field(

326 math.nan, description="Zeropoint scaling of the effective exposure time."

327 )

328

329 magLim: float = pydantic.Field(

330 math.nan,

331 description=(

332 "Magnitude limit at fixed SNR (default SNR=5) calculated from psfSigma, skyBg,"

333 " zeroPoint, and readNoise."

334 ),

335 )

336

337 psfTE1e1: float = pydantic.Field(

338 math.nan,

339 description=(

340 "Per-exposure TE1e1 ~ <de1 de1> of PSF residual ellipticity, averaged over theta "

341 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

342 "full-survey TE1 metric."

343 ),

344 )

345

346 psfTE1e2: float = pydantic.Field(

347 math.nan,

348 description=(

349 "Per-exposure TE1e2 ~ <de2 de2> of PSF residual ellipticity, averaged over theta "

350 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

351 "full-survey TE1 metric."

352 ),

353 )

354

355 psfTE1ex: float = pydantic.Field(

356 math.nan,

357 description=(

358 "Per-exposure TE1ex ~ <de1 de2> of PSF residual ellipticity, averaged over theta "

359 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

360 "full-survey TE1 metric."

361 ),

362 )

363

364 psfTE2e1: float = pydantic.Field(

365 math.nan,

366 description=(

367 "Per-exposure TE2e1 ~ <de1 de1> of PSF residual ellipticity, averaged over theta "

368 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

369 "full-survey TE2 metric."

370 ),

371 )

372

373 psfTE2e2: float = pydantic.Field(

374 math.nan,

375 description=(

376 "Per-exposure TE2e2 ~ <de2 de2> of PSF residual ellipticity, averaged over theta "

377 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

378 "full-survey TE2 metric."

379 ),

380 )

381

382 psfTE2ex: float = pydantic.Field(

383 math.nan,

384 description=(

385 "Per-exposure TE2ex ~ <de1 de2> of PSF residual ellipticity, averaged over theta "

386 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

387 "full-survey TE2 metric."

388 ),

389 )

390

391 psfTE3e1: float = pydantic.Field(

392 math.nan,

393 description=(

394 "Per-exposure median-over-CCDs of TE3e1 ~ <de1 de1> of PSF residual ellipticity, "

395 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

396 "pipelines take the 85th percentile over images to evaluate TE3."

397 ),

398 )

399

400 psfTE3e2: float = pydantic.Field(

401 math.nan,

402 description=(

403 "Per-exposure median-over-CCDs of TE3e2 ~ <de2 de2> of PSF residual ellipticity, "

404 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

405 "pipelines take the 85th percentile over images to evaluate TE3."

406 ),

407 )

408

409 psfTE3ex: float = pydantic.Field(

410 math.nan,

411 description=(

412 "Per-exposure median-over-CCDs of TE3ex ~ <de1 de2> of PSF residual ellipticity, "

413 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

414 "pipelines take the 85th percentile over images to evaluate TE3."

415 ),

416 )

417

418 psfTE4e1: float = pydantic.Field(

419 math.nan,

420 description=(

421 "Per-exposure median-over-CCDs of TE4e1 ~ <de1 de1> of PSF residual ellipticity, "

422 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

423 "pipelines take the 85th percentile over images to evaluate TE4."

424 ),

425 )

426

427 psfTE4e2: float = pydantic.Field(

428 math.nan,

429 description=(

430 "Per-exposure median-over-CCDs of TE4e2 ~ <de2 de2> of PSF residual ellipticity, "

431 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

432 "pipelines take the 85th percentile over images to evaluate TE4."

433 ),

434 )

435

436 psfTE4ex: float = pydantic.Field(

437 math.nan,

438 description=(

439 "Per-exposure median-over-CCDs of TE4ex ~ <de1 de2> of PSF residual ellipticity, "

440 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

441 "pipelines take the 85th percentile over images to evaluate TE4."

442 ),

443 )

444

445 def __eq__(self, other: object) -> bool:

446 if not isinstance(other, ObservationSummaryStats): 446 ↛ 447line 446 didn't jump to line 447 because the condition on line 446 was never true

447 return NotImplemented

448 for name in ObservationSummaryStats.model_fields:

449 if name in ArchiveTree.model_fields:

450 # Parent class fields, not summary statistics.

451 continue

452 a = getattr(self, name)

453 b = getattr(other, name)

454 if isinstance(a, tuple) and isinstance(b, tuple):

455 if len(a) != len(b): 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true

456 return False

457 for ai, bi in zip(a, b):

458 if ai != bi and not (math.isnan(ai) and math.isnan(bi)): 458 ↛ 459line 458 didn't jump to line 459 because the condition on line 458 was never true

459 return False

460 elif a != b and not (math.isnan(a) and math.isnan(b)):

461 return False

462 return True

463

464 def _describe(self, options: DescribeOptions = DescribeOptions(), /) -> Report:

465 """Return a `Report` describing these summary statistics.

466

467 Parameters

468 ----------

469 options : `DescribeOptions`, optional

470 Rendering options. `DescribeOptions.brief` reports how many

471 statistics are set instead of listing them, and ``"corners"`` in

472 `DescribeOptions.exclude` drops `raCorners` and `decCorners`.

473

474 Notes

475 -----

476 Which statistics an observation carries depends on which pipeline

477 steps produced it, and most of the many fields are unset in any given

478 one, so the report covers whichever of them hold a value rather than a

479 fixed selection.

480 """

481 # Everything this class adds to ArchiveTree is a statistic; what it

482 # inherits from it is serialization plumbing.

483 names = sorted(set(type(self).model_fields) - set(ArchiveTree.model_fields))

484 present = [(name, value) for name in names if not _is_empty(value := getattr(self, name))]

485 summary = f"ObservationSummaryStats({len(present)} of {len(names)} statistics set)"

486 if options.brief:

487 # The count is worth stating on its own where the values are not

488 # listed; alongside them it would only restate what is visible.

489 return Report(

490 type_name="ObservationSummaryStats",

491 summary=summary,

492 fields=[

493 ReportField(

494 label="statistics set",

495 value=f"{len(present)} of {len(names)}",

496 role=FieldRole.DERIVED,

497 )

498 ],

499 )

500 scalars: list[tuple[str, Any]] = []

501 fields: list[ReportField] = []

502 # The count in the summary stays over every statistic that is set,

503 # whatever the caller chose not to render.

504 hide_corners = "corners" in options.exclude

505 for name, value in present:

506 if hide_corners and name in _CORNER_NAMES:

507 continue

508 if isinstance(value, list | tuple):

509 # Too long to pack alongside the scalars; give it a line.

510 fields.append(ReportField(label=name, value=value, role=FieldRole.DERIVED))

511 else:

512 scalars.append((name, value))

513 return Report(

514 type_name="ObservationSummaryStats",

515 summary=summary,

516 fields=fields,

517 value_groups=[ReportValueGroup(values=scalars)] if scalars else [],

518 )

519

520 def __str__(self) -> str:

521 # pydantic's __str__ precedes the mixin's in the MRO and spells out all

522 # of the fields, nearly all of which are unset. ``repr`` keeps that

523 # exhaustive form, since it is the one that round-trips; ``str`` is the

524 # readable one, so it takes the report's summary.

525 return DescribableMixin.__str__(self)

526

527 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Self:

528 """Extract this object from an archive.

529

530 Parameters

531 ----------

532 archive

533 Archive to read from.

534 **kwargs

535 Optional parameters. Not supported by this class.

536 """

537 if kwargs: 537 ↛ 538line 537 didn't jump to line 538 because the condition on line 537 was never true

538 raise InvalidParameterError(

539 f"Unrecognized parameters for ObservationSummaryStats: {set(kwargs.keys())}."

540 )

541 # The model we want *is* the ArchiveTree.

542 return self

543

544 def serialize(self, archive: OutputArchive[Any]) -> Self:

545 """Write this object to an archive.

546

547 Parameters

548 ----------

549 archive

550 Archive to write to.

551 """

552 # Copy so write-time overrides (metadata, butler_info) applied

553 # by serialize_root do not mutate this object.

554 return self.model_copy(deep=True)

555

556 @classmethod

557 def from_legacy(cls, exposure_summary_stats: LegacyExposureSummaryStats) -> Self:

558 """Return an `ObservationSummaryStats` from a legacy

559 `lsst.afw.image.ExposureSummaryStats`.

560

561 Parameters

562 ----------

563 exposure_summary_stats

564 Legacy exposure summary statistics to convert.

565

566 Notes

567 -----

568 Legacy fields that are empty (NaN) are dropped, so a legacy struct that

569 carries fields unknown to this class is accepted as long as those

570 fields are empty. A legacy field that holds a real value but is

571 unknown here raises `ValueError`, since dropping it would lose data.

572 """

573 known_fields = set(cls.model_fields)

574 kwargs: dict[str, Any] = {}

575 for name, value in dataclasses.asdict(exposure_summary_stats).items():

576 if _is_empty(value):

577 continue

578 # Strip version since it carries no information (it is always 0

579 # in all our existing files) and this class uses explicit schema

580 # versioning.

581 if name == "version":

582 continue

583 if name not in known_fields:

584 raise ValueError(

585 f"Legacy field {name!r} has a value ({value!r}) but is not known to "

586 f"ObservationSummaryStats."

587 )

588 kwargs[name] = value

589 return cls.model_validate(kwargs)

590

591 def to_legacy(self) -> LegacyExposureSummaryStats:

592 """Convert to an `lsst.afw.image.ExposureSummaryStats` instance.

593

594 Notes

595 -----

596 Empty (NaN) fields are not passed to the legacy struct, so fields

597 defined here that are unknown to the installed version of

598 `~lsst.afw.image.ExposureSummaryStats` are dropped when empty. A field

599 that holds a real value but is unknown to the legacy struct raises

600 `ValueError`, since dropping it would lose data.

601 """

602 from lsst.afw.image import ExposureSummaryStats as LegacyExposureSummaryStats

603

604 legacy_fields = {field.name for field in dataclasses.fields(LegacyExposureSummaryStats)}

605 kwargs: dict[str, Any] = {}

606 for name, info in ObservationSummaryStats.model_fields.items():

607 if name in ArchiveTree.model_fields:

608 # Parent class fields, not summary statistics.

609 continue

610 value = getattr(self, name)

611 if _is_empty(value):

612 continue

613 if name not in legacy_fields: 613 ↛ 614line 613 didn't jump to line 614 because the condition on line 613 was never true

614 raise ValueError(

615 f"Field {name!r} has a value ({value!r}) but is not supported by this "

616 f"version of lsst.afw.image.ExposureSummaryStats."

617 )

618 # Doing this in general is hard, so we handle the fields that we

619 # know about and raise if somebody adds a field with a new type

620 # without updating this function.

621 if info.annotation in (float, int): 621 ↛ 623line 621 didn't jump to line 623 because the condition on line 621 was always true

622 kwargs[name] = value

623 elif get_origin(info.annotation) is tuple:

624 kwargs[name] = list(value)

625 else:

626 raise NotImplementedError(f"Unsupported field type: {info.annotation}.")

627 return LegacyExposureSummaryStats(**kwargs)

628

629

630# Can not assign to itself in construction so assign now.

631ObservationSummaryStats.PUBLIC_TYPE = ObservationSummaryStats

Coverage for python/lsst/images/_observation_summary_stats.py: 49%

160 statements