Coverage for python/lsst/images/serialization/_common.py: 56%
103 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-06-09 02:02 -0700
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "ArchiveReadError",
16 "ArchiveTree",
17 "ButlerInfo",
18 "InvalidComponentError",
19 "InvalidParameterError",
20 "JsonRef",
21 "MetadataValue",
22 "OpaqueArchiveMetadata",
23 "no_header_updates",
24)
26import operator
27from abc import ABC, abstractmethod
28from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
30import astropy.table
31import astropy.units
32import pydantic
34from .._geom import Box
35from ..utils import is_none
37try:
38 from lsst.daf.butler import DatasetProvenance, SerializedDatasetRef
39except ImportError:
40 type DatasetProvenance = Any # type: ignore[no-redef]
41 type SerializedDatasetRef = Any # type: ignore[no-redef]
43if TYPE_CHECKING:
44 import astropy.io.fits
46 from ._input_archive import InputArchive
49type MetadataValue = (
50 pydantic.StrictInt | pydantic.StrictFloat | pydantic.StrictStr | pydantic.StrictBool | None
51)
53SCHEMA_URL_HOST = "images.lsst.io"
54"""Canonical hostname for lsst.images schema URLs."""
56SCHEMA_URL_BASE = f"https://{SCHEMA_URL_HOST}/schemas"
57"""Base for schema URLs, used as ``{SCHEMA_URL_BASE}/{name}-{version}``."""
60class ButlerInfo(pydantic.BaseModel):
61 """Information about a butler dataset."""
63 dataset: SerializedDatasetRef
64 provenance: DatasetProvenance = pydantic.Field(default_factory=DatasetProvenance)
67class JsonRef(pydantic.BaseModel, serialize_by_alias=True):
68 """Pydantic model for JSON Reference / Pointer (IETF RFC 6901).
70 Notes
71 -----
72 This model does not do any of the escaping or special-character
73 interpretation required by the spec; it assumes that's already been done,
74 so its job is *just* putting a ``$ref`` field inside another model.
75 """
77 ref: str = pydantic.Field(alias="$ref")
80class ArchiveTree(
81 pydantic.BaseModel, ABC, ser_json_inf_nan="constants", ser_json_bytes="base64", val_json_bytes="base64"
82):
83 """An intermediate base class of `pydantic.BaseModel` that should be used
84 for all objects that may be used as the top-level tree models written to
85 archives.
87 See :ref:`lsst.images-schema-versioning` for how the ``SCHEMA_NAME`` /
88 ``SCHEMA_VERSION`` / ``MIN_READ_VERSION`` constants and the
89 ``schema_version`` / ``min_read_version`` / ``schema_url`` fields are used.
90 """
92 SCHEMA_NAME: ClassVar[str]
93 SCHEMA_VERSION: ClassVar[str]
94 MIN_READ_VERSION: ClassVar[int]
95 PUBLIC_TYPE: ClassVar[type]
96 """In-memory Python type produced by this tree's ``deserialize`` (e.g.
97 `dict` for a mapping return). Declared explicitly by each concrete
98 subclass and surfaced by
99 `~lsst.images.serialization.public_type_for_schema`."""
101 schema_version: str = pydantic.Field(
102 default="1.0.0",
103 description="Data-model schema version of this tree (major.minor.patch).",
104 )
105 min_read_version: int = pydantic.Field(
106 default=1,
107 description="Smallest reader major that can interpret this tree.",
108 )
109 metadata: dict[str, MetadataValue] = pydantic.Field(
110 default_factory=dict, description="Additional unstructured metadata.", exclude_if=operator.not_
111 )
112 butler_info: ButlerInfo | None = pydantic.Field(
113 default=None,
114 description="Information about the butler dataset backed by this file.",
115 exclude_if=is_none,
116 )
117 indirect: list[Any] = pydantic.Field(
118 default_factory=list,
119 description="Serialized nested objects that may be saved or read more than once.",
120 exclude_if=operator.not_,
121 )
123 @pydantic.computed_field(description="Canonical schema URL for this tree.") # type: ignore[prop-decorator]
124 @property
125 def schema_url(self) -> str:
126 """Return the schema URL of this tree's class.
128 Computed from ``SCHEMA_NAME`` and ``SCHEMA_VERSION`` ClassVars.
129 """
130 cls = type(self)
131 return f"{SCHEMA_URL_BASE}/{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION}"
133 @pydantic.model_validator(mode="after")
134 def _check_and_normalize_schema_version(self) -> Self:
135 """Validate and normalise the schema version fields.
137 Compares the on-tree ``schema_version`` / ``min_read_version`` against
138 the in-code values from the subclass's ClassVars; raises if
139 incompatible, otherwise normalises the fields to the in-code values.
140 """
141 cls = type(self)
142 # ArchiveTree itself is abstract (deserialize is @abstractmethod).
143 # Subclasses that haven't yet declared SCHEMA_NAME are skipped — this
144 # matters during incremental rollout and remains a safe no-op
145 # afterwards (a class-invariants test ensures every concrete subclass
146 # has the constants).
147 if not hasattr(cls, "SCHEMA_NAME"):
148 return self
149 _check_compat(
150 cls.SCHEMA_NAME,
151 self.schema_version,
152 self.min_read_version,
153 cls.SCHEMA_VERSION,
154 )
155 if self.schema_version != cls.SCHEMA_VERSION:
156 self.schema_version = cls.SCHEMA_VERSION
157 if self.min_read_version != cls.MIN_READ_VERSION:
158 self.min_read_version = cls.MIN_READ_VERSION
159 return self
161 @classmethod
162 def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
163 """Inject ``$id`` and ``title`` into the subclass's JSON Schema, and
164 register the subclass in the schema-name registry.
166 Populates ``model_config['json_schema_extra']`` with values derived
167 from the subclass's ``SCHEMA_NAME`` / ``SCHEMA_VERSION`` ClassVars,
168 then registers the subclass so it can be looked up by schema name.
169 Subclasses that haven't declared the ClassVars are skipped.
170 """
171 super().__pydantic_init_subclass__(**kwargs)
172 name = cls.__dict__.get("SCHEMA_NAME")
173 version = cls.__dict__.get("SCHEMA_VERSION")
174 if name is None or version is None:
175 return
176 json_schema_extra = cls.model_config.get("json_schema_extra") or {}
177 if isinstance(json_schema_extra, dict): 177 ↛ 184line 177 didn't jump to line 184 because the condition on line 177 was always true
178 existing = dict(json_schema_extra)
179 existing.setdefault("$id", f"{SCHEMA_URL_BASE}/{name}-{version}")
180 existing.setdefault("title", name)
181 cls.model_config = {**cls.model_config, "json_schema_extra": existing}
182 # Local import to avoid the _io -> _common circular dependency at
183 # module load time.
184 from ._io import register_schema_class
186 register_schema_class(cls)
188 @abstractmethod
189 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Any:
190 """Return the in-memory object that was serialized to this tree.
192 Parameters
193 ----------
194 archive
195 The input archive to read from.
196 **kwargs
197 Additional keyword arguments specific to this type.
199 Raises
200 ------
201 ~lsst.images.serialization.InvalidParameterError
202 Raised for unsupported ``**kwargs``.
204 Notes
205 -----
206 Subclass implementations may take additional keyword-only arguments.
207 Callers that invoke this method without knowing what those might be
208 should catch `TypeError` and re-raise as
209 `~lsst.images.serialization.InvalidParameterError` if they pass
210 additional keyword arguments.
211 """
212 raise NotImplementedError()
214 def deserialize_component(self, component: str, archive: InputArchive[Any], **kwargs: Any) -> Any:
215 """Return a component in-memory object that was serialized to this
216 tree.
218 Parameters
219 ----------
220 component
221 Name of the component to read.
222 archive
223 The input archive to read from.
224 **kwargs
225 Additional keyword arguments specific to this type.
227 Raises
228 ------
229 ~lsst.images.serialization.InvalidComponentError
230 Raise if ``component`` is not recognized.
231 ~lsst.images.serialization.InvalidParameterError
232 Raised for unsupported ``**kwargs``.
234 Notes
235 -----
236 The default implementation for this method tries to get an attribute
237 with the component's name from ``self``, and then:
239 - returns `None` if it is `None`;
240 - calls `deserialize` on that object if it is also an
241 `~lsst.images.serialization.ArchiveTree`;
242 - returns it directly otherwise.
244 If there is no such attribute, it raises
245 `~lsst.images.serialization.InvalidComponentError`.
247 ``**kwargs`` are forwarded to component `deserialize` methods, but
248 are otherwise not checked. Subclasses are generally expected to
249 implement this method to do that checking and handle any components
250 for which the other will not work, and then delegate to `super` at
251 the end.
252 """
253 try:
254 component_model = getattr(self, component)
255 except AttributeError:
256 raise InvalidComponentError(
257 f"Component {component!r} is not recognized by {type(self).__name__}."
258 ) from None
259 if component_model is None:
260 return None
261 if isinstance(component_model, ArchiveTree):
262 return component_model.deserialize(archive, **kwargs)
263 return component_model
266class ArchiveReadError(RuntimeError):
267 """Exception raised when the contents of an archive cannot be read."""
270class InvalidParameterError(ArchiveReadError):
271 """Exception raised by `ArchiveTree.deserialize` or
272 `ArchiveTree.deserialize_component` when passed an invalid keyword
273 argument.
274 """
277class InvalidComponentError(ArchiveReadError):
278 """Exception `ArchiveTree.deserialize_component` when passed an invalid
279 component name.
280 """
283class OpaqueArchiveMetadata(Protocol):
284 """Interface for opaque archive metadata.
286 In addition to implementing the methods defined here, all implementations
287 must be pickleable.
288 """
290 def copy(self) -> Self | None:
291 """Copy, reference, or discard metadata when its holding object is
292 copied.
293 """
294 ...
296 def subset(self, bbox: Box) -> Self | None:
297 """Copy, reference, or discard metadata when a subset of its its
298 holding object is extracted.
299 """
300 ...
303def no_header_updates(header: astropy.io.fits.Header) -> None:
304 """Do not make any modifications to the given FITS header."""
307def _parse_major(version: str) -> int:
308 """Return the integer major component of a major.minor.patch string.
310 Raises
311 ------
312 ArchiveReadError
313 If ``version`` is not a non-empty string of the form
314 ``major.minor.patch`` with integer components.
315 """
316 if not isinstance(version, str) or not version:
317 raise ArchiveReadError(f"Schema version {version!r} is not a non-empty string.")
318 head = version.split(".", 1)[0]
319 try:
320 return int(head)
321 except ValueError as exc:
322 raise ArchiveReadError(f"Schema version {version!r} has non-integer major.") from exc
325def _check_compat(
326 name: str,
327 on_disk_version: str,
328 on_disk_min_read: int,
329 in_code_version: str,
330) -> None:
331 """Raise `ArchiveReadError` if a tree written with the given
332 schema_version/min_read_version cannot be read by the current code.
334 See :ref:`lsst.images-schema-versioning` for the compatibility rule.
335 """
336 in_code_major = _parse_major(in_code_version)
337 if on_disk_min_read > in_code_major:
338 raise ArchiveReadError(
339 f"{name}: tree requires reader major >= {on_disk_min_read}; this release is {in_code_version}."
340 )
343def _check_format_version(name: str, on_disk: int, in_code: int) -> None:
344 """Raise `ArchiveReadError` if a backend file's container layout
345 version is newer than this release knows how to read.
346 """
347 if on_disk > in_code:
348 raise ArchiveReadError(
349 f"{name}: on-disk container format version {on_disk} is "
350 f"newer than this release ({in_code}); cannot read."
351 )