Coverage for python/lsst/images/serialization/_common.py: 89%
104 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-06-24 01:50 -0700
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "ArchiveAccessRequiredError",
16 "ArchiveReadError",
17 "ArchiveTree",
18 "ButlerInfo",
19 "InvalidComponentError",
20 "InvalidParameterError",
21 "JsonRef",
22 "MetadataValue",
23 "OpaqueArchiveMetadata",
24 "no_header_updates",
25)
27import operator
28from abc import ABC, abstractmethod
29from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
31import astropy.table
32import astropy.units
33import pydantic
35from .._geom import Box
36from ..utils import is_none
38try:
39 from lsst.daf.butler import DatasetProvenance, SerializedDatasetRef
40except ImportError:
41 type DatasetProvenance = Any # type: ignore[no-redef]
42 type SerializedDatasetRef = Any # type: ignore[no-redef]
44if TYPE_CHECKING:
45 import astropy.io.fits
47 from ._input_archive import InputArchive
50type MetadataValue = (
51 pydantic.StrictInt | pydantic.StrictFloat | pydantic.StrictStr | pydantic.StrictBool | None
52)
54SCHEMA_URL_HOST = "images.lsst.io"
55"""Canonical hostname for lsst.images schema URLs."""
57SCHEMA_URL_BASE = f"https://{SCHEMA_URL_HOST}/schemas"
58"""Base for schema URLs, used as ``{SCHEMA_URL_BASE}/{name}-{version}``."""
61class ButlerInfo(pydantic.BaseModel):
62 """Information about a butler dataset."""
64 dataset: SerializedDatasetRef
65 provenance: DatasetProvenance = pydantic.Field(default_factory=DatasetProvenance)
68class JsonRef(pydantic.BaseModel, serialize_by_alias=True):
69 """Pydantic model for JSON Reference / Pointer (IETF RFC 6901).
71 Notes
72 -----
73 This model does not do any of the escaping or special-character
74 interpretation required by the spec; it assumes that's already been done,
75 so its job is *just* putting a ``$ref`` field inside another model.
76 """
78 ref: str = pydantic.Field(alias="$ref")
81class ArchiveTree(
82 pydantic.BaseModel, ABC, ser_json_inf_nan="constants", ser_json_bytes="base64", val_json_bytes="base64"
83):
84 """An intermediate base class of `pydantic.BaseModel` that should be used
85 for all objects that may be used as the top-level tree models written to
86 archives.
88 See :ref:`lsst.images-schema-versioning` for how the ``SCHEMA_NAME`` /
89 ``SCHEMA_VERSION`` / ``MIN_READ_VERSION`` constants and the
90 ``schema_version`` / ``min_read_version`` / ``schema_url`` fields are used.
91 """
93 SCHEMA_NAME: ClassVar[str]
94 SCHEMA_VERSION: ClassVar[str]
95 MIN_READ_VERSION: ClassVar[int]
96 PUBLIC_TYPE: ClassVar[type]
97 """In-memory Python type produced by this tree's ``deserialize`` (e.g.
98 `dict` for a mapping return). Declared explicitly by each concrete
99 subclass and surfaced by
100 `~lsst.images.serialization.public_type_for_schema`."""
102 schema_version: str = pydantic.Field(
103 default="1.0.0",
104 description="Data-model schema version of this tree (major.minor.patch).",
105 )
106 min_read_version: int = pydantic.Field(
107 default=1,
108 description="Smallest reader major that can interpret this tree.",
109 )
110 metadata: dict[str, MetadataValue] = pydantic.Field(
111 default_factory=dict, description="Additional unstructured metadata.", exclude_if=operator.not_
112 )
113 butler_info: ButlerInfo | None = pydantic.Field(
114 default=None,
115 description="Information about the butler dataset backed by this file.",
116 exclude_if=is_none,
117 )
118 indirect: list[Any] = pydantic.Field(
119 default_factory=list,
120 description="Serialized nested objects that may be saved or read more than once.",
121 exclude_if=operator.not_,
122 )
124 @pydantic.computed_field(description="Canonical schema URL for this tree.") # type: ignore[prop-decorator]
125 @property
126 def schema_url(self) -> str:
127 """Return the schema URL of this tree's class.
129 Computed from ``SCHEMA_NAME`` and ``SCHEMA_VERSION`` ClassVars.
130 """
131 cls = type(self)
132 return f"{SCHEMA_URL_BASE}/{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION}"
134 @pydantic.model_validator(mode="after")
135 def _check_and_normalize_schema_version(self) -> Self:
136 """Validate and normalise the schema version fields.
138 Compares the on-tree ``schema_version`` / ``min_read_version`` against
139 the in-code values from the subclass's ClassVars; raises if
140 incompatible, otherwise normalises the fields to the in-code values.
141 """
142 cls = type(self)
143 # ArchiveTree itself is abstract (deserialize is @abstractmethod).
144 # Subclasses that haven't yet declared SCHEMA_NAME are skipped — this
145 # matters during incremental rollout and remains a safe no-op
146 # afterwards (a class-invariants test ensures every concrete subclass
147 # has the constants).
148 if not hasattr(cls, "SCHEMA_NAME"): 148 ↛ 149line 148 didn't jump to line 149 because the condition on line 148 was never true
149 return self
150 _check_compat(
151 cls.SCHEMA_NAME,
152 self.schema_version,
153 self.min_read_version,
154 cls.SCHEMA_VERSION,
155 )
156 if self.schema_version != cls.SCHEMA_VERSION:
157 self.schema_version = cls.SCHEMA_VERSION
158 if self.min_read_version != cls.MIN_READ_VERSION: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 self.min_read_version = cls.MIN_READ_VERSION
160 return self
162 @classmethod
163 def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
164 """Inject ``$id`` and ``title`` into the subclass's JSON Schema, and
165 register the subclass in the schema-name registry.
167 Populates ``model_config['json_schema_extra']`` with values derived
168 from the subclass's ``SCHEMA_NAME`` / ``SCHEMA_VERSION`` ClassVars,
169 then registers the subclass so it can be looked up by schema name.
170 Subclasses that haven't declared the ClassVars are skipped.
171 """
172 super().__pydantic_init_subclass__(**kwargs)
173 name = cls.__dict__.get("SCHEMA_NAME")
174 version = cls.__dict__.get("SCHEMA_VERSION")
175 if name is None or version is None:
176 return
177 json_schema_extra = cls.model_config.get("json_schema_extra") or {}
178 if isinstance(json_schema_extra, dict): 178 ↛ 185line 178 didn't jump to line 185 because the condition on line 178 was always true
179 existing = dict(json_schema_extra)
180 existing.setdefault("$id", f"{SCHEMA_URL_BASE}/{name}-{version}")
181 existing.setdefault("title", name)
182 cls.model_config = {**cls.model_config, "json_schema_extra": existing}
183 # Local import to avoid the _io -> _common circular dependency at
184 # module load time.
185 from ._io import register_schema_class
187 register_schema_class(cls)
189 @abstractmethod
190 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Any:
191 """Return the in-memory object that was serialized to this tree.
193 Parameters
194 ----------
195 archive
196 The input archive to read from.
197 **kwargs
198 Additional keyword arguments specific to this type.
200 Raises
201 ------
202 ~lsst.images.serialization.InvalidParameterError
203 Raised for unsupported ``**kwargs``.
205 Notes
206 -----
207 Subclass implementations may take additional keyword-only arguments.
208 Callers that invoke this method without knowing what those might be
209 should catch `TypeError` and re-raise as
210 `~lsst.images.serialization.InvalidParameterError` if they pass
211 additional keyword arguments.
212 """
213 raise NotImplementedError()
215 def deserialize_component(self, component: str, archive: InputArchive[Any], **kwargs: Any) -> Any:
216 """Return a component in-memory object that was serialized to this
217 tree.
219 Parameters
220 ----------
221 component
222 Name of the component to read.
223 archive
224 The input archive to read from.
225 **kwargs
226 Additional keyword arguments specific to this type.
228 Raises
229 ------
230 ~lsst.images.serialization.InvalidComponentError
231 Raise if ``component`` is not recognized.
232 ~lsst.images.serialization.InvalidParameterError
233 Raised for unsupported ``**kwargs``.
235 Notes
236 -----
237 The default implementation for this method tries to get an attribute
238 with the component's name from ``self``, and then:
240 - returns `None` if it is `None`;
241 - calls `deserialize` on that object if it is also an
242 `~lsst.images.serialization.ArchiveTree`;
243 - returns it directly otherwise.
245 If there is no such attribute, it raises
246 `~lsst.images.serialization.InvalidComponentError`.
248 ``**kwargs`` are forwarded to component `deserialize` methods, but
249 are otherwise not checked. Subclasses are generally expected to
250 implement this method to do that checking and handle any components
251 for which the other will not work, and then delegate to `super` at
252 the end.
253 """
254 try:
255 component_model = getattr(self, component)
256 except AttributeError:
257 raise InvalidComponentError(
258 f"Component {component!r} is not recognized by {type(self).__name__}."
259 ) from None
260 if component_model is None:
261 return None
262 if isinstance(component_model, ArchiveTree):
263 return component_model.deserialize(archive, **kwargs)
264 return component_model
267class ArchiveReadError(RuntimeError):
268 """Exception raised when the contents of an archive cannot be read."""
271class InvalidParameterError(ArchiveReadError):
272 """Exception raised by `ArchiveTree.deserialize` or
273 `ArchiveTree.deserialize_component` when passed an invalid keyword
274 argument.
275 """
278class InvalidComponentError(ArchiveReadError):
279 """Exception `ArchiveTree.deserialize_component` when passed an invalid
280 component name.
281 """
284class ArchiveAccessRequiredError(RuntimeError):
285 """Exception raised when a deserialization needs data from the file.
287 Raised by all data-access methods of
288 `~lsst.images.serialization.DetachedArchive`, signaling that the
289 requested object cannot be deserialized from the JSON tree alone.
291 Notes
292 -----
293 This deliberately does not inherit from `ArchiveReadError`: it signals
294 that a live archive is required rather than that an archive is corrupt,
295 and it must not be swallowed by ``except ArchiveReadError`` handlers.
296 """
299class OpaqueArchiveMetadata(Protocol):
300 """Interface for opaque archive metadata.
302 In addition to implementing the methods defined here, all implementations
303 must be pickleable.
304 """
306 def copy(self) -> Self | None:
307 """Copy, reference, or discard metadata when its holding object is
308 copied.
309 """
310 ...
312 def subset(self, bbox: Box) -> Self | None:
313 """Copy, reference, or discard metadata when a subset of its its
314 holding object is extracted.
315 """
316 ...
319def no_header_updates(header: astropy.io.fits.Header) -> None:
320 """Do not make any modifications to the given FITS header."""
323def _parse_major(version: str) -> int:
324 """Return the integer major component of a major.minor.patch string.
326 Raises
327 ------
328 ArchiveReadError
329 If ``version`` is not a non-empty string of the form
330 ``major.minor.patch`` with integer components.
331 """
332 if not isinstance(version, str) or not version: 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true
333 raise ArchiveReadError(f"Schema version {version!r} is not a non-empty string.")
334 head = version.split(".", 1)[0]
335 try:
336 return int(head)
337 except ValueError as exc:
338 raise ArchiveReadError(f"Schema version {version!r} has non-integer major.") from exc
341def _check_compat(
342 name: str,
343 on_disk_version: str,
344 on_disk_min_read: int,
345 in_code_version: str,
346) -> None:
347 """Raise `ArchiveReadError` if a tree written with the given
348 schema_version/min_read_version cannot be read by the current code.
350 See :ref:`lsst.images-schema-versioning` for the compatibility rule.
351 """
352 in_code_major = _parse_major(in_code_version)
353 if on_disk_min_read > in_code_major:
354 raise ArchiveReadError(
355 f"{name}: tree requires reader major >= {on_disk_min_read}; this release is {in_code_version}."
356 )
359def _check_format_version(name: str, on_disk: int, in_code: int) -> None:
360 """Raise `ArchiveReadError` if a backend file's container layout
361 version is newer than this release knows how to read.
362 """
363 if on_disk > in_code:
364 raise ArchiveReadError(
365 f"{name}: on-disk container format version {on_disk} is "
366 f"newer than this release ({in_code}); cannot read."
367 )