Coverage for python/lsst/images/serialization/_input_archive.py: 100%
64 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-06-24 08:46 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = ("ArchiveInfo", "DetachedArchive", "InputArchive")
16from abc import ABC, abstractmethod
17from collections.abc import Callable
18from contextlib import AbstractContextManager
19from types import EllipsisType
20from typing import TYPE_CHECKING, Any, TypeVar
22import astropy.io.fits
23import astropy.table
24import astropy.units
25import numpy as np
26import pydantic
28from lsst.resources import ResourcePath, ResourcePathExpression
30from ._asdf_utils import ArrayReferenceModel, InlineArrayModel
31from ._common import (
32 SCHEMA_URL_HOST,
33 ArchiveAccessRequiredError,
34 ArchiveTree,
35 OpaqueArchiveMetadata,
36 no_header_updates,
37)
38from ._tables import TableModel
40if TYPE_CHECKING:
41 from .._transforms import FrameSet
44# This pre-python-3.12 declaration is needed by Sphinx (probably the
45# autodoc-typehints plugin.
46P = TypeVar("P", bound=pydantic.BaseModel)
49class ArchiveInfo(pydantic.BaseModel, frozen=True):
50 """Basic identifying information about an on-disk archive.
52 Read from a file's headers/metadata without deserializing pixel data.
53 """
55 schema_url: str
56 """Canonical schema URL of the top-level tree."""
58 schema_name: str
59 """Schema name parsed from ``schema_url``."""
61 schema_version: str
62 """Schema version parsed from ``schema_url``."""
64 format_version: int | None
65 """Container layout version (FITS ``FMTVER`` / NDF ``FORMAT_VERSION``);
66 `None` for formats with no separate container version (JSON)."""
68 @classmethod
69 def from_schema_url(cls, schema_url: str, *, format_version: int | None) -> ArchiveInfo:
70 """Build an `ArchiveInfo` by parsing a schema URL of the form
71 ``https://images.lsst.io/schemas/{name}-{version}``.
73 The URL is parsed with `~lsst.resources.ResourcePath` and its
74 hostname must be ``images.lsst.io``, so a ``DATAMODL`` header written
75 by an unrelated tool cannot steer reads toward an arbitrary schema.
76 """
77 parsed = ResourcePath(schema_url)
78 if parsed.netloc != SCHEMA_URL_HOST:
79 raise ValueError(
80 f"Schema URL {schema_url!r} is not hosted at {SCHEMA_URL_HOST!r}; "
81 "this file was not written by lsst.images."
82 )
83 tail = parsed.basename()
84 # Split on the last hyphen: schema names may contain hyphens; the
85 # version (after the final hyphen) is assumed not to.
86 name, _, version = tail.rpartition("-")
87 if not name or not version:
88 raise ValueError(f"Cannot parse schema name/version from URL {schema_url!r}.")
89 return cls(
90 schema_url=schema_url,
91 schema_name=name,
92 schema_version=version,
93 format_version=format_version,
94 )
97class InputArchive[P: pydantic.BaseModel](ABC):
98 """Abstract interface for reading from a file format.
100 Notes
101 -----
102 An input archive instance is assumed to be paired with a Pydantic model
103 that represents a JSON tree, with the archive used to deserialize data that
104 is not native JSON from data that is (which may just be a reference to
105 binary data stored elsewhere in the file). The archive doesn't actually
106 hold that model instance because we'd prefer to avoid making the input
107 archive generic over the model type. It is expected that most concrete
108 archive implementations will provide a method to load the paired model from
109 a file, but this is not part of the base class interface.
110 """
112 @classmethod
113 def get_basic_info(cls, path: ResourcePathExpression) -> ArchiveInfo:
114 """Return basic identifying information for the archive at ``path``
115 without deserializing pixel data.
117 Each concrete backend reads only the headers/metadata it needs.
118 """
119 raise NotImplementedError(f"{cls.__name__} does not implement get_basic_info.")
121 @classmethod
122 def open_tree(
123 cls,
124 path: ResourcePathExpression,
125 *,
126 partial: bool = True,
127 **backend_kwargs: Any,
128 ) -> AbstractContextManager[tuple[InputArchive[P], ArchiveTree, ArchiveInfo]]:
129 """Open ``path``, load and validate its top-level tree, and yield
130 ``(archive, tree, info)`` as a context manager.
132 Parameters
133 ----------
134 path
135 File to be opened. Can be local or remote.
136 partial
137 Whether the file should be opened for incremental reads or not.
138 Can be ignored by a backend where not relevant.
139 **backend_kwargs
140 Any keyword parameters that should be forwarded to the backend
141 open.
143 Raises
144 ------
145 ArchiveReadError
146 If the file's schema is not registered.
148 Notes
149 -----
150 Each concrete backend implements this.
151 """
152 raise NotImplementedError(f"{cls.__name__} does not implement open_tree.")
154 @abstractmethod
155 def deserialize_pointer[U: ArchiveTree, V](
156 self, pointer: P, model_type: type[U], deserializer: Callable[[U, InputArchive[P]], V]
157 ) -> V:
158 """Deserialize an object that was saved by
159 `~lsst.serialization.OutputArchive.serialize_pointer`.
161 Parameters
162 ----------
163 pointer
164 JSON Pointer model to dereference.
165 model_type
166 Pydantic model type that the pointer should dereference to.
167 deserializer
168 Callable that takes an instance of ``model_type`` and an input
169 archive, and returns the deserialized object.
171 Returns
172 -------
173 V
174 The deserialized object.
176 Notes
177 -----
178 Implementations are required to remember previously-deserialized
179 objects and return them when the same pointer is passed in multiple
180 times.
182 There is no ``deserialize_direct`` (to pair with
183 `~lsst.serialization.OutputArchive.serialize_direct`) because the
184 caller can just call a deserializer function directly on a sub-model
185 of its Pydantic tree.
186 """
187 raise NotImplementedError()
189 @abstractmethod
190 def get_frame_set(self, ref: P) -> FrameSet:
191 """Return an already-deserialized frame set from the archive.
193 Parameters
194 ----------
195 ref
196 Implementation-specific reference to the frame set.
198 Returns
199 -------
200 FrameSet
201 Loaded frame set.
202 """
203 raise NotImplementedError()
205 @abstractmethod
206 def get_array(
207 self,
208 model: ArrayReferenceModel | InlineArrayModel,
209 *,
210 slices: tuple[slice, ...] | EllipsisType = ...,
211 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
212 ) -> np.ndarray:
213 """Load an array from the archive.
215 Parameters
216 ----------
217 model
218 A Pydantic model that references or holds the array.
219 slices
220 Slices that specify a subset of the original array to read.
221 strip_header
222 A callable that strips out any FITS header cards added by the
223 ``update_header`` argument in the corresponding call to
224 `~lsst.images.serialization.OutputArchive.add_array`.
225 """
226 raise NotImplementedError()
228 @abstractmethod
229 def get_table(
230 self,
231 model: TableModel,
232 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
233 ) -> astropy.table.Table:
234 """Load a table from the archive.
236 Parameters
237 ----------
238 model
239 A Pydantic model that references or holds the table.
240 strip_header
241 A callable that strips out any FITS header cards added by the
242 ``update_header`` argument in the corresponding call to
243 `~lsst.serialization.OutputArchive.add_table`.
245 Returns
246 -------
247 astropy.table.Table
248 The loaded table.
249 """
250 raise NotImplementedError()
252 @abstractmethod
253 def get_structured_array(
254 self,
255 model: TableModel,
256 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
257 ) -> np.ndarray:
258 """Load a table from the archive as a structured array.
260 Parameters
261 ----------
262 model
263 A Pydantic model that references or holds the table.
264 strip_header
265 A callable that strips out any FITS header cards added by the
266 ``update_header`` argument in the corresponding call to
267 `~lsst.serialization.OutputArchive.add_structured_array`.
269 Returns
270 -------
271 numpy.ndarray
272 The loaded table as a structured array.
273 """
274 raise NotImplementedError()
276 def get_opaque_metadata(self) -> OpaqueArchiveMetadata | None:
277 """Return opaque metadata loaded from the file that should be saved if
278 another version of the object is saved to the same file format.
280 Returns
281 -------
282 OpaqueArchiveMetadata
283 Opaque metadata specific to this archive type that should be
284 round-tripped if it is saved in the same format.
285 """
286 return None
289class DetachedArchive(InputArchive[Any]):
290 """An input archive that is not attached to any file.
292 Every method that would read data from a file raises
293 `ArchiveAccessRequiredError`.
295 Notes
296 -----
297 Passing an instance to `ArchiveTree.deserialize_component` probes
298 whether a component can be deserialized from the tree alone: success
299 means no file access was needed, while `ArchiveAccessRequiredError`
300 means the caller must use a live archive instead. Instances hold no
301 state, so a single instance can be shared by any number of probes.
302 """
304 def deserialize_pointer[U: ArchiveTree, V](
305 self, pointer: Any, model_type: type[U], deserializer: Callable[[U, InputArchive[Any]], V]
306 ) -> V:
307 # Docstring inherited.
308 raise ArchiveAccessRequiredError("Dereferencing an archive pointer requires file access.")
310 def get_frame_set(self, ref: Any) -> FrameSet:
311 # Docstring inherited.
312 raise ArchiveAccessRequiredError("Reading a frame set requires file access.")
314 def get_array(
315 self,
316 model: ArrayReferenceModel | InlineArrayModel,
317 *,
318 slices: tuple[slice, ...] | EllipsisType = ...,
319 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
320 ) -> np.ndarray:
321 # Docstring inherited.
322 raise ArchiveAccessRequiredError("Reading an array requires file access.")
324 def get_table(
325 self,
326 model: TableModel,
327 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
328 ) -> astropy.table.Table:
329 # Docstring inherited.
330 raise ArchiveAccessRequiredError("Reading a table requires file access.")
332 def get_structured_array(
333 self,
334 model: TableModel,
335 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,
336 ) -> np.ndarray:
337 # Docstring inherited.
338 raise ArchiveAccessRequiredError("Reading a structured array requires file access.")