Coverage for python/lsst/images/serialization/_input_archive.py: 100%

1# This file is part of lsst-images. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ("ArchiveInfo", "DetachedArchive", "InputArchive") 

15 

16from abc import ABC, abstractmethod 

17from collections.abc import Callable 

18from contextlib import AbstractContextManager 

19from types import EllipsisType 

20from typing import TYPE_CHECKING, Any, TypeVar 

21 

22import astropy.io.fits 

23import astropy.table 

24import astropy.units 

25import numpy as np 

26import pydantic 

27 

28from lsst.resources import ResourcePath, ResourcePathExpression 

29 

30from ._asdf_utils import ArrayReferenceModel, InlineArrayModel 

31from ._common import ( 

32 SCHEMA_URL_HOST, 

33 ArchiveAccessRequiredError, 

34 ArchiveTree, 

35 OpaqueArchiveMetadata, 

36 no_header_updates, 

37) 

38from ._tables import TableModel 

39 

40if TYPE_CHECKING: 

41 from .._transforms import FrameSet 

42 

43 

44# This pre-python-3.12 declaration is needed by Sphinx (probably the 

45# autodoc-typehints plugin. 

46P = TypeVar("P", bound=pydantic.BaseModel) 

47 

48 

49class ArchiveInfo(pydantic.BaseModel, frozen=True): 

50 """Basic identifying information about an on-disk archive. 

51 

52 Read from a file's headers/metadata without deserializing pixel data. 

53 """ 

54 

55 schema_url: str 

56 """Canonical schema URL of the top-level tree.""" 

57 

58 schema_name: str 

59 """Schema name parsed from ``schema_url``.""" 

60 

61 schema_version: str 

62 """Schema version parsed from ``schema_url``.""" 

63 

64 format_version: int | None 

65 """Container layout version (FITS ``FMTVER`` / NDF ``FORMAT_VERSION``); 

66 `None` for formats with no separate container version (JSON).""" 

67 

68 @classmethod 

69 def from_schema_url(cls, schema_url: str, *, format_version: int | None) -> ArchiveInfo: 

70 """Build an `ArchiveInfo` by parsing a schema URL of the form 

71 ``https://images.lsst.io/schemas/{name}-{version}``. 

72 

73 The URL is parsed with `~lsst.resources.ResourcePath` and its 

74 hostname must be ``images.lsst.io``, so a ``DATAMODL`` header written 

75 by an unrelated tool cannot steer reads toward an arbitrary schema. 

76 

77 Parameters 

78 ---------- 

79 schema_url 

80 Schema URL to parse for the schema name and version. 

81 format_version 

82 Container layout version, or `None` for formats with no 

83 separate container version. 

84 """ 

85 parsed = ResourcePath(schema_url) 

86 if parsed.netloc != SCHEMA_URL_HOST: 

87 raise ValueError( 

88 f"Schema URL {schema_url!r} is not hosted at {SCHEMA_URL_HOST!r}; " 

89 "this file was not written by lsst.images." 

90 ) 

91 tail = parsed.basename() 

92 # Split on the last hyphen: schema names may contain hyphens; the 

93 # version (after the final hyphen) is assumed not to. 

94 name, _, version = tail.rpartition("-") 

95 if not name or not version: 

96 raise ValueError(f"Cannot parse schema name/version from URL {schema_url!r}.") 

97 return cls( 

98 schema_url=schema_url, 

99 schema_name=name, 

100 schema_version=version, 

101 format_version=format_version, 

102 ) 

103 

104 

105class InputArchive[P: pydantic.BaseModel](ABC): 

106 """Abstract interface for reading from a file format. 

107 

108 Notes 

109 ----- 

110 An input archive instance is assumed to be paired with a Pydantic model 

111 that represents a JSON tree, with the archive used to deserialize data that 

112 is not native JSON from data that is (which may just be a reference to 

113 binary data stored elsewhere in the file). The archive doesn't actually 

114 hold that model instance because we'd prefer to avoid making the input 

115 archive generic over the model type. It is expected that most concrete 

116 archive implementations will provide a method to load the paired model from 

117 a file, but this is not part of the base class interface. 

118 """ 

119 

120 @classmethod 

121 def get_basic_info(cls, path: ResourcePathExpression) -> ArchiveInfo: 

122 """Return basic identifying information for the archive at ``path`` 

123 without deserializing pixel data. 

124 

125 Each concrete backend reads only the headers/metadata it needs. 

126 

127 Parameters 

128 ---------- 

129 path 

130 Path to the archive to read. 

131 """ 

132 raise NotImplementedError(f"{cls.__name__} does not implement get_basic_info.") 

133 

134 @classmethod 

135 def open_tree( 

136 cls, 

137 path: ResourcePathExpression, 

138 *, 

139 partial: bool = True, 

140 **backend_kwargs: Any, 

141 ) -> AbstractContextManager[tuple[InputArchive[P], ArchiveTree, ArchiveInfo]]: 

142 """Open ``path``, load and validate its top-level tree, and yield 

143 ``(archive, tree, info)`` as a context manager. 

144 

145 Parameters 

146 ---------- 

147 path 

148 File to be opened. Can be local or remote. 

149 partial 

150 Whether the file should be opened for incremental reads or not. 

151 Can be ignored by a backend where not relevant. 

152 **backend_kwargs 

153 Any keyword parameters that should be forwarded to the backend 

154 open. 

155 

156 Raises 

157 ------ 

158 ArchiveReadError 

159 If the file's schema is not registered. 

160 

161 Notes 

162 ----- 

163 Each concrete backend implements this. 

164 """ 

165 raise NotImplementedError(f"{cls.__name__} does not implement open_tree.") 

166 

167 @abstractmethod 

168 def deserialize_pointer[U: ArchiveTree, V]( 

169 self, pointer: P, model_type: type[U], deserializer: Callable[[U, InputArchive[P]], V] 

170 ) -> V: 

171 """Deserialize an object that was saved by 

172 `~lsst.serialization.OutputArchive.serialize_pointer`. 

173 

174 Parameters 

175 ---------- 

176 pointer 

177 JSON Pointer model to dereference. 

178 model_type 

179 Pydantic model type that the pointer should dereference to. 

180 deserializer 

181 Callable that takes an instance of ``model_type`` and an input 

182 archive, and returns the deserialized object. 

183 

184 Returns 

185 ------- 

186 V 

187 The deserialized object. 

188 

189 Notes 

190 ----- 

191 Implementations are required to remember previously-deserialized 

192 objects and return them when the same pointer is passed in multiple 

193 times. 

194 

195 There is no ``deserialize_direct`` (to pair with 

196 `~lsst.serialization.OutputArchive.serialize_direct`) because the 

197 caller can just call a deserializer function directly on a sub-model 

198 of its Pydantic tree. 

199 """ 

200 raise NotImplementedError() 

201 

202 @abstractmethod 

203 def get_frame_set(self, ref: P) -> FrameSet: 

204 """Return an already-deserialized frame set from the archive. 

205 

206 Parameters 

207 ---------- 

208 ref 

209 Implementation-specific reference to the frame set. 

210 

211 Returns 

212 ------- 

213 FrameSet 

214 Loaded frame set. 

215 """ 

216 raise NotImplementedError() 

217 

218 @abstractmethod 

219 def get_array( 

220 self, 

221 model: ArrayReferenceModel | InlineArrayModel, 

222 *, 

223 slices: tuple[slice, ...] | EllipsisType = ..., 

224 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

225 ) -> np.ndarray: 

226 """Load an array from the archive. 

227 

228 Parameters 

229 ---------- 

230 model 

231 A Pydantic model that references or holds the array. 

232 slices 

233 Slices that specify a subset of the original array to read. 

234 strip_header 

235 A callable that strips out any FITS header cards added by the 

236 ``update_header`` argument in the corresponding call to 

237 `~lsst.images.serialization.OutputArchive.add_array`. 

238 """ 

239 raise NotImplementedError() 

240 

241 @abstractmethod 

242 def get_table( 

243 self, 

244 model: TableModel, 

245 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

246 ) -> astropy.table.Table: 

247 """Load a table from the archive. 

248 

249 Parameters 

250 ---------- 

251 model 

252 A Pydantic model that references or holds the table. 

253 strip_header 

254 A callable that strips out any FITS header cards added by the 

255 ``update_header`` argument in the corresponding call to 

256 `~lsst.serialization.OutputArchive.add_table`. 

257 

258 Returns 

259 ------- 

260 astropy.table.Table 

261 The loaded table. 

262 """ 

263 raise NotImplementedError() 

264 

265 @abstractmethod 

266 def get_structured_array( 

267 self, 

268 model: TableModel, 

269 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

270 ) -> np.ndarray: 

271 """Load a table from the archive as a structured array. 

272 

273 Parameters 

274 ---------- 

275 model 

276 A Pydantic model that references or holds the table. 

277 strip_header 

278 A callable that strips out any FITS header cards added by the 

279 ``update_header`` argument in the corresponding call to 

280 `~lsst.serialization.OutputArchive.add_structured_array`. 

281 

282 Returns 

283 ------- 

284 numpy.ndarray 

285 The loaded table as a structured array. 

286 """ 

287 raise NotImplementedError() 

288 

289 def get_opaque_metadata(self) -> OpaqueArchiveMetadata | None: 

290 """Return opaque metadata loaded from the file that should be saved if 

291 another version of the object is saved to the same file format. 

292 

293 Returns 

294 ------- 

295 OpaqueArchiveMetadata 

296 Opaque metadata specific to this archive type that should be 

297 round-tripped if it is saved in the same format. 

298 """ 

299 return None 

300 

301 

302class DetachedArchive(InputArchive[Any]): 

303 """An input archive that is not attached to any file. 

304 

305 Every method that would read data from a file raises 

306 `ArchiveAccessRequiredError`. 

307 

308 Notes 

309 ----- 

310 Passing an instance to `ArchiveTree.deserialize_component` probes 

311 whether a component can be deserialized from the tree alone: success 

312 means no file access was needed, while `ArchiveAccessRequiredError` 

313 means the caller must use a live archive instead. Instances hold no 

314 state, so a single instance can be shared by any number of probes. 

315 """ 

316 

317 def deserialize_pointer[U: ArchiveTree, V]( 

318 self, pointer: Any, model_type: type[U], deserializer: Callable[[U, InputArchive[Any]], V] 

319 ) -> V: 

320 # Docstring inherited. 

321 raise ArchiveAccessRequiredError("Dereferencing an archive pointer requires file access.") 

322 

323 def get_frame_set(self, ref: Any) -> FrameSet: 

324 # Docstring inherited. 

325 raise ArchiveAccessRequiredError("Reading a frame set requires file access.") 

326 

327 def get_array( 

328 self, 

329 model: ArrayReferenceModel | InlineArrayModel, 

330 *, 

331 slices: tuple[slice, ...] | EllipsisType = ..., 

332 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

333 ) -> np.ndarray: 

334 # Docstring inherited. 

335 raise ArchiveAccessRequiredError("Reading an array requires file access.") 

336 

337 def get_table( 

338 self, 

339 model: TableModel, 

340 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

341 ) -> astropy.table.Table: 

342 # Docstring inherited. 

343 raise ArchiveAccessRequiredError("Reading a table requires file access.") 

344 

345 def get_structured_array( 

346 self, 

347 model: TableModel, 

348 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

349 ) -> np.ndarray: 

350 # Docstring inherited. 

351 raise ArchiveAccessRequiredError("Reading a structured array requires file access.")