Coverage for python/lsst/images/serialization/_frozen_schemas.py: 86%
126 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-29 02:31 -0700
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11"""Frozen JSON schema files for the serialization data models.
13Every `~lsst.images.serialization.ArchiveTree` subclass has a canonical JSON
14Schema derived from its pydantic model. These are written to git-committed
15``schemas/`` files so the published schema at
16``https://images.lsst.io/schemas/{name}-{version}`` is a stable artifact
17rather than whatever the code currently produces, and so superseded versions
18remain available after the models move on.
19"""
21from __future__ import annotations
23__all__ = (
24 "FrozenSchemaError",
25 "available_schema_classes",
26 "check_frozen_schemas",
27 "dump_schema",
28 "frozen_schema_filename",
29 "frozen_schema_path",
30 "schema_dependencies",
31 "write_frozen_schemas",
32)
34import importlib.metadata
35import json
36import re
37from pathlib import Path
38from typing import Any, TypeVar, get_args, get_origin
40from ._asdf_utils import ArrayReferenceModel
41from ._common import ArchiveTree, is_development_version
42from ._io import (
43 _BUILTIN_SCHEMA_PROVIDERS,
44 _REGISTRY,
45 _SCHEMA_ENTRY_POINT_GROUP,
46 class_for_schema,
47 parameterize_tree,
48)
51class FrozenSchemaError(RuntimeError):
52 """A finalized frozen schema would change without a version bump."""
55def available_schema_classes(package: str = "lsst.images") -> list[type[ArchiveTree]]:
56 """Return every `~lsst.images.serialization.ArchiveTree` subclass owned
57 by ``package``, sorted by schema name.
59 Parameters
60 ----------
61 package
62 Only classes whose defining module is this package (or a
63 subpackage of it) are returned, so a package freezes and publishes
64 exactly the schemas it owns, and schema classes created elsewhere
65 (e.g. test doubles) are never picked up by accident.
67 Notes
68 -----
69 Candidate schemas come from the in-memory registry, the built-in lazy
70 providers, and the ``lsst.images.schemas`` entry point group, so an
71 external package's schemas are found even when nothing has imported
72 their modules yet.
73 """
74 entry_point_names = {
75 entry_point.name for entry_point in importlib.metadata.entry_points(group=_SCHEMA_ENTRY_POINT_GROUP)
76 }
77 classes: list[type[ArchiveTree]] = []
78 for name in sorted(set(_REGISTRY) | set(_BUILTIN_SCHEMA_PROVIDERS) | entry_point_names):
79 cls = class_for_schema(name)
80 if cls is None: 80 ↛ 81line 80 didn't jump to line 81 because the condition on line 80 was never true
81 raise RuntimeError(f"Schema {name!r} is registered but its class could not be loaded.")
82 if cls.__module__ != package and not cls.__module__.startswith(f"{package}."):
83 continue
84 classes.append(cls)
85 return classes
88def _summary_description(text: str) -> str:
89 """Return a docstring-derived description trimmed to its summary.
91 Text is kept up to (but not including) the first numpydoc section header:
92 a non-blank line immediately followed by a line of dashes at least as long
93 as it. Text with no such header (e.g. a one-line ``pydantic.Field``
94 description) is returned unchanged, so only the sectioned content of a
95 class docstring (``Notes``, ``Parameters``, ...) is dropped and the summary
96 plus extended summary are kept.
97 """
98 lines = text.split("\n")
99 for i in range(len(lines) - 1):
100 header = lines[i].strip()
101 underline = lines[i + 1].strip()
102 if header and re.fullmatch(r"-+", underline) and len(underline) >= len(header):
103 return "\n".join(lines[:i]).rstrip()
104 return text
107def _summarize_descriptions(node: Any) -> None:
108 """Trim every ``description`` in a schema tree to its summary, in place."""
109 if isinstance(node, dict):
110 description = node.get("description")
111 if isinstance(description, str):
112 node["description"] = _summary_description(description)
113 for value in node.values():
114 _summarize_descriptions(value)
115 elif isinstance(node, list):
116 for item in node:
117 _summarize_descriptions(item)
120def dump_schema(tree_cls: type[ArchiveTree]) -> dict[str, Any]:
121 """Return the JSON Schema for ``tree_cls``.
123 Parameters
124 ----------
125 tree_cls
126 Serialization model class to dump.
128 Notes
129 -----
130 Generic trees are parameterized over
131 `~lsst.images.serialization.ArrayReferenceModel`, matching the convention
132 used by ``lsst-images-admin diagram``. Model descriptions derived from
133 class docstrings are trimmed to their summary so the numpydoc sections
134 that follow it do not leak into the published schema.
135 """
136 schema = parameterize_tree(tree_cls, ArrayReferenceModel).model_json_schema()
137 # A recursive model (e.g. sum_field) produces a root that is just a $ref
138 # into $defs, with the class's json_schema_extra landing on the $def.
139 # Hoist the canonical identity to the document root so every frozen
140 # document self-identifies; $ref siblings are valid in draft 2020-12.
141 schema.setdefault("$id", f"{tree_cls.SCHEMA_URL_BASE}/{tree_cls.SCHEMA_NAME}-{tree_cls.SCHEMA_VERSION}")
142 schema.setdefault("title", tree_cls.SCHEMA_NAME)
143 # Nested ArchiveTree definitions inherit their class's $id, but $id
144 # starts a new resolution scope in draft 2020-12, which would break the
145 # root-relative "#/$defs/..." references pydantic generates inside them.
146 # Record the canonical URL under a non-reserved key instead, which
147 # validators ignore and documentation tooling can still use to identify
148 # published sub-schemas.
149 for definition in schema.get("$defs", {}).values():
150 if isinstance(definition, dict) and "$id" in definition:
151 definition["x-lsst-schema-url"] = definition.pop("$id")
152 _summarize_descriptions(schema)
153 return schema
156def frozen_schema_filename(tree_cls: type[ArchiveTree]) -> str:
157 """Return the frozen-schema filename for ``tree_cls``.
159 Parameters
160 ----------
161 tree_cls
162 Serialization model class to name the file for.
163 """
164 return f"{tree_cls.SCHEMA_NAME}-{tree_cls.SCHEMA_VERSION}.json"
167def frozen_schema_path(directory: Path, tree_cls: type[ArchiveTree]) -> Path:
168 """Return the frozen-schema file path for ``tree_cls`` under
169 ``directory``.
171 Parameters
172 ----------
173 directory
174 Directory holding the frozen schema files.
175 tree_cls
176 Serialization model class to locate the file for.
178 Notes
179 -----
180 Files are laid out as ``{name}/{name}-{version}.json``: one
181 subdirectory per schema so the directory stays navigable as versions
182 accumulate, with the full name-version filename kept so a file is
183 self-identifying when copied elsewhere.
184 """
185 return directory / tree_cls.SCHEMA_NAME / frozen_schema_filename(tree_cls)
188def _canonical_text(schema: dict[str, Any]) -> str:
189 """Return the canonical file serialization of ``schema``."""
190 return json.dumps(schema, indent=2, sort_keys=True) + "\n"
193def _declaring_classes(tree_cls: type[ArchiveTree]) -> list[type[ArchiveTree]]:
194 """Return the classes in ``tree_cls``'s MRO that declare a schema, nearest
195 first.
197 A class declares a schema when it sets both ``SCHEMA_NAME`` and
198 ``SCHEMA_VERSION`` itself, the same convention
199 ``ArchiveTree.__pydantic_init_subclass__`` uses to decide what to register.
200 The first entry is the schema ``tree_cls`` *is*; any that follow are
201 schemas it inherits from. A pydantic-parameterized generic (``Image[P]``)
202 has no declarations of its own, so it resolves to the generic it was made
203 from.
204 """
205 return [
206 base
207 for base in tree_cls.__mro__
208 if isinstance(base, type)
209 and issubclass(base, ArchiveTree)
210 and "SCHEMA_NAME" in base.__dict__
211 and "SCHEMA_VERSION" in base.__dict__
212 ]
215def _archive_trees_in(annotation: Any) -> set[type[ArchiveTree]]:
216 """Return every `ArchiveTree` subclass reachable from a type annotation.
218 Containers, unions and type variable bounds are all descended into, so a
219 model reached only as ``dict[str, Model | None]`` is still found. Nested
220 models are not themselves expanded; that is the caller's job.
221 """
222 found: set[type[ArchiveTree]] = set()
223 stack: list[Any] = [annotation]
224 seen: set[int] = set()
225 while stack:
226 node = stack.pop()
227 if id(node) in seen:
228 continue
229 seen.add(id(node))
230 # Check the origin before isinstance(node, type): a parameterized
231 # builtin such as dict[str, X] answers to both on some Python
232 # versions, and only the origin branch descends into its arguments.
233 origin = get_origin(node)
234 if origin is not None:
235 stack.append(origin)
236 stack.extend(get_args(node))
237 elif isinstance(node, type):
238 if issubclass(node, ArchiveTree):
239 found.add(node)
240 elif isinstance(node, TypeVar) and node.__bound__ is not None:
241 stack.append(node.__bound__)
242 return found
245def schema_dependencies(tree_cls: type[ArchiveTree]) -> dict[str, type[ArchiveTree]]:
246 """Return every other schema whose content is inlined into ``tree_cls``'s
247 frozen document.
249 Parameters
250 ----------
251 tree_cls
252 Serialization model class to inspect.
254 Returns
255 -------
256 `dict` [ `str`, `type` ]
257 Mapping from ``SCHEMA_NAME`` to the class declaring it, for every
258 schema ``tree_cls`` depends on, directly or transitively.
259 ``tree_cls``'s own schema is never included, so a self-referential
260 model reports nothing.
262 Notes
263 -----
264 Fields and base classes are both followed. An embedded model records its
265 version in the document as a ``x-lsst-schema-url``, but pydantic flattens
266 an inherited model's fields into the subclass and records nothing, so a
267 base's version cannot be recovered from the document it contributed to.
268 """
269 own = _declaring_classes(tree_cls)
270 own_name = own[0].SCHEMA_NAME if own else None
271 result: dict[str, type[ArchiveTree]] = {}
272 stack: list[type[ArchiveTree]] = [tree_cls]
273 visited: set[type[ArchiveTree]] = set()
274 while stack:
275 current = stack.pop()
276 if current in visited: 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true
277 continue
278 visited.add(current)
279 # Skip the first declaration, which is the schema `current` itself is.
280 candidates = list(_declaring_classes(current)[1:])
281 for field in current.model_fields.values():
282 for referenced in _archive_trees_in(field.annotation):
283 candidates.extend(_declaring_classes(referenced)[:1])
284 for candidate in candidates:
285 name = candidate.SCHEMA_NAME
286 if name == own_name or name in result:
287 continue
288 result[name] = candidate
289 stack.append(candidate)
290 return result
293def write_frozen_schemas(directory: Path, package: str = "lsst.images") -> list[Path]:
294 """Write the frozen schema file for every current schema.
296 Parameters
297 ----------
298 directory
299 Directory to write the ``{name}-{version}.json`` files into; created
300 if necessary.
301 package
302 Package whose schemas to freeze; see
303 `~lsst.images.serialization.available_schema_classes`.
305 Returns
306 -------
307 `list` [ `pathlib.Path` ]
308 Paths that were created or rewritten.
310 Notes
311 -----
312 Schemas at a development version (a PEP 440 ``.devN`` release) are skipped
313 and never frozen. A finalized schema is frozen only on its first write; an
314 existing frozen file is immutable, so a live-model change to it raises
315 rather than overwriting. Frozen files for superseded versions are never
316 touched, so old schema URLs keep resolving.
318 A schema is frozen only if every schema it depends on is already finalized;
319 see `schema_dependencies`. The check runs when a frozen file is first
320 written, so an already-committed file is never re-validated.
322 Raises
323 ------
324 FrozenSchemaError
325 If a finalized schema's frozen file exists and the live model would
326 change its content; bump ``SCHEMA_VERSION`` instead of overwriting.
327 Also raised if a schema being frozen depends on a schema still in
328 development.
329 """
330 changed: list[Path] = []
331 for cls in available_schema_classes(package):
332 if is_development_version(cls.SCHEMA_VERSION):
333 continue
334 path = frozen_schema_path(directory, cls)
335 text = _canonical_text(dump_schema(cls))
336 if path.exists():
337 if path.read_text() != text:
338 raise FrozenSchemaError(
339 f"{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION} is finalized and frozen; "
340 "bump SCHEMA_VERSION to change it rather than overwriting the frozen file."
341 )
342 continue
343 if developing := sorted(
344 f"{name}-{dependency.SCHEMA_VERSION}"
345 for name, dependency in schema_dependencies(cls).items()
346 if is_development_version(dependency.SCHEMA_VERSION)
347 ):
348 raise FrozenSchemaError(
349 f"{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION} cannot be frozen: it depends on "
350 f"development schema(s) {', '.join(developing)}, whose content it inlines and "
351 "which are still free to change. Finalize them first, or return this schema "
352 "to a development version."
353 )
354 path.parent.mkdir(parents=True, exist_ok=True)
355 path.write_text(text)
356 changed.append(path)
357 return changed
360def check_frozen_schemas(directory: Path, package: str = "lsst.images") -> list[str]:
361 """Check the frozen schema files against the current models.
363 Parameters
364 ----------
365 directory
366 Directory holding the frozen ``{name}-{version}.json`` files.
367 package
368 Package whose schemas to check; see
369 `~lsst.images.serialization.available_schema_classes`.
371 Returns
372 -------
373 `list` [ `str` ]
374 One problem description per current schema whose frozen file is
375 missing or does not match the current model; empty when the frozen
376 files are up to date.
378 Notes
379 -----
380 Schemas at a development version (a PEP 440 ``.devN`` release) are
381 skipped and not reported as missing.
382 """
383 problems: list[str] = []
384 for cls in available_schema_classes(package):
385 if is_development_version(cls.SCHEMA_VERSION):
386 continue
387 path = frozen_schema_path(directory, cls)
388 if not path.exists():
389 problems.append(f"{path.relative_to(directory)}: missing")
390 elif path.read_text() != _canonical_text(dump_schema(cls)):
391 problems.append(f"{path.relative_to(directory)}: finalized schema changed; bump SCHEMA_VERSION")
392 return problems