Coverage for python/lsst/images/tests/_schema_fixtures.py: 95%
323 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 01:57 -0700
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11"""Committed reference fixtures for the serialization data models.
13Every retained ``{name}-{version}`` fixture under ``tests/data/schemas`` is
14the instance-level twin of the frozen schema document of the same version
15under ``schemas``: it is read through the live model on every test run, so a
16model change that would alter what a file looks like shows up as fixture
17drift rather than as silence.
19The layout, the lifecycle and the checks are described in
20:ref:`lsst.images-schema-versioning`.
21"""
23from __future__ import annotations
25__all__ = (
26 "SchemaFixture",
27 "SchemaFixtureError",
28 "canonical_fixture_text",
29 "check_schema_fixtures",
30 "compare_fixture_versions",
31 "current_fixture_path",
32 "fixture_version",
33 "freeze_schema_fixtures",
34 "iter_schema_fixtures",
35 "read_fixture_tree",
36 "refresh_schema_fixtures",
37)
39import dataclasses
40import json
41import math
42import re
43from collections.abc import Collection, Iterator
44from pathlib import Path
46import pydantic
47from packaging.version import InvalidVersion, Version
49from ..serialization import (
50 ArchiveReadError,
51 ArchiveTree,
52 JsonRef,
53 available_schema_classes,
54 class_for_schema,
55 frozen_schema_path,
56 is_development_version,
57 parameterize_tree,
58)
59from ..serialization._common import _ARCHIVE_READ_CONTEXT
61_AS_SHIPPED_VARIANT = "as_shipped"
62"""Variant name reserved for a fixture whose bytes are preserved exactly as a
63real shipped file produced them. Never canonicalized or rewritten."""
65_CANONICAL_VARIANT = "canonical"
66"""Variant name reserved for the canonicalized twin of an ``as_shipped``
67fixture: the same tree as the current code reads and would write it."""
69_FIXTURE_RE = re.compile(r"^(?P<version>\d+\.\d+\.\d+(?:\.dev)?)(?:-(?P<variant>[a-z0-9_]+))?$")
70"""Filename pattern, applied to the stem with the ``{name}-`` prefix removed.
72The version token is matched greedily first, so a variant name can never be
73mistaken for part of the version.
74"""
76_RETIRED_DIR = "retired"
79def fixture_version(schema_version: str) -> str:
80 """Return the fixture-filename version for a schema version.
82 Parameters
83 ----------
84 schema_version
85 Schema version string, e.g. ``1.0.0`` or ``1.0.0.dev0``.
87 Returns
88 -------
89 `str`
90 The release part, with a bare ``.dev`` suffix for a development
91 release. The exact development counter lives only in the
92 ``schema_version`` stamp inside the file, so a development schema has
93 exactly one fixture path whatever its counter.
94 """
95 version = Version(schema_version)
96 release = f"{version.major}.{version.minor}.{version.micro}"
97 return f"{release}.dev" if is_development_version(schema_version) else release
100def _fixture_filename(name: str, version: str, variant: str | None = None) -> str:
101 """Return the fixture filename for a schema name, version and variant.
103 Parameters
104 ----------
105 name
106 Schema name.
107 version
108 Fixture-filename version, as returned by `fixture_version`.
109 variant
110 Variant name, or `None` for the base fixture.
111 """
112 stem = f"{name}-{version}" if variant is None else f"{name}-{version}-{variant}"
113 return f"{stem}.json"
116def _fixture_dir_path(directory: Path, name: str, version: str, variant: str | None = None) -> Path:
117 """Return the path of a fixture within a fixture tree.
119 Parameters
120 ----------
121 directory
122 Directory holding the fixture tree.
123 name
124 Schema name.
125 version
126 Fixture-filename version, as returned by `fixture_version`.
127 variant
128 Variant name, or `None` for the base fixture.
130 Notes
131 -----
132 Files are laid out as ``{name}/{name}-{version}.json``, mirroring the
133 frozen schema documents so the instance-level and schema-level trees are
134 navigable the same way.
135 """
136 return directory / name / _fixture_filename(name, version, variant)
139def current_fixture_path(directory: Path, name: str, *, variant: str | None = None) -> Path:
140 """Return the fixture path for a schema's live version, by name.
142 Parameters
143 ----------
144 directory
145 Directory holding the fixture tree.
146 name
147 Schema name, e.g. ``visit_image``.
148 variant
149 Variant name, or `None` for the base fixture.
151 Returns
152 -------
153 `pathlib.Path`
154 Path to the fixture for whatever version the code is currently at.
156 Raises
157 ------
158 LookupError
159 If no schema is registered under ``name``.
161 Notes
162 -----
163 This is the entry point for tests that just want representative data.
164 Resolving through the live class means such a test does not need editing
165 when a schema is frozen or its version is bumped.
166 """
167 tree_cls = class_for_schema(name)
168 if tree_cls is None:
169 raise LookupError(f"No schema is registered under {name!r}.")
170 return _fixture_dir_path(directory, name, fixture_version(tree_cls.SCHEMA_VERSION), variant)
173def canonical_fixture_text(tree: ArchiveTree) -> str:
174 """Return the canonical file serialization of a tree.
176 Parameters
177 ----------
178 tree
179 Serialization model instance to write.
181 Notes
182 -----
183 Model field order is kept rather than sorting keys, so the version stamps
184 stay at the top of the file where a human reading it looks first. The form
185 is idempotent: re-reading the result and re-serializing reproduces it byte
186 for byte.
187 """
188 return tree.model_dump_json(indent=2) + "\n"
191@dataclasses.dataclass(frozen=True)
192class SchemaFixture:
193 """One committed fixture file, located and classified."""
195 path: Path
196 """Path to the fixture file."""
198 name: str
199 """Schema name, taken from the containing directory."""
201 version: str
202 """Fixture-filename version, e.g. ``1.0.0`` or ``1.0.0.dev``."""
204 variant: str | None
205 """Variant name, or `None` for the base fixture."""
207 retired: bool
208 """Whether the fixture sits in the schema's ``retired`` subdirectory, and
209 so is expected to be rejected rather than read."""
211 tree_cls: type[ArchiveTree] | None
212 """The registered model class, or `None` if no schema of this name is
213 registered."""
215 @property
216 def is_as_shipped(self) -> bool:
217 """Whether this fixture preserves real shipped bytes."""
218 return self.variant == _AS_SHIPPED_VARIANT
220 @property
221 def is_canonical_twin(self) -> bool:
222 """Whether this fixture is the canonicalized twin of an ``as_shipped``
223 sibling.
224 """
225 return self.variant == _CANONICAL_VARIANT
227 def problem(self, message: str) -> str:
228 """Return ``message`` prefixed with this fixture's filename.
230 Parameters
231 ----------
232 message
233 Problem description, phrased to read after the filename.
234 """
235 return f"{self.path.name}: {message}"
238def iter_schema_fixtures(directory: Path) -> Iterator[SchemaFixture]:
239 """Yield every fixture found under a fixture tree.
241 Parameters
242 ----------
243 directory
244 Directory holding the fixture tree.
246 Yields
247 ------
248 `SchemaFixture`
249 One entry per file whose name matches the fixture grammar, sorted by
250 path. Files that do not match are skipped, so a README or other
251 supporting file in the tree is not reported as a fixture.
252 """
253 if not directory.is_dir(): 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true
254 return
255 for schema_dir in sorted(p for p in directory.iterdir() if p.is_dir()):
256 name = schema_dir.name
257 tree_cls = class_for_schema(name)
258 for retired, parent in ((False, schema_dir), (True, schema_dir / _RETIRED_DIR)):
259 if not parent.is_dir():
260 continue
261 for path in sorted(parent.glob("*.json")):
262 if (match := _FIXTURE_RE.fullmatch(path.stem.removeprefix(f"{name}-"))) is None:
263 continue
264 yield SchemaFixture(
265 path=path,
266 name=name,
267 version=match.group("version"),
268 variant=match.group("variant"),
269 retired=retired,
270 tree_cls=tree_cls,
271 )
274def read_fixture_tree(fixture: SchemaFixture) -> ArchiveTree:
275 """Validate a fixture through its live model and return the tree.
277 Parameters
278 ----------
279 fixture
280 The fixture to read.
282 Returns
283 -------
284 `~lsst.images.serialization.ArchiveTree`
285 The validated tree, parameterized over
286 `~lsst.images.serialization.JsonRef` as the JSON backend does.
288 Raises
289 ------
290 ArchiveReadError
291 Raised when a retired fixture is rejected by Pydantic validation.
292 RuntimeError
293 If the fixture's schema is not registered.
294 """
295 if fixture.tree_cls is None: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true
296 raise RuntimeError(f"No schema is registered under {fixture.name!r}.")
297 parameterized = parameterize_tree(fixture.tree_cls, JsonRef)
298 try:
299 return parameterized.model_validate_json(fixture.path.read_text(), context=_ARCHIVE_READ_CONTEXT)
300 except pydantic.ValidationError as exc:
301 if fixture.retired:
302 raise ArchiveReadError(
303 f"Retired fixture {fixture.path.name!r} is rejected by its current schema: {exc}"
304 ) from exc
305 raise
308def _check_one(fixture: SchemaFixture) -> list[str]:
309 """Return the problems found in a single fixture."""
310 if fixture.tree_cls is None:
311 return [fixture.problem(f"schema {fixture.name!r} is not registered")]
312 problems: list[str] = []
313 try:
314 on_disk_json = fixture.path.read_text()
315 on_disk = json.loads(on_disk_json)
316 except (OSError, UnicodeError, json.JSONDecodeError) as exc:
317 return [fixture.problem(f"is not valid JSON: {type(exc).__name__}: {exc}")]
318 if not isinstance(on_disk, dict): 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true
319 return [fixture.problem(f"top-level JSON value is {type(on_disk).__name__}, expected object")]
321 # Identity applies even to retired fixtures. Check it before validation,
322 # because rejection is expected for a retired tree and must not hide a bad
323 # stamp, URL, or filename.
324 on_disk_version = on_disk.get("schema_version")
325 if not isinstance(on_disk_version, str):
326 problems.append(fixture.problem(f"schema_version is {on_disk_version!r}, expected a version string"))
327 else:
328 expected_url = f"{fixture.tree_cls.SCHEMA_URL_BASE}/{fixture.name}-{on_disk_version}"
329 on_disk_url = on_disk.get("schema_url")
330 if on_disk_url != expected_url:
331 problems.append(fixture.problem(f"schema_url is {on_disk_url!r}, expected {expected_url!r}"))
332 try:
333 on_disk_fixture_version = fixture_version(on_disk_version)
334 except InvalidVersion as exc:
335 # fixture_version parses the stamp with packaging.version.Version,
336 # which raises for a malformed-but-string value (e.g. a
337 # hand-edited "1.0.O", letter O for zero); this function's
338 # contract is that it never raises, so report it like any other
339 # fixture problem.
340 problems.append(
341 fixture.problem(f"schema_version {on_disk_version!r} is not a valid version: {exc}")
342 )
343 else:
344 if fixture.version != on_disk_fixture_version:
345 problems.append(
346 fixture.problem(
347 f"filename version {fixture.version!r} disagrees with stamp {on_disk_version!r}",
348 )
349 )
351 if fixture.retired:
352 # Retirement keeps a *superseded* version's fixture, so a retired file
353 # at or above the live version is misplaced rather than retired. No
354 # other check catches that: its stamps can be self-consistent, being
355 # rejected is what a retired fixture is required to do, and the
356 # never-frozen check below deliberately skips retired fixtures.
357 retired_live_version = fixture_version(fixture.tree_cls.SCHEMA_VERSION)
358 if Version(fixture.version) >= Version(retired_live_version):
359 problems.append(
360 fixture.problem(
361 f"is retired at {fixture.version}, which is not older than the live version "
362 f"{retired_live_version}; retirement is for a superseded version"
363 )
364 )
366 try:
367 tree = read_fixture_tree(fixture)
368 except Exception as exc:
369 if fixture.retired:
370 if not isinstance(exc, ArchiveReadError):
371 problems.append(
372 fixture.problem(
373 f"rejection raised unexpected {type(exc).__name__}: {exc}",
374 )
375 )
376 return problems
377 problems.append(fixture.problem(f"does not validate: {type(exc).__name__}: {exc}"))
378 return problems
379 if fixture.retired:
380 problems.append(fixture.problem("is retired but still validates; move it back or update the model"))
381 return problems
382 live_version = fixture_version(fixture.tree_cls.SCHEMA_VERSION)
383 if fixture.version == live_version and not fixture.is_as_shipped:
384 if on_disk_json != canonical_fixture_text(tree):
385 if is_development_version(fixture.tree_cls.SCHEMA_VERSION):
386 remedy = "run 'lsst-images-admin fixtures refresh'"
387 else:
388 remedy = "bump SCHEMA_VERSION rather than rewriting a frozen fixture"
389 problems.append(fixture.problem(f"is not canonical; {remedy}"))
390 return problems
393def _check_as_shipped_pairs(fixtures: list[SchemaFixture]) -> list[str]:
394 """Return the problems found in as_shipped / canonical fixture pairs.
396 A retired fixture is excluded from both directions: it is checked only
397 for being rejected, and (being retired) it no longer validates, so
398 running it through `read_fixture_tree` here would always fail.
399 """
400 problems: list[str] = []
401 twins = {(f.name, f.version): f for f in fixtures if f.is_canonical_twin and not f.retired}
402 for shipped in fixtures:
403 if shipped.retired or not shipped.is_as_shipped or shipped.tree_cls is None:
404 continue
405 key = (shipped.name, shipped.version)
406 twin = twins.get(key)
407 expected = _fixture_filename(shipped.name, shipped.version, _CANONICAL_VARIANT)
408 if twin is None:
409 problems.append(shipped.problem(f"canonical twin {expected} is missing"))
410 continue
411 try:
412 text = canonical_fixture_text(read_fixture_tree(shipped))
413 except Exception as exc:
414 problems.append(shipped.problem(f"cannot be canonicalized: {exc}"))
415 continue
416 if twin.path.read_text() != text:
417 problems.append(
418 twin.problem(
419 "does not match the canonical read of its as_shipped sibling; "
420 "how a shipped file is read has changed, which is a compatibility "
421 "change and not a fixture to refresh",
422 )
423 )
424 for twin in fixtures:
425 if twin.retired or not twin.is_canonical_twin:
426 continue
427 sibling = twin.path.with_name(_fixture_filename(twin.name, twin.version, _AS_SHIPPED_VARIANT))
428 if not sibling.exists(): 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true
429 problems.append(twin.problem(f"has no as_shipped sibling {sibling.name}"))
430 return problems
433def check_schema_fixtures(
434 directory: Path,
435 *,
436 schema_directory: Path | None = None,
437 package: str = "lsst.images",
438 exempt: Collection[str] = (),
439) -> list[str]:
440 """Check the committed fixtures against the current models.
442 Parameters
443 ----------
444 directory
445 Directory holding the fixture tree.
446 schema_directory
447 Directory holding the frozen schema documents. When given, the
448 fixture and schema trees are also checked for pairing in both
449 directions.
450 package
451 Package whose schemas to check; see
452 `~lsst.images.serialization.available_schema_classes`.
453 exempt
454 Schema names that are allowed to have no fixture, for schemas whose
455 data this package cannot construct.
457 Returns
458 -------
459 `list` [ `str` ]
460 One problem description per defect found; empty when the fixture tree
461 is sound. This never raises, so a caller can report every problem at
462 once.
464 Notes
465 -----
466 A ``retired`` fixture is checked only for being rejected; it cannot be
467 validated, so the canonical and pairing checks do not apply to it. An
468 ``as_shipped`` fixture is exempt from the canonical check, and its
469 ``canonical`` twin carries the stronger pairwise check in its place.
470 """
471 exempt = frozenset(exempt)
472 fixtures = list(iter_schema_fixtures(directory))
473 problems: list[str] = []
474 for fixture in fixtures:
475 problems.extend(_check_one(fixture))
476 problems.extend(_check_as_shipped_pairs(fixtures))
477 live: set[tuple[str, str]] = set()
478 scope: set[str] = set()
479 for tree_cls in available_schema_classes(package):
480 name = tree_cls.SCHEMA_NAME
481 scope.add(name)
482 version = fixture_version(tree_cls.SCHEMA_VERSION)
483 live.add((name, version))
484 if name in exempt:
485 continue
486 if not any(f.name == name and f.version == version for f in fixtures if not f.retired):
487 problems.append(f"{_fixture_filename(name, version)}: missing")
488 if schema_directory is not None and not is_development_version(tree_cls.SCHEMA_VERSION):
489 if not frozen_schema_path(schema_directory, tree_cls).exists():
490 problems.append(
491 f"{_fixture_filename(name, version)}: has no frozen document; "
492 "run 'lsst-images-admin schemas write'"
493 )
494 if schema_directory is not None:
495 # Direction 1: a fixture at a version that is neither the live one
496 # nor frozen anywhere -- an interrupted freeze left it behind. A
497 # retired fixture is excluded: it is checked only for being
498 # rejected, and it is expected to have no frozen document once its
499 # schema has moved on.
500 for fixture in fixtures:
501 if fixture.tree_cls is None or fixture.retired or (fixture.name, fixture.version) in live: 501 ↛ 503line 501 didn't jump to line 503 because the condition on line 501 was always true
502 continue
503 document = schema_directory / fixture.name / f"{fixture.name}-{fixture.version}.json"
504 if not document.exists():
505 problems.append(
506 fixture.problem(
507 "is at a version that was never frozen and is not the live version; "
508 "an interrupted freeze leaves this behind",
509 )
510 )
511 # Direction 2: every frozen document -- not just the live one --
512 # needs a same-version fixture, so a superseded version's fixture is
513 # never silently dropped. A fixture under retired/ still counts as
514 # present: retirement is how a superseded version's fixture is kept.
515 if schema_directory.is_dir(): 515 ↛ 529line 515 didn't jump to line 529 because the condition on line 515 was always true
516 for schema_dir in sorted(p for p in schema_directory.iterdir() if p.is_dir()):
517 name = schema_dir.name
518 if name not in scope or name in exempt:
519 continue
520 for path in sorted(schema_dir.glob("*.json")):
521 match = _FIXTURE_RE.fullmatch(path.stem.removeprefix(f"{name}-"))
522 if match is None: 522 ↛ 523line 522 didn't jump to line 523 because the condition on line 522 was never true
523 continue
524 version = match.group("version")
525 if not any(f.name == name and f.version == version for f in fixtures):
526 problems.append(
527 f"{path.name}: has no fixture; run 'lsst-images-admin fixtures refresh'"
528 )
529 return problems
532class SchemaFixtureError(RuntimeError):
533 """A fixture that must not be rewritten would change."""
536def _newest_source(fixtures: list[SchemaFixture], name: str, variant: str | None) -> SchemaFixture | None:
537 """Return the highest-version non-retired fixture to seed from."""
538 candidates = [
539 f
540 for f in fixtures
541 if f.name == name and f.variant == variant and not f.retired and f.tree_cls is not None
542 ]
543 if not candidates:
544 return None
545 return max(candidates, key=lambda f: Version(f.version))
548def refresh_schema_fixtures(directory: Path, *, package: str = "lsst.images") -> list[Path]:
549 """Rewrite every development fixture in canonical form.
551 Parameters
552 ----------
553 directory
554 Directory holding the fixture tree.
555 package
556 Package whose schemas to refresh; see
557 `~lsst.images.serialization.available_schema_classes`.
559 Returns
560 -------
561 `list` [ `pathlib.Path` ]
562 Paths that were created or rewritten.
564 Raises
565 ------
566 SchemaFixtureError
567 If a fixture at a finalized version would change. Bump
568 ``SCHEMA_VERSION`` instead of rewriting it.
570 Notes
571 -----
572 Only fixtures of a schema whose live version is a development release are
573 rewritten. A development schema with no fixture yet is seeded from its
574 newest non-retired fixture, per variant, so the exemplar is carried forward
575 rather than reinvented. ``retired`` fixtures are never touched.
577 Canonical twins are regenerated from their ``as_shipped`` siblings only
578 while their schema version is in development. At a finalized version the
579 twin pins how shipped bytes normalize on read, so changing or creating it
580 is a reviewed, manual operation rather than something ``refresh`` may do.
581 A retired ``as_shipped`` fixture is excluded, since it no longer validates
582 and has no twin to regenerate.
584 This writes and creates files; it never invokes version control.
585 """
586 fixtures = list(iter_schema_fixtures(directory))
587 changed: list[Path] = []
588 for tree_cls in available_schema_classes(package):
589 name = tree_cls.SCHEMA_NAME
590 live_version = fixture_version(tree_cls.SCHEMA_VERSION)
591 if is_development_version(tree_cls.SCHEMA_VERSION):
592 variants = {
593 f.variant for f in fixtures if f.name == name and not f.retired and not f.is_canonical_twin
594 } or {None}
595 for variant in sorted(variants, key=lambda v: (v is not None, v or "")):
596 if variant == _AS_SHIPPED_VARIANT:
597 continue
598 target = _fixture_dir_path(directory, name, live_version, variant)
599 source = (
600 SchemaFixture(
601 path=target,
602 name=name,
603 version=live_version,
604 variant=variant,
605 retired=False,
606 tree_cls=tree_cls,
607 )
608 if target.exists()
609 else _newest_source(fixtures, name, variant)
610 )
611 if source is None:
612 continue
613 text = canonical_fixture_text(read_fixture_tree(source))
614 if not target.exists() or target.read_text() != text:
615 target.parent.mkdir(parents=True, exist_ok=True)
616 target.write_text(text)
617 changed.append(target)
618 else:
619 for fixture in fixtures:
620 if fixture.name != name or fixture.retired or fixture.is_canonical_twin:
621 continue
622 if fixture.is_as_shipped or fixture.version != live_version:
623 continue
624 text = canonical_fixture_text(read_fixture_tree(fixture))
625 if fixture.path.read_text() != text:
626 raise SchemaFixtureError(
627 f"{fixture.path.name} is at a finalized version and would change; "
628 "bump SCHEMA_VERSION rather than rewriting a frozen fixture."
629 )
630 for shipped in fixtures:
631 if shipped.name != name or shipped.retired or not shipped.is_as_shipped:
632 continue
633 twin = shipped.path.with_name(_fixture_filename(name, shipped.version, _CANONICAL_VARIANT))
634 text = canonical_fixture_text(read_fixture_tree(shipped))
635 if twin.exists() and twin.read_text() == text:
636 continue
637 if not shipped.version.endswith(".dev"):
638 raise SchemaFixtureError(
639 f"{twin.name} is at a finalized version and would change; "
640 "how shipped bytes normalize is a compatibility contract, so bump "
641 "SCHEMA_VERSION rather than refreshing this twin."
642 )
643 twin.write_text(text)
644 changed.append(twin)
645 return changed
648def freeze_schema_fixtures(directory: Path, *, package: str = "lsst.images") -> list[tuple[Path, Path]]:
649 """Move ordinary development fixtures to their finalized versions.
651 Parameters
652 ----------
653 directory
654 Directory holding the fixture tree.
655 package
656 Package whose schemas to freeze; see
657 `~lsst.images.serialization.available_schema_classes`.
659 Returns
660 -------
661 `list` [ `tuple` [ `pathlib.Path`, `pathlib.Path` ] ]
662 One ``(written, removed)`` pair per fixture that was frozen.
664 Raises
665 ------
666 SchemaFixtureError
667 If a target path already exists or an ``as_shipped`` development
668 fixture cannot be frozen without rewriting its preserved bytes.
670 Notes
671 -----
672 Writes the final-version fixture from the ``.dev`` fixture's content in
673 canonical form, which normalizes the stamp from ``X.Y.Z.devN`` to
674 ``X.Y.Z``, then deletes the ``.dev`` file. Every ordinary variant is
675 carried over.
677 An ``as_shipped`` fixture cannot be frozen: changing its embedded
678 development stamp would violate its byte-preservation contract, while
679 copying it unchanged under a final-version filename would make the stamp
680 and filename disagree. Such a fixture must be replaced with bytes from a
681 genuinely final-version shipped artifact before freezing.
683 All targets, source reads, and conflicts are checked before any file is
684 written or deleted, so a predictable validation or target conflict cannot
685 leave a partially frozen fixture tree.
687 This writes and deletes files; it never invokes version control. Staging
688 the addition and the deletion is left to the caller.
689 """
690 pending: list[tuple[Path, Path, str]] = []
691 fixtures = list(iter_schema_fixtures(directory))
692 for tree_cls in available_schema_classes(package):
693 if is_development_version(tree_cls.SCHEMA_VERSION):
694 continue
695 name = tree_cls.SCHEMA_NAME
696 live_version = fixture_version(tree_cls.SCHEMA_VERSION)
697 for fixture in fixtures:
698 if fixture.name != name or fixture.retired or not fixture.version.endswith(".dev"):
699 continue
700 if fixture.version.removesuffix(".dev") != live_version: 700 ↛ 701line 700 didn't jump to line 701 because the condition on line 700 was never true
701 continue
702 if fixture.is_canonical_twin: 702 ↛ 706line 702 didn't jump to line 706 because the condition on line 702 was never true
703 # Its as_shipped sibling below produces a clean error for the
704 # pair. An orphan twin is left for `check_schema_fixtures` to
705 # diagnose without mutating it.
706 continue
707 if fixture.is_as_shipped:
708 raise SchemaFixtureError(
709 f"{fixture.path.name} preserves development-version shipped bytes and cannot "
710 "be frozen without changing them; replace it with bytes from a final-version "
711 "shipped artifact."
712 )
713 target = fixture.path.with_name(_fixture_filename(name, live_version, fixture.variant))
714 if target.exists():
715 raise SchemaFixtureError(
716 f"{target.name} already exists; refusing to overwrite it while freezing "
717 f"{fixture.path.name}."
718 )
719 pending.append((target, fixture.path, canonical_fixture_text(read_fixture_tree(fixture))))
721 # Write all destinations before removing any sources. Preflight above
722 # handles expected failures; this ordering also avoids data loss if an
723 # unexpected filesystem error occurs during a write.
724 for target, _, text in pending:
725 target.write_text(text)
726 for _, source, _ in pending:
727 source.unlink()
728 return [(target, source) for target, source, _ in pending]
731def _values_equal(old: object, current: object) -> bool:
732 """Return whether two leaf values are equal, treating NaN as equal.
734 Types must match as well as values, so an integer in a field the current
735 model writes as a float is a disagreement rather than a coincidence.
736 """
737 if isinstance(old, float) and isinstance(current, float):
738 if math.isnan(old) and math.isnan(current):
739 return True
740 return type(old) is type(current) and old == current
743def _paths_and_values(value: object, path: str = "") -> dict[str, object]:
744 """Map every path a raw JSON-like value expresses to its value there.
746 A path is recorded for every dict key and list index encountered, not
747 only for leaves, so a query can find that a key exists (and inspect its
748 value) even when the read side gives it a different shape; see
749 `_is_expressed`. The root is excluded, matching the path syntax
750 `compare_fixture_versions` uses for problem messages.
751 """
752 values: dict[str, object] = {path: value} if path else {}
753 if isinstance(value, dict):
754 for key, sub in value.items():
755 values.update(_paths_and_values(sub, f"{path}.{key}"))
756 elif isinstance(value, list):
757 for index, item in enumerate(value):
758 values.update(_paths_and_values(item, f"{path}[{index}]"))
759 return values
762def _container_kind(value: object) -> str | None:
763 """Return ``"dict"``, ``"list"``, or `None` for a non-container value."""
764 if isinstance(value, dict):
765 return "dict"
766 if isinstance(value, list): 766 ↛ 768line 766 didn't jump to line 768 because the condition on line 766 was always true
767 return "list"
768 return None
771def _parent_path(path: str) -> str:
772 """Return the path one level up from ``path``, or ``""`` at the root."""
773 return path[: path.rindex("[")] if path.endswith("]") else path.rpartition(".")[0]
776def _is_expressed(path: str, on_disk_values: dict[str, object], dump_values: dict[str, object]) -> bool:
777 """Return whether the old file expresses ``path``.
779 An exact path match on disk always counts. Otherwise this finds the
780 nearest proper ancestor (excluding the root) present on disk, and counts
781 ``path`` as expressed only if that ancestor holds different kinds of
782 container on disk and in the old dump -- one a list, the other a dict.
784 That recovery is deliberately narrow, covering only a field whose spelling
785 changed because its container was reshaped, so that no exact path below it
786 can match (e.g. a pair collapsed from a list into named components).
787 Dropping the kind check would extend it to any path under any container
788 found on disk, which would wrongly compare a later-born field nested under
789 an unreshaped container -- the most likely way these schemas evolve.
790 """
791 if path in on_disk_values:
792 return True
793 ancestor = _parent_path(path)
794 while ancestor:
795 if ancestor in on_disk_values: 795 ↛ 797line 795 didn't jump to line 797 because the condition on line 795 was always true
796 return _container_kind(on_disk_values[ancestor]) != _container_kind(dump_values.get(ancestor))
797 ancestor = _parent_path(ancestor)
798 return False
801def _compare_expressed(
802 old: object,
803 current: object,
804 on_disk_values: dict[str, object],
805 dump_values: dict[str, object],
806 path: str,
807) -> list[str]:
808 """Recursive worker for `compare_fixture_versions`.
810 Identical to the public function's comparison logic, except that it
811 takes the already-computed on-disk and old-dump value maps instead of
812 deriving them, so each recursive call need not recompute them from an
813 ever-shrinking ``old`` subtree.
814 """
815 problems: list[str] = []
816 if isinstance(old, dict) and isinstance(current, dict):
817 for key in sorted(old):
818 child = f"{path}.{key}"
819 if not _is_expressed(child, on_disk_values, dump_values):
820 continue
821 if key not in current:
822 problems.append(f"{child}: in the older fixture but not the current one")
823 else:
824 problems.extend(
825 _compare_expressed(old[key], current[key], on_disk_values, dump_values, child)
826 )
827 elif isinstance(old, list) and isinstance(current, list):
828 if len(old) != len(current):
829 problems.append(f"{path}: list length {len(old)} != {len(current)}")
830 else:
831 for index, (o, c) in enumerate(zip(old, current, strict=True)):
832 problems.extend(_compare_expressed(o, c, on_disk_values, dump_values, f"{path}[{index}]"))
833 elif not _values_equal(old, current):
834 problems.append(f"{path}: {old!r} != {current!r}")
835 return problems
838_NOT_GIVEN = object()
839"""Sentinel marking an omitted ``on_disk`` argument, distinct from `None`."""
842def compare_fixture_versions(
843 old: object, current: object, *, path: str = "", on_disk: object = _NOT_GIVEN
844) -> list[str]:
845 """Compare an older fixture's read against the current-version fixture.
847 Parameters
848 ----------
849 old
850 Canonical dump of the older fixture, read under current code.
851 current
852 Canonical dump of the current-version fixture.
853 path
854 Path prefix used in problem messages; callers leave this empty.
855 on_disk
856 The older fixture's raw content as stored on disk, before model
857 validation. Comparing a real fixture means passing the parsed file
858 content here: this decides which paths the older file expressed, and
859 reading it through a model materializes every field at its default,
860 which would make later-born fields look present. The default of
861 ``old`` itself is correct only for a caller whose ``old`` never went
862 through validation, such as a test comparing literals.
864 Returns
865 -------
866 `list` [ `str` ]
867 One problem description per disagreement; empty when the older
868 fixture projects cleanly onto the current one.
870 Notes
871 -----
872 Both fixtures encode the same logical exemplar, which is allowed to grow
873 as versions add fields. Whether a path counts as one the older file
874 expresses is decided from ``on_disk``, not from ``old``: reading a
875 fixture through a pydantic model materializes every field, including
876 ones the file never mentioned, at its declared default, so deciding
877 "later-born" from ``old`` itself would treat every additive field as
878 already present whenever its default happened to match, and would then
879 reject any real, meaningful value chosen for it in the current exemplar.
881 A path present in both ``old`` and ``current`` and expressed by
882 ``on_disk`` must agree; a path in ``current`` that is not expressed is a
883 later-born field and is ignored; an expressed path missing from
884 ``current`` is a failure, because the older file said something the
885 current exemplar does not.
887 Whether an unmatched path is expressed is narrow by design; see
888 `_is_expressed`. It recovers only a reshaped container whose exact path
889 changed on disk, not any later-born field nested under an existing
890 container, so a migration that renames or restructures a field beyond
891 that one recovery is registered as expected divergence by its caller,
892 and its migration test asserts the morphed result directly instead.
893 """
894 reference = old if on_disk is _NOT_GIVEN else on_disk
895 on_disk_values = _paths_and_values(reference)
896 dump_values = _paths_and_values(old)
897 return _compare_expressed(old, current, on_disk_values, dump_values, path)