Coverage for tests/test_metadata.py: 99%
340 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-09-30 11:30 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14import copy
15import warnings
16from collections.abc import Mapping
17from pathlib import Path
19import astropy.io.fits
20import numpy as np
21import pytest
23from lsst.images import Box, Image, Mask, MaskedImage, MaskPlane, MaskSchema, MetadataView
24from lsst.images._metadata import _NativeMetadata
25from lsst.images.fits import ExtensionKey, FitsExternalMetadata, FitsOpaqueMetadata
26from lsst.images.serialization import EmptyExternalMetadata
27from lsst.images.tests import RoundtripFits, RoundtripNdf, reset_afw_mask_planes # noqa: F401
29try:
30 import h5py # noqa: F401
32 HAVE_H5PY = True
33except ImportError:
34 HAVE_H5PY = False
36EXTERNAL_KEYS = ["EXPTIME", "BGMEAN", "LSST ISR UNITS", "LOWER KEY", "NOVAL", "CPLX"]
39def _make_header() -> astropy.io.fits.Header:
40 """Return a primary header exercising repeated, HIERARCH, commentary,
41 valueless, and complex cards.
42 """
43 header = astropy.io.fits.Header()
44 header.append(("EXPTIME", 30.0), end=True)
45 header.append(("BGMEAN", 1.5), end=True)
46 header.append(("BGMEAN", 2.5), end=True)
47 with warnings.catch_warnings():
48 warnings.simplefilter("ignore", astropy.io.fits.verify.VerifyWarning)
49 header.append(("HIERARCH LSST ISR UNITS", "adu"), end=True)
50 # astropy preserves the case of HIERARCH keywords.
51 header.append(("HIERARCH lower key", 3), end=True)
52 header.append(("COMMENT", "a comment"), end=True)
53 header.append(("HISTORY", "a history"), end=True)
54 header.append(("", "blank"), end=True)
55 header.append(astropy.io.fits.Card.fromstring("NOVAL =".ljust(80)), end=True)
56 header.append(("CPLX", 1 + 2j), end=True)
57 return header
60def test_empty_external_metadata() -> None:
61 """Test that EmptyExternalMetadata behaves as an empty read-only
62 mapping.
63 """
64 external = EmptyExternalMetadata()
65 assert isinstance(external, Mapping)
66 assert len(external) == 0
67 assert list(external) == []
68 assert "EXPTIME" not in external
69 assert external.get("EXPTIME") is None
70 with pytest.raises(KeyError):
71 external["EXPTIME"]
72 with pytest.raises(KeyError):
73 external.get_all("EXPTIME")
74 assert repr(external) == "EmptyExternalMetadata({})"
77def test_fits_external_metadata_lookup() -> None:
78 """Test case-insensitive lookup, repeated keywords, HIERARCH keywords,
79 and value conversion.
80 """
81 external = FitsExternalMetadata(_make_header())
82 assert external["EXPTIME"] == 30.0
83 assert external["exptime"] == 30.0
84 assert external["lsst isr units"] == "adu"
85 assert external["HIERARCH LSST ISR UNITS"] == "adu"
86 assert external["LOWER KEY"] == 3
87 assert external["lower key"] == 3
88 assert external["NOVAL"] is None
89 assert external["CPLX"] == 1 + 2j
90 assert external["BGMEAN"] in (1.5, 2.5)
91 assert external.get_all("bgmean") == (1.5, 2.5)
92 assert external.get_all("exptime") == (30.0,)
93 assert external.get_all("NOVAL") == (None,)
94 assert external.get_all("Lower Key") == (3,)
95 assert "Exptime" in external
96 assert "lower key" in external
99def test_fits_external_metadata_hides_commentary_cards() -> None:
100 """Test that COMMENT, HISTORY, and blank cards are not visible."""
101 external = FitsExternalMetadata(_make_header())
102 for key in ("COMMENT", "comment", "HISTORY", ""):
103 assert key not in external
104 with pytest.raises(KeyError):
105 external[key]
106 with pytest.raises(KeyError):
107 external.get_all(key)
110def test_fits_external_metadata_odd_keys() -> None:
111 """Test that keys that cannot be FITS keywords are simply absent."""
112 external = FitsExternalMetadata(_make_header())
113 for key in ("roundtrip_test_1", "a=b", "é", "x" * 100):
114 assert key not in external
115 with pytest.raises(KeyError):
116 external[key]
117 assert 1 not in external
120def test_fits_external_metadata_iteration() -> None:
121 """Test that iteration yields each upper-case keyword once."""
122 external = FitsExternalMetadata(_make_header())
123 assert list(external) == EXTERNAL_KEYS
124 assert len(external) == len(EXTERNAL_KEYS)
125 assert repr(external).startswith("FitsExternalMetadata({'EXPTIME': 30.0")
128def test_fits_opaque_metadata_external_metadata_tracks_primary_header() -> None:
129 """Test that external metadata reflects a primary header added after it
130 was first requested.
131 """
132 opaque_metadata = FitsOpaqueMetadata()
133 assert len(opaque_metadata.external_metadata()) == 0
134 opaque_metadata.add_header(_make_header(), name="", ver=1)
135 assert list(opaque_metadata.external_metadata()) == EXTERNAL_KEYS
138def test_fits_external_metadata_empty() -> None:
139 """Test that a missing header behaves as an empty one."""
140 external = FitsExternalMetadata(None)
141 assert len(external) == 0
142 assert "EXPTIME" not in external
145def test_fits_opaque_metadata_external_metadata() -> None:
146 """Test that only the primary header is exposed."""
147 opaque_metadata = FitsOpaqueMetadata()
148 assert len(opaque_metadata.external_metadata()) == 0
149 opaque_metadata.add_header(_make_header(), name="", ver=1)
150 extension_header = astropy.io.fits.Header()
151 extension_header["EXTRA"] = 1
152 opaque_metadata.add_header(extension_header, name="IMAGE", ver=1)
153 external = opaque_metadata.external_metadata()
154 assert list(external) == EXTERNAL_KEYS
155 assert "EXTRA" not in external
156 assert opaque_metadata.headers[ExtensionKey("IMAGE")]["EXTRA"] == 1
159def test_extract_legacy_primary_header_strips_native_cards() -> None:
160 """Test that the cards holding native metadata in a legacy file are
161 returned as native metadata and not kept in the opaque header.
162 """
163 header = astropy.io.fits.Header()
164 header.append(("PLATFORM", "lsstcam"), end=True)
165 header.append(("HIERARCH LSST IMAGES KEY 1", "native_key"), end=True)
166 header.append(("HIERARCH LSST IMAGES VALUE 1", 7), end=True)
167 header.append(("HIERARCH LSST IMAGES KEY 2", "MixedCase"), end=True)
168 header.append(("HIERARCH LSST IMAGES VALUE 2", "yes"), end=True)
169 opaque_metadata = FitsOpaqueMetadata()
170 assert opaque_metadata.extract_legacy_primary_header(header) == {"native_key": 7, "MixedCase": "yes"}
171 stored = opaque_metadata.headers[ExtensionKey()]
172 assert stored["PLATFORM"] == "lsstcam"
173 assert not [keyword for keyword in stored if keyword.startswith("LSST IMAGES")]
174 assert list(opaque_metadata.external_metadata()) == ["PLATFORM"]
177def test_legacy_readers_restore_native_metadata(
178 tmp_path: Path,
179 reset_afw_mask_planes: None, # noqa: F811
180) -> None:
181 """Test that every legacy reader restores native metadata from the
182 ``LSST IMAGES`` cards and does not keep those cards as opaque metadata.
183 """
184 from lsst.daf.base import PropertyList
186 masked_image = MaskedImage(
187 Image(1.0, shape=(4, 5), dtype=np.float32),
188 mask_schema=MaskSchema([MaskPlane("BAD", "Pixel is bad.")]),
189 metadata={"native_key": 7, "MixedCase": "yes"},
190 )
191 legacy_metadata = PropertyList()
192 masked_image._fill_legacy_metadata(legacy_metadata)
193 path = tmp_path / "legacy_masked_image.fits"
194 masked_image.to_legacy().writeFits(str(path), metadata=legacy_metadata)
195 with astropy.io.fits.open(path) as hdu_list:
196 assert hdu_list[0].header["LSST IMAGES KEY 1"] == "native_key"
197 results = {
198 "MaskedImage": MaskedImage.read_legacy(path),
199 "MaskedImage image component": MaskedImage.read_legacy(path, component="image"),
200 "MaskedImage mask component": MaskedImage.read_legacy(path, component="mask"),
201 "Image": Image.read_legacy(path),
202 "Mask": Mask.read_legacy(path, ext=2),
203 }
204 for label, result in results.items():
205 assert result._metadata == {"native_key": 7, "MixedCase": "yes"}, label
206 opaque_header = result._opaque_metadata.headers[ExtensionKey()]
207 assert not [keyword for keyword in opaque_header if keyword.startswith("LSST IMAGES")], label
210def _make_view(data: dict) -> MetadataView:
211 external = FitsExternalMetadata(_make_header())
212 return MetadataView(_NativeMetadata(data, external), external)
215def test_native_metadata_shadowing() -> None:
216 """Test that new native keys may not shadow external keys, while keys
217 already present may be updated.
218 """
219 external = FitsExternalMetadata(_make_header())
220 data: dict = {"exptime": 1.0}
221 native = _NativeMetadata(data, external)
222 native["exptime"] = 2.0
223 assert data == {"exptime": 2.0}
224 for key in ("EXPTIME", "Bgmean", "lsst isr units", "LOWER KEY", "noval"):
225 with pytest.raises(KeyError):
226 native[key] = 3.0
227 with pytest.raises(KeyError):
228 native.update({"cplx": 1})
229 with pytest.raises(KeyError):
230 native.setdefault("NOVAL", 1)
231 native["fresh"] = 1
232 del native["fresh"]
233 assert data == {"exptime": 2.0}
234 assert dict(native) == {"exptime": 2.0}
235 assert len(native) == 1
238def test_native_metadata_accepts_non_fits_keys() -> None:
239 """Test that keys that cannot be FITS keywords are accepted."""
240 external = FitsExternalMetadata(_make_header())
241 data: dict = {}
242 native = _NativeMetadata(data, external)
243 for key in ("roundtrip_test_1", "a=b", "é", "MixedCaseKey"):
244 native[key] = 1
245 assert set(data) == {"roundtrip_test_1", "a=b", "é", "MixedCaseKey"}
248def test_metadata_view_lookup() -> None:
249 """Test that native keys are found first with exact case, then external
250 keys case-insensitively.
251 """
252 view = _make_view({"native_key": 7, "exptime": 1.0})
253 assert view["native_key"] == 7
254 assert view["exptime"] == 1.0
255 assert view["EXPTIME"] == 30.0
256 assert view["ExpTime"] == 30.0
257 assert view["BGMEAN"] in (1.5, 2.5)
258 assert view["noval"] is None
259 assert "lsst isr units" in view
260 assert view.get("missing") is None
261 assert view.get_all("BGMEAN") == (1.5, 2.5)
262 assert view.get_all("exptime") == (1.0,)
263 assert view.get_all("EXPTIME") == (30.0,)
264 for key in ("COMMENT", "HISTORY", ""):
265 assert key not in view
266 with pytest.raises(KeyError):
267 view["missing"]
268 with pytest.raises(KeyError):
269 view.get_all("missing")
272def test_metadata_view_iteration() -> None:
273 """Test that iteration is the exact-case union of both sources and
274 agrees with lookup.
275 """
276 view = _make_view({"native_key": 7, "exptime": 1.0})
277 keys = list(view)
278 assert sorted(keys) == sorted(["native_key", "exptime", *EXTERNAL_KEYS])
279 assert len(view) == len(keys) == len(set(keys))
280 as_dict = dict(view)
281 assert as_dict["exptime"] == 1.0
282 assert as_dict["EXPTIME"] == 30.0
283 for key in keys:
284 assert as_dict[key] == view[key]
285 assert view == as_dict
288def test_metadata_view_writes() -> None:
289 """Test that writes and deletes go to native and respect shadowing."""
290 data: dict = {"native_key": 7}
291 view = _make_view(data)
292 view["new"] = 1
293 assert data == {"native_key": 7, "new": 1}
294 with pytest.raises(KeyError):
295 view["cplx"] = 1
296 with pytest.raises(KeyError):
297 del view["EXPTIME"]
298 with pytest.raises(KeyError):
299 view.pop("EXPTIME")
300 assert view.pop("missing", None) is None
301 assert view.pop("new") == 1
302 view |= {"another": 2}
303 assert data == {"native_key": 7, "another": 2}
304 assert view.popitem() in {("native_key", 7), ("another", 2)}
305 assert len(data) == 1
306 view.clear()
307 assert data == {}
308 assert view["EXPTIME"] == 30.0
311def test_metadata_view_dict_operations() -> None:
312 """Test the dict-like operations that the view provides."""
313 data: dict = {"native_key": 7}
314 view = _make_view(data)
315 copied = view.copy()
316 assert type(copied) is dict
317 assert copied == data
318 assert copied is not data
319 shallow = copy.copy(view)
320 assert type(shallow) is dict
321 assert shallow == data
322 merged = view | {"z": 1}
323 assert type(merged) is dict
324 assert merged == {**dict(view), "z": 1}
325 reverse_merged = {"z": 1} | view
326 assert type(reverse_merged) is dict
327 assert reverse_merged == {"z": 1, **dict(view)}
328 assert view.native is not None
329 assert list(view.external) == EXTERNAL_KEYS
332def _make_image(metadata: dict | None = None) -> Image:
333 """Return a small image whose opaque metadata holds `_make_header`."""
334 image = Image(0.0, shape=(4, 5), dtype=np.float32, metadata=metadata)
335 opaque_metadata = FitsOpaqueMetadata()
336 opaque_metadata.add_header(_make_header(), name="", ver=1)
337 image._opaque_metadata = opaque_metadata
338 return image
341def test_image_metadata_view() -> None:
342 """Test the metadata view on an image with external metadata."""
343 image = _make_image({"native_key": 7})
344 assert isinstance(image.metadata, MetadataView)
345 assert image.metadata["native_key"] == 7
346 assert image.metadata["exptime"] == 30.0
347 assert image.metadata.native == {"native_key": 7}
348 assert list(image.metadata.external) == EXTERNAL_KEYS
349 image.metadata["other"] = "x"
350 assert image.metadata.native == {"native_key": 7, "other": "x"}
351 with pytest.raises(KeyError):
352 image.metadata["ExpTime"] = 1.0
353 with pytest.raises(KeyError):
354 image.metadata.native["ExpTime"] = 1.0
355 with pytest.raises(KeyError):
356 image.metadata.update({"bgmean": 1.0})
359def test_image_metadata_without_opaque_metadata() -> None:
360 """Test that an in-memory image has empty external metadata."""
361 image = Image(0.0, shape=(4, 5), dtype=np.float32, metadata={"a": 1})
362 assert len(image.metadata.external) == 0
363 assert image.metadata == {"a": 1}
364 image.metadata["EXPTIME"] = 2.0
365 assert image.metadata.native == {"a": 1, "EXPTIME": 2.0}
368def test_image_metadata_view_reflects_later_opaque_metadata() -> None:
369 """Test that the view sees opaque metadata attached after
370 construction.
371 """
372 image = Image(0.0, shape=(4, 5), dtype=np.float32, metadata={"exptime": 1.0})
373 opaque_metadata = FitsOpaqueMetadata()
374 opaque_metadata.add_header(_make_header(), name="", ver=1)
375 image._opaque_metadata = opaque_metadata
376 assert image.metadata["exptime"] == 1.0
377 assert image.metadata["EXPTIME"] == 30.0
378 image.metadata["exptime"] = 2.0
379 assert image.metadata.native == {"exptime": 2.0}
382def test_subimage_metadata() -> None:
383 """Test that subimages share native metadata and see the same external
384 metadata.
385 """
386 image = _make_image({"native_key": 7})
387 subimage = image[Box.factory[0:2, 0:3]]
388 subimage.metadata["new"] = 1
389 assert image.metadata["new"] == 1
390 assert subimage.metadata["EXPTIME"] == 30.0
391 copied = image.copy()
392 copied.metadata["copied_only"] = 1
393 assert "copied_only" not in image.metadata
394 assert copied.metadata["EXPTIME"] == 30.0
397def test_constructor_with_metadata_view() -> None:
398 """Test that a metadata view passed to a constructor shares the native
399 dict rather than being nested.
400 """
401 source = _make_image({"a": 1})
402 # A view is accepted at runtime even though the annotation asks for a
403 # dict.
404 image = Image(0.0, shape=(4, 5), dtype=np.float32, metadata=source.metadata) # type: ignore[arg-type]
405 assert type(image._metadata) is dict
406 assert image._metadata is source._metadata
407 assert image.metadata.native == {"a": 1}
408 assert "EXPTIME" not in image.metadata
411def test_metadata_setter() -> None:
412 """Test that the setter replaces native metadata with a copy, without a
413 shadowing check, and leaves external metadata readable.
414 """
415 image = _make_image()
416 image.metadata = {"exptime": 1.0}
417 assert image.metadata["exptime"] == 1.0
418 assert image.metadata["EXPTIME"] == 30.0
419 assigned: dict = {"s": 1}
420 image.metadata = assigned
421 assigned["t"] = 2
422 assert "t" not in image.metadata
423 image.metadata["u"] = 3
424 assert assigned == {"s": 1, "t": 2}
425 other = _make_image()
426 other.metadata = image.metadata
427 assert other.metadata.native == {"s": 1, "u": 3}
428 assert "EXPTIME" in other.metadata
429 other.metadata["v"] = 4
430 assert "v" not in image.metadata
433@pytest.mark.parametrize("disconnect", ["empty", "copy", "self"])
434def test_metadata_setter_disconnects_subimage(disconnect: str) -> None:
435 """Test that a subimage shares native metadata with its parent in both
436 directions until its metadata is assigned, which disconnects it while
437 keeping external metadata readable.
438 """
439 image = _make_image({"a": 1})
440 subimage = image[image.bbox]
441 subimage.metadata["b"] = 2
442 image.metadata["c"] = 3
443 assert image.metadata.native == subimage.metadata.native == {"a": 1, "b": 2, "c": 3}
444 if disconnect == "empty":
445 subimage.metadata = {}
446 expected = {}
447 else:
448 if disconnect == "copy":
449 subimage.metadata = subimage.metadata.copy()
450 else:
451 subimage.metadata = subimage.metadata
452 expected = {"a": 1, "b": 2, "c": 3}
453 assert subimage.metadata.native == expected
454 subimage.metadata["from_subimage"] = 4
455 image.metadata["from_parent"] = 5
456 assert "from_subimage" not in image.metadata
457 assert "from_parent" not in subimage.metadata
458 assert subimage.metadata["EXPTIME"] == 30.0
461def _make_masked_image() -> MaskedImage:
462 """Return a small masked image with native metadata and opaque metadata
463 holding `_make_header`.
464 """
465 masked_image = MaskedImage(
466 Image(1.0, shape=(4, 5), dtype=np.float32),
467 mask_schema=MaskSchema([MaskPlane("BAD", "Pixel is bad.")]),
468 metadata={"native_key": 7, "MixedCase": "yes"},
469 )
470 opaque_metadata = FitsOpaqueMetadata()
471 opaque_metadata.add_header(_make_header(), name="", ver=1)
472 masked_image._opaque_metadata = opaque_metadata
473 return masked_image
476def _check_round_trip(result: MaskedImage) -> None:
477 assert result.metadata.native == {"native_key": 7, "MixedCase": "yes"}
478 assert list(result.metadata.external) == EXTERNAL_KEYS
479 assert result.metadata["exptime"] == 30.0
480 assert result.metadata.get_all("BGMEAN") == (1.5, 2.5)
481 assert result.metadata["NOVAL"] is None
482 assert result.metadata["CPLX"] == 1 + 2j
483 assert result.metadata["lower key"] == 3
484 assert "COMMENT" not in result.metadata
487def test_external_metadata_fits_round_trip() -> None:
488 """Test that native and external metadata survive a FITS round trip."""
489 with RoundtripFits(_make_masked_image()) as roundtrip:
490 pass
491 _check_round_trip(roundtrip.result)
494@pytest.mark.skipif(not HAVE_H5PY, reason="h5py is not installed")
495def test_external_metadata_ndf_round_trip() -> None:
496 """Test that native and external metadata survive an NDF round trip."""
497 with RoundtripNdf(_make_masked_image()) as roundtrip:
498 pass
499 _check_round_trip(roundtrip.result)