Coverage for python/lsst/daf/butler/tests/_repo_template_cache.py: 88%
143 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-25 22:03 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = [
31 "TemplateCacheStats",
32 "clear_repo_template_cache",
33 "make_repo_for_test",
34 "template_cache_stats",
35]
37import atexit
38import dataclasses
39import hashlib
40import json
41import os
42import shutil
43import tempfile
44import urllib.parse
45from typing import Any
47import pydantic
49from lsst.resources import ResourcePath, ResourcePathExpression
50from lsst.resources.file import FileResourcePath
52from .. import Butler, Config
53from ..dimensions import DimensionConfig
54from ..repo_relocation import replaceRoot
56# The environment variable that alters which default configuration files are
57# found, and therefore what a given input configuration expands to.
58_CONFIG_PATH_ENV = "DAF_BUTLER_CONFIG_PATH"
61class TemplateCacheStats(pydantic.BaseModel):
62 """Counts of cache activity, for tests and diagnostics."""
64 served: int = 0
65 """Requests handled by this helper."""
67 bypassed: int = 0
68 """Requests that went straight to `lsst.daf.butler.Butler.makeRepo`."""
70 config_templates: int = 0
71 """Configurations written from scratch and retained for reuse."""
73 reused_config: int = 0
74 """Requests whose ``butler.yaml`` was copied from an earlier identical
75 one."""
77 templates: int = 0
78 """Databases actually built."""
80 reused_database: int = 0
81 """Requests whose database was copied from an earlier identical one."""
84@dataclasses.dataclass(frozen=True)
85class _ConfigTemplate:
86 """A retained repository configuration, ready to be copied."""
88 path: str
89 """Path to a pristine copy of ``butler.yaml``."""
91 config: Config
92 """The configuration that repository creation returned.
94 This is retained alongside the file rather than re-read from it because
95 the two differ: the obscore manager configuration is deliberately stripped
96 before writing, since it is stored in the registry instead, but registry
97 creation still needs it.
98 """
101# Whole-configuration hash -> the configuration it produces.
102_configs: dict[str, _ConfigTemplate] = {}
103# Registry-and-dimensions hash -> path to a pristine copy of the database.
104_databases: dict[str, str] = {}
105_tmpdirs: list[str] = []
106_stats = TemplateCacheStats()
109def template_cache_stats() -> TemplateCacheStats:
110 """Return counts of cache activity, for tests and diagnostics.
112 Returns
113 -------
114 stats : `TemplateCacheStats`
115 A snapshot of the counters. Later activity does not change it.
116 """
117 return _stats.model_copy()
120def clear_repo_template_cache() -> None:
121 """Discard all cached templates and reset the statistics."""
122 global _stats
124 for directory in _tmpdirs:
125 shutil.rmtree(directory, ignore_errors=True)
126 _tmpdirs.clear()
127 _configs.clear()
128 _databases.clear()
129 _stats = TemplateCacheStats()
132atexit.register(clear_repo_template_cache)
135def _is_cacheable_registry(config: Config | None) -> bool:
136 """Return whether this repository's registry can be served from a copy.
138 Parameters
139 ----------
140 config : `lsst.daf.butler.Config` or `None`
141 Repository configuration, or `None` to accept the defaults.
143 Returns
144 -------
145 cacheable : `bool`
146 `True` if the registry lives in a SQLite file inside the repository,
147 which is the only case a directory copy can reproduce.
149 Notes
150 -----
151 A client/server database such as PostgreSQL keeps its contents outside the
152 repository directory, so copying the directory does not copy the registry.
153 Such repositories also carry a per-repository ``namespace``, which makes
154 every configuration unique and every cache lookup a miss. Caching them
155 would build a template that is used exactly once and then retained, which
156 is strictly more work than creating the repository directly.
157 """
158 if config is None:
159 # The default registry is SQLite inside the repository.
160 return True
161 db = config.get(("registry", "db"))
162 if db is None:
163 return True
164 return str(db).startswith("sqlite")
167def make_repo_for_test(
168 root: ResourcePathExpression,
169 config: Config | str | None = None,
170 dimensionConfig: Config | str | None = None,
171 standalone: bool = False,
172 searchPaths: list[str] | None = None,
173 forceConfigRoot: bool = True,
174 outfile: ResourcePathExpression | None = None,
175 overwrite: bool = False,
176) -> Config:
177 """Create a test repository, reusing a cached template when possible.
179 The parameters and return value match
180 `lsst.daf.butler.Butler.makeRepo`. The first request for a given
181 configuration builds a real repository; later requests copy it, which is
182 substantially cheaper.
184 Parameters
185 ----------
186 root : `lsst.resources.ResourcePathExpression`
187 Path to the root location of the new repository.
188 config : `lsst.daf.butler.Config` or `str`, optional
189 Configuration to write to the repository.
190 dimensionConfig : `lsst.daf.butler.Config` or `str`, optional
191 Configuration for dimensions.
192 standalone : `bool`, optional
193 If `True`, write all expanded defaults. Bypasses the cache.
194 searchPaths : `list` [`str`], optional
195 Directory paths to search when calculating the full configuration.
196 Bypasses the cache.
197 forceConfigRoot : `bool`, optional
198 If `False`, values present in ``config`` that would normally be reset
199 are not overridden.
200 outfile : `lsst.resources.ResourcePathExpression`, optional
201 Path at which to write the config. Bypasses the cache.
202 overwrite : `bool`, optional
203 If `True`, allow an existing config to be overwritten. Bypasses the
204 cache.
206 Returns
207 -------
208 config : `lsst.daf.butler.Config`
209 The configuration of the new repository.
211 Raises
212 ------
213 FileExistsError
214 Raised if the repository already has a configuration and ``overwrite``
215 is `False`.
217 Notes
218 -----
219 This helper is for test code only. Production code, and any test that
220 asserts on the behavior of repository creation itself rather than on its
221 result, must call `lsst.daf.butler.Butler.makeRepo` directly.
222 """
223 if isinstance(config, str):
224 # Read the file now rather than treating its name as the identity of
225 # its contents: tests rewrite temporary configuration files in place,
226 # and a cache keyed on the pathname would serve the stale version.
227 # This is the same conversion repository creation performs.
228 config = Config(config)
230 # RemoteTestResourcePath subclasses FileResourcePath and reports
231 # isLocal=False while remaining backed by a local path, so isLocal is the
232 # wrong question to ask here.
233 copyable = isinstance(ResourcePath(root, forceDirectory=True), FileResourcePath)
235 usable = (
236 copyable
237 and _is_cacheable_registry(config)
238 and outfile is None
239 and not standalone
240 and not overwrite
241 and not searchPaths
242 )
244 if not usable:
245 _stats.bypassed += 1
246 return Butler.makeRepo(
247 root,
248 config=config,
249 dimensionConfig=dimensionConfig,
250 standalone=standalone,
251 searchPaths=searchPaths,
252 forceConfigRoot=forceConfigRoot,
253 outfile=outfile,
254 overwrite=overwrite,
255 )
257 # Phase one: the repository directory and its butler.yaml. This depends on
258 # the whole configuration, so it is cached on a hash of all of it. The
259 # written file is root-independent because paths are stored against the
260 # repository root tag, so a copy is valid anywhere.
261 written, root_uri = _make_butler_config(root, config, forceConfigRoot)
263 # Phase two: the database. Only the registry and dimension configurations
264 # affect its contents, so it is cached on those alone and copied into
265 # place.
266 db_key = _database_key(written, dimensionConfig)
267 db_path = _sqlite_path(written, root_uri)
268 if db_key is None or db_path is None: 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 _stats.served += 1
270 Butler._make_repo_registry(written, dimensionConfig=dimensionConfig, root_uri=root_uri)
271 return written
273 cached_db = _databases.get(db_key)
274 if cached_db is None:
275 _stats.templates += 1
276 Butler._make_repo_registry(written, dimensionConfig=dimensionConfig, root_uri=root_uri)
277 if not os.path.exists(db_path): 277 ↛ 282line 277 didn't jump to line 282 because the condition on line 277 was never true
278 # Registry creation put the database somewhere other than where
279 # this helper expects it, so there is nothing safe to retain. The
280 # repository itself is complete, so report it as served and leave
281 # the database uncached rather than failing.
282 _stats.served += 1
283 return written
284 holder = tempfile.mkdtemp(prefix="butler-registry-template-")
285 _tmpdirs.append(holder)
286 cached_db = os.path.join(holder, os.path.basename(db_path))
287 shutil.copyfile(db_path, cached_db)
288 _databases[db_key] = cached_db
289 else:
290 _stats.reused_database += 1
291 shutil.copyfile(cached_db, db_path)
293 _stats.served += 1
294 return written
297def _make_butler_config(
298 root: ResourcePathExpression,
299 config: Config | None,
300 forceConfigRoot: bool,
301) -> tuple[Config, ResourcePath]:
302 """Write the repository's ``butler.yaml``, reusing an identical one.
304 Parameters
305 ----------
306 root : `lsst.resources.ResourcePathExpression`
307 Path to the root location of the new repository.
308 config : `lsst.daf.butler.Config` or `None`
309 Repository configuration.
310 forceConfigRoot : `bool`
311 Whether root-dependent options are overridden.
313 Returns
314 -------
315 written : `lsst.daf.butler.Config`
316 The configuration written to the repository.
317 root_uri : `lsst.resources.ResourcePath`
318 The root of the new repository.
320 Raises
321 ------
322 FileExistsError
323 Raised if the repository already has a configuration.
324 """
325 key = _config_key(config, forceConfigRoot)
326 cached = _configs.get(key) if key is not None else None
327 if cached is None:
328 written, root_uri = Butler._make_repo_butler_config(
329 root, config=config, forceConfigRoot=forceConfigRoot
330 )
331 if key is not None: 331 ↛ 340line 331 didn't jump to line 340 because the condition on line 331 was always true
332 _stats.config_templates += 1
333 holder = tempfile.mkdtemp(prefix="butler-config-template-")
334 _tmpdirs.append(holder)
335 path = os.path.join(holder, "butler.yaml")
336 shutil.copyfile(os.path.join(root_uri.ospath, "butler.yaml"), path)
337 # Retain a copy so that a caller mutating the returned
338 # configuration cannot reach the template.
339 _configs[key] = _ConfigTemplate(path=path, config=written.copy())
340 return written, root_uri
342 _stats.reused_config += 1
343 root_uri = ResourcePath(root, forceDirectory=True)
344 root_uri.mkdir()
345 destination = ResourcePath(os.path.join(root_uri.ospath, "butler.yaml"), forceDirectory=False)
346 # Exclusive creation reproduces the FileExistsError that writing the
347 # configuration would raise, since only overwrite=False reaches here.
348 with open(cached.path, "rb") as source, open(destination.ospath, "xb") as target:
349 shutil.copyfileobj(source, target)
350 written = cached.config.copy()
351 written.configFile = destination
352 return written, root_uri
355def _config_key(config: Config | None, forceConfigRoot: bool) -> str | None:
356 """Return a key covering everything that affects ``butler.yaml``.
358 Parameters
359 ----------
360 config : `lsst.daf.butler.Config` or `None`
361 Repository configuration.
362 forceConfigRoot : `bool`
363 Whether root-dependent options are overridden.
365 Returns
366 -------
367 key : `str` or `None`
368 A hash of the inputs, or `None` if they cannot be rendered
369 deterministically.
371 Notes
372 -----
373 Two inputs beyond the configuration itself change what is written, so both
374 take part in the key. ``forceConfigRoot`` decides whether root-dependent
375 values in the supplied configuration survive into the file, and
376 ``DAF_BUTLER_CONFIG_PATH`` decides which default configuration files the
377 supplied one is expanded against. Only a handful of tests vary either, so
378 including them costs a cache miss in those tests and nothing elsewhere.
379 """
380 try:
381 rendered = json.dumps(
382 [
383 config.toDict() if config is not None else None,
384 forceConfigRoot,
385 os.environ.get(_CONFIG_PATH_ENV),
386 ],
387 sort_keys=True,
388 default=str,
389 )
390 except (TypeError, ValueError):
391 return None
392 return hashlib.sha256(rendered.encode()).hexdigest()
395def _database_key(written: Config, dimensionConfig: Config | str | None) -> str | None:
396 """Return a key covering everything that affects the database contents.
398 Parameters
399 ----------
400 written : `lsst.daf.butler.Config`
401 The repository configuration that was written to ``butler.yaml``.
402 dimensionConfig : `lsst.daf.butler.Config` or `str` or `None`
403 Dimension universe configuration.
405 Returns
406 -------
407 key : `str` or `None`
408 A hash of the registry and dimension configurations, or `None` if
409 they cannot be rendered deterministically.
411 Notes
412 -----
413 Datastore configuration, storage classes and other sections do not reach
414 the database, so they are deliberately excluded. The ``db`` entry is also
415 excluded because it only names the file's location, which differs between
416 repositories that are otherwise identical.
417 """
418 try:
419 registry = dict(written["registry"].toDict())
420 registry.pop("db", None)
421 rendered = json.dumps(
422 [registry, _dimension_key_material(dimensionConfig), os.environ.get(_CONFIG_PATH_ENV)],
423 sort_keys=True,
424 default=str,
425 )
426 except Exception:
427 # Any failure here means the inputs cannot be identified cheaply, and
428 # the caller falls back to creating the database directly. Letting the
429 # exception out would report it from key derivation rather than from
430 # the registry creation that will raise it again in context.
431 return None
432 return hashlib.sha256(rendered.encode()).hexdigest()
435def _dimension_key_material(dimensionConfig: Config | str | None) -> Any:
436 """Return the part of a cache key that identifies the dimension universe.
438 Parameters
439 ----------
440 dimensionConfig : `lsst.daf.butler.Config` or `str` or `None`
441 Dimension universe configuration, as passed to repository creation.
443 Returns
444 -------
445 material : `object`
446 A JSON-serializable description of the configuration.
448 Notes
449 -----
450 `None` contributes nothing, because the defaults it selects are determined
451 by the configuration search path, which the key covers separately.
452 """
453 if dimensionConfig is None:
454 return None
455 if isinstance(dimensionConfig, Config): 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true
456 return dimensionConfig.toDict()
457 # A pathname says nothing about the file's contents, and a relative name is
458 # resolved against the configuration search path, so expand it exactly as
459 # registry creation will.
460 return DimensionConfig(dimensionConfig).toDict()
463def _sqlite_path(written: Config, root_uri: ResourcePath) -> str | None:
464 """Return the local path of the repository's SQLite file, if it has one.
466 Parameters
467 ----------
468 written : `lsst.daf.butler.Config`
469 The repository configuration that was written to ``butler.yaml``.
470 root_uri : `lsst.resources.ResourcePath`
471 Root of the repository, substituted for the repository root tag.
473 Returns
474 -------
475 path : `str` or `None`
476 Path to the SQLite file, or `None` if the registry is not a SQLite
477 file inside the repository.
479 Notes
480 -----
481 The location is derived the way the registry derives it, in two steps:
482 `lsst.daf.butler.repo_relocation.replaceRoot` substitutes the repository
483 root, then the result is parsed as a URI, which is what
484 ``SqliteDatabase.makeEngine`` does to find the file it opens.
486 Reconstructing the path instead of following those two steps gives the
487 wrong answer whenever the root holds a URI metacharacter, because
488 ``replaceRoot`` substitutes a root whose ``#`` fragment has already been
489 dropped and the parse then discards everything from a ``?`` onwards. Such
490 a root is mangled by repository creation itself, and this helper has to
491 land on the same mangled path rather than on the one the caller asked for.
492 """
493 db = written.get(("registry", "db"))
494 if db is None: 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true
495 return None
496 resolved = replaceRoot(str(db), root_uri)
497 parsed = urllib.parse.urlparse(resolved)
498 if parsed.scheme != "sqlite" or not parsed.path.startswith("/"): 498 ↛ 499line 498 didn't jump to line 499 because the condition on line 498 was never true
499 return None
500 location = parsed.path[1:]
501 if not location or location == ":memory:": 501 ↛ 502line 501 didn't jump to line 502 because the condition on line 501 was never true
502 return None
503 return location