Coverage for python/lsst/daf/butler/registry/wildcards.py: 65%
180 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-09-29 09:13 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "CategorizedWildcard",
31 "CollectionWildcard",
32 "DatasetTypeWildcard",
33)
35import contextlib
36import dataclasses
37import re
38import warnings
39from collections.abc import Callable, Iterable, Mapping
40from types import EllipsisType
41from typing import Any
43from lsst.utils.iteration import ensure_iterable
45from .._dataset_type import DatasetType
46from .._exceptions import DatasetTypeExpressionError
47from ..utils import globToRegex
48from ._exceptions import CollectionExpressionError
51@dataclasses.dataclass
52class CategorizedWildcard:
53 """The results of preprocessing a wildcard expression to separate match
54 patterns from strings.
56 The `fromExpression` method should almost always be used to construct
57 instances, as the regular constructor performs no checking of inputs (and
58 that can lead to confusing error messages downstream).
59 """
61 @classmethod
62 def fromExpression(
63 cls,
64 expression: Any,
65 *,
66 allowAny: bool = True,
67 allowPatterns: bool = True,
68 coerceUnrecognized: Callable[[Any], tuple[str, Any] | str] | None = None,
69 coerceItemValue: Callable[[Any], Any] | None = None,
70 defaultItemValue: Any | None = None,
71 ) -> CategorizedWildcard | EllipsisType:
72 """Categorize a wildcard expression.
74 Parameters
75 ----------
76 expression : `~typing.Any`
77 The expression to categorize. May be any of:
79 - `str` (including glob patterns if ``allowPatterns`` is `True`);
80 - `re.Pattern` (only if ``allowPatterns`` is `True`);
81 - objects recognized by ``coerceUnrecognized`` (if provided);
82 - two-element tuples of (`str`, value) where value is recognized
83 by ``coerceItemValue`` (if provided);
84 - a non-`str`, non-mapping iterable containing any of the above;
85 - the special value ``...`` (only if ``allowAny`` is `True`),
86 which matches anything;
87 - a mapping from `str` to a value are recognized by
88 ``coerceItemValue`` (if provided);
89 - a `CategorizedWildcard` instance (passed through unchanged if
90 it meets the requirements specified by keyword arguments).
91 allowAny : `bool`, optional
92 If `False` (`True` is default) raise `TypeError` if ``...`` is
93 encountered.
94 allowPatterns : `bool`, optional
95 If `False` (`True` is default) raise `TypeError` if a `re.Pattern`
96 is encountered, or if ``expression`` is a `CategorizedWildcard`
97 with `patterns` not empty.
98 coerceUnrecognized : `~collections.abc.Callable`, optional
99 A callback that takes a single argument of arbitrary type and
100 returns either a `str` - appended to `strings` - or a `tuple` of
101 (`str`, `typing.Any`) to be appended to `items`. This will be
102 called on objects of unrecognized type. Exceptions will be reraised
103 as `TypeError` (and chained).
104 coerceItemValue : `~collections.abc.Callable`, optional
105 If provided, ``expression`` may be a mapping from `str` to any
106 type that can be passed to this function; the result of that call
107 will be stored instead as the value in ``self.items``.
108 defaultItemValue : `typing.Any`, optional
109 If provided, combine this value with any string values encountered
110 (including any returned by ``coerceUnrecognized``) to form a
111 `tuple` and add it to `items`, guaranteeing that `strings` will be
112 empty. Patterns are never added to `items`.
114 Returns
115 -------
116 categorized : `CategorizedWildcard` or ``...``.
117 The struct describing the wildcard. ``...`` is passed through
118 unchanged.
120 Raises
121 ------
122 TypeError
123 Raised if an unsupported type is found in the expression.
124 """
125 assert expression is not None
126 # See if we were given ...; just return that if we were.
127 if expression is ...:
128 if not allowAny: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true
129 raise TypeError("This expression may not be unconstrained.")
130 return ...
131 if isinstance(expression, cls): 131 ↛ 134line 131 didn't jump to line 134 because the condition on line 131 was never true
132 # This is already a CategorizedWildcard. Make sure it meets the
133 # reqs. implied by the kwargs we got.
134 if not allowPatterns and expression.patterns:
135 raise TypeError(
136 f"Regular expression(s) {expression.patterns} are not allowed in this context."
137 )
138 if defaultItemValue is not None and expression.strings:
139 if expression.items:
140 raise TypeError(
141 "Incompatible preprocessed expression: an ordered sequence of str is "
142 "needed, but the original order was lost in the preprocessing."
143 )
144 return cls(
145 strings=[],
146 patterns=expression.patterns,
147 items=[(k, defaultItemValue) for k in expression.strings],
148 )
149 elif defaultItemValue is None and expression.items:
150 if expression.strings:
151 raise TypeError(
152 "Incompatible preprocessed expression: an ordered sequence of items is "
153 "needed, but the original order was lost in the preprocessing."
154 )
155 return cls(strings=[k for k, _ in expression.items], patterns=expression.patterns, items=[])
156 else:
157 # Original expression was created with keyword arguments that
158 # were at least as restrictive as what we just got; pass it
159 # through.
160 return expression
162 # If we get here, we know we'll be creating a new instance.
163 # Initialize an empty one now.
164 self = cls(strings=[], patterns=[], items=[])
166 # If mappings are allowed, see if we were given a single mapping by
167 # trying to get items.
168 if coerceItemValue is not None: 168 ↛ 169line 168 didn't jump to line 169 because the condition on line 168 was never true
169 rawItems = None
170 with contextlib.suppress(AttributeError):
171 rawItems = expression.items()
173 if rawItems is not None:
174 for k, v in rawItems:
175 try:
176 self.items.append((k, coerceItemValue(v)))
177 except Exception as err:
178 raise TypeError(f"Could not coerce mapping value '{v}' for key '{k}'.") from err
179 return self
181 # Not ..., a CategorizedWildcard instance, or a mapping. Just
182 # process scalars or an iterable. We put the body of the loop inside
183 # a local function so we can recurse after coercion.
185 def process(element: Any, alreadyCoerced: bool = False) -> EllipsisType | None:
186 was_string = False
187 if isinstance(element, str):
188 was_string = True
189 if defaultItemValue is not None: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 self.items.append((element, defaultItemValue))
191 return None
192 else:
193 # This returns a list but we know we only passed in
194 # single value.
195 converted = globToRegex(element)
196 if converted is ...:
197 return ...
198 element = converted[0]
199 # Let regex and ... go through to the next check
200 if isinstance(element, str):
201 self.strings.append(element)
202 return None
203 if allowPatterns and isinstance(element, re.Pattern):
204 if not was_string:
205 warnings.warn(
206 "Regular expressions should no longer be used in collection or dataset type searches."
207 " Use globs ('*' wildcards) instead. Will be removed after v28.",
208 FutureWarning,
209 )
210 self.patterns.append(element)
211 return None
212 if alreadyCoerced:
213 try:
214 k, v = element
215 except TypeError:
216 raise TypeError(
217 f"Object '{element!r}' returned by coercion function must be `str` or `tuple`."
218 ) from None
219 else:
220 self.items.append((k, v))
221 return None
222 if coerceItemValue is not None: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true
223 try:
224 k, v = element
225 except TypeError:
226 pass
227 else:
228 if not isinstance(k, str):
229 raise TypeError(f"Item key '{k}' is not a string.")
230 try:
231 v = coerceItemValue(v)
232 except Exception as err:
233 raise TypeError(f"Could not coerce tuple item value '{v}' for key '{k}'.") from err
234 self.items.append((k, v))
235 return None
236 if coerceUnrecognized is not None: 236 ↛ 244line 236 didn't jump to line 244 because the condition on line 236 was always true
237 try:
238 # This should be safe but flake8 cant tell that the
239 # function will be re-declared next function call
240 process(coerceUnrecognized(element), alreadyCoerced=True) # noqa: F821
241 except Exception as err:
242 raise TypeError(f"Could not coerce expression element '{element!r}'.") from err
243 else:
244 extra = "."
245 if isinstance(element, re.Pattern):
246 extra = " and patterns are not allowed."
247 raise TypeError(f"Unsupported object in wildcard expression: '{element!r}'{extra}")
248 return None
250 for element in ensure_iterable(expression):
251 retval = process(element)
252 if retval is ...:
253 # One of the globs matched everything
254 if not allowAny: 254 ↛ 255line 254 didn't jump to line 255 because the condition on line 254 was never true
255 raise TypeError("This expression may not be unconstrained.")
256 return ...
257 del process
258 return self
260 strings: list[str]
261 """Explicit string values found in the wildcard (`list` [ `str` ]).
262 """
264 patterns: list[re.Pattern]
265 """Regular expression patterns found in the wildcard
266 (`list` [ `re.Pattern` ]).
267 """
269 items: list[tuple[str, Any]]
270 """Two-item tuples that relate string values to other objects
271 (`list` [ `tuple` [ `str`, `typing.Any` ] ]).
272 """
275@dataclasses.dataclass(frozen=True)
276class CollectionWildcard:
277 """A validated wildcard for collection names.
279 The `from_expression` method should almost always be used to construct
280 instances, as the regular constructor performs no checking of inputs (and
281 that can lead to confusing error messages downstream).
283 Notes
284 -----
285 `CollectionWildcard` is expected to be rarely used outside of `Registry`
286 (which uses it to back several of its "query" methods that take general
287 expressions for collections), but it may occasionally be useful outside
288 `Registry` as a way to preprocess expressions that contain single-pass
289 iterators into a form that can be used to call those `Registry` methods
290 multiple times.
291 """
293 strings: tuple[str, ...] = ()
294 """An an ordered list of explicitly-named collections. (`tuple` [ `str` ]).
295 """
297 patterns: tuple[re.Pattern, ...] | EllipsisType = ...
298 """Regular expression patterns to match against collection names, or the
299 special value ``...`` indicating all collections.
301 ``...`` must be accompanied by ``strings=()``.
302 """
304 def __post_init__(self) -> None:
305 if self.patterns is ... and self.strings: 305 ↛ 306line 305 didn't jump to line 306 because the condition on line 305 was never true
306 raise ValueError(
307 f"Collection wildcard matches any string, but still has explicit strings {self.strings}."
308 )
310 @classmethod
311 def from_expression(cls, expression: Any, require_ordered: bool = False) -> CollectionWildcard:
312 """Process a general expression to construct a `CollectionWildcard`
313 instance.
315 Parameters
316 ----------
317 expression : `~typing.Any`
318 May be:
320 - a `str` collection name;
321 - an `re.Pattern` instance to match (with `re.Pattern.fullmatch`)
322 against collection names;
323 - any iterable containing any of the above;
324 - another `CollectionWildcard` instance (passed through unchanged).
326 Duplicate collection names will be removed (preserving the first
327 appearance of each collection name).
328 require_ordered : `bool`, optional
329 If `True` (`False` is default) require the expression to be
330 ordered, and raise `CollectionExpressionError` if it is not.
332 Returns
333 -------
334 wildcard : `CollectionWildcard`
335 A `CollectionWildcard` instance.
337 Raises
338 ------
339 CollectionExpressionError
340 Raised if the patterns has regular expression, glob patterns, or
341 the ``...`` wildcard, and ``require_ordered=True``.
342 """
343 if isinstance(expression, cls): 343 ↛ 344line 343 didn't jump to line 344 because the condition on line 343 was never true
344 return expression
345 if expression is ...:
346 return cls()
347 wildcard = CategorizedWildcard.fromExpression(
348 expression,
349 allowAny=True,
350 allowPatterns=True,
351 )
352 if wildcard is ...:
353 return cls()
354 result = cls(
355 strings=tuple(wildcard.strings),
356 patterns=tuple(wildcard.patterns),
357 )
358 if require_ordered: 358 ↛ 359line 358 didn't jump to line 359 because the condition on line 358 was never true
359 result.require_ordered()
360 return result
362 @classmethod
363 def from_names(cls, names: Iterable[str]) -> CollectionWildcard:
364 """Construct from an iterable of explicit collection names.
366 Parameters
367 ----------
368 names : `~collections.abc.Iterable` [ `str` ]
369 Iterable of collection names.
371 Returns
372 -------
373 wildcard : `CollectionWildcard`
374 A `CollectionWildcard` instance. `require_ordered` is guaranteed
375 to succeed and return the given names in order.
376 """
377 return cls(strings=tuple(names), patterns=())
379 def require_ordered(self) -> tuple[str, ...]:
380 """Require that this wildcard contains no patterns, and return the
381 ordered tuple of names that it does hold.
383 Returns
384 -------
385 names : `tuple` [ `str` ]
386 Ordered tuple of collection names.
388 Raises
389 ------
390 CollectionExpressionError
391 Raised if the patterns has regular expression, glob patterns, or
392 the ``...`` wildcard.
393 """
394 if self.patterns: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 raise CollectionExpressionError(
396 f"An ordered collection expression is required; got patterns {self.patterns}."
397 )
398 return self.strings
400 def empty(self) -> bool:
401 """Return true if both ``strings`` and ``patterns`` are empty."""
402 # bool(Ellipsis) is True
403 return not self.strings and not self.patterns
405 def __str__(self) -> str:
406 if self.patterns is ...: 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true
407 return "..."
408 else:
409 terms = list(self.strings)
410 terms.extend(str(p) for p in self.patterns)
411 return "[{}]".format(", ".join(terms))
414@dataclasses.dataclass
415class DatasetTypeWildcard:
416 """A validated expression that resolves to one or more dataset types.
418 The `from_expression` method should almost always be used to construct
419 instances, as the regular constructor performs no checking of inputs (and
420 that can lead to confusing error messages downstream).
421 """
423 values: Mapping[str, DatasetType | None] = dataclasses.field(default_factory=dict)
424 """A mapping with `str` dataset type name keys and optional `DatasetType`
425 instances.
426 """
428 patterns: tuple[re.Pattern, ...] | EllipsisType = ...
429 """Regular expressions to be matched against dataset type names, or the
430 special value ``...`` indicating all dataset types.
432 Any pattern matching a dataset type is considered an overall match for
433 the expression.
434 """
436 @classmethod
437 def from_expression(cls, expression: Any) -> DatasetTypeWildcard:
438 """Construct an instance by analyzing the given expression.
440 Parameters
441 ----------
442 expression : `~typing.Any`
443 Expression to analyze. May be any of the following:
445 - a `str` dataset type name;
446 - a `DatasetType` instance;
447 - an iterable whose elements may be any of the above (any dataset
448 type matching any element in the list is an overall match);
449 - an existing `DatasetTypeWildcard` instance;
450 - the special ``...`` ellipsis object, which matches any dataset
451 type.
453 Returns
454 -------
455 query : `DatasetTypeWildcard`
456 An instance of this class (new unless an existing instance was
457 passed in).
459 Raises
460 ------
461 DatasetTypeExpressionError
462 Raised if the given expression does not have one of the allowed
463 types.
464 """
465 if isinstance(expression, cls):
466 return expression
467 # CategorizedWildcard currently allows globs and regex as patterns
468 # but RFC-879 drops support for regex in dataset type specifications.
469 # Therefore check for their presence.
470 for exp in ensure_iterable(expression):
471 if isinstance(exp, re.Pattern):
472 raise DatasetTypeExpressionError("Regular expressions are not supported.")
473 try:
474 wildcard = CategorizedWildcard.fromExpression(
475 expression,
476 coerceUnrecognized=lambda d: (d.name, d),
477 )
478 except TypeError as err:
479 raise DatasetTypeExpressionError(f"Invalid dataset type expression: {expression!r}.") from err
480 if wildcard is ...:
481 return cls()
482 values: dict[str, DatasetType | None] = {}
483 for name in wildcard.strings:
484 values[name] = None
485 for name, item in wildcard.items:
486 if not isinstance(item, DatasetType): 486 ↛ 487line 486 didn't jump to line 487 because the condition on line 486 was never true
487 raise DatasetTypeExpressionError(
488 f"Invalid value '{item}' of type {type(item)} in dataset type expression; "
489 "expected str, re.Pattern, DatasetType objects, iterables thereof, or '...'."
490 )
491 values[name] = item
492 return cls(values, patterns=tuple(wildcard.patterns))
494 def __str__(self) -> str:
495 if self.patterns is ...:
496 return "..."
497 else:
498 terms = list(self.values.keys())
499 terms.extend(str(p) for p in self.patterns)
500 return "[{}]".format(", ".join(terms))