Coverage for python/lsst/daf/butler/_storage_class.py: 94%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for Storage Classes.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("StorageClass", "StorageClassConfig", "StorageClassFactory") 

33 

34import builtins 

35import hashlib 

36import itertools 

37import logging 

38import os 

39from collections import ChainMap 

40from collections.abc import Callable, Collection, Mapping, Sequence, Set 

41from threading import RLock 

42from typing import Any 

43 

44import pydantic 

45import pydantic_core 

46 

47from lsst.utils import doImportType 

48from lsst.utils.classes import Singleton 

49from lsst.utils.introspection import get_full_type_name 

50 

51from ._config import Config, ConfigSubset 

52from ._config_support import LookupKey 

53from ._storage_class_delegate import StorageClassDelegate 

54 

55log = logging.getLogger(__name__) 

56 

57 

58class StorageClassConfig(ConfigSubset): 

59 """Configuration class for defining Storage Classes.""" 

60 

61 component = "storageClasses" 

62 defaultConfigFile = "storageClasses.yaml" 

63 

64 

65# Cache of storage class definitions that have already been derived from a 

66# given configuration, keyed by a hash of that configuration. Deriving them is 

67# far more expensive than recognizing that we have seen the same definitions 

68# before, and identical definitions are derived repeatedly because every 

69# Butler applies the same defaults. 

70# 

71# Sharing instances is safe because a `StorageClass` describes a definition 

72# rather than holding state: the only mutation it undergoes is memoization of 

73# the Python type and of the converters whose types cannot be imported, both 

74# of which are determined by the definition itself. The factory is a singleton 

75# and already shares instances between every use of a given definition. 

76_derived_cache: dict[str, dict[str, StorageClass]] = {} 

77 

78# The environment variable that alters which default configuration files are 

79# found, and therefore what a given input configuration expands to. 

80_CONFIG_PATH_ENV = "DAF_BUTLER_CONFIG_PATH" 

81 

82 

83def _derived_cache_key(config: StorageClassConfig | Config | str) -> str | None: 

84 """Return a key identifying the definitions this configuration produces. 

85 

86 Parameters 

87 ---------- 

88 config : `StorageClassConfig`, `Config` or `str` 

89 Storage class configuration, as passed to 

90 `StorageClassFactory.addFromConfig`. 

91 

92 Returns 

93 ------- 

94 key : `str` or `None` 

95 A hash of the storage class definitions in ``config`` together with 

96 the environment that determines which defaults are applied, or `None` 

97 if a key cannot be derived cheaply. 

98 

99 Notes 

100 ----- 

101 The key is computed from the supplied configuration without expanding it, 

102 because expanding it is the cost this is trying to avoid. 

103 

104 Serialization does not sort keys, so two configurations holding the same 

105 definitions in a different order produce different keys. That costs a 

106 cache miss and the definitions are then derived as usual, which is 

107 correct but slower; it never produces a wrong answer. 

108 """ 

109 if isinstance(config, StorageClassConfig): 

110 # Already a subset, so the data are the definitions themselves. 

111 raw: Any = config._data 

112 elif isinstance(config, Config): 

113 # A configuration that may carry overrides under the component key. 

114 # Its absence is meaningful: it means the defaults apply unmodified. 

115 raw = config._data.get(StorageClassConfig.component, {}) 

116 else: 

117 # A file path or URI would have to be read to be hashed, so leave it 

118 # to the uncached path. 

119 return None 

120 

121 try: 

122 rendered = pydantic_core.to_json([raw, os.environ.get(_CONFIG_PATH_ENV)], fallback=str) 

123 except (TypeError, ValueError): 

124 return None 

125 return hashlib.sha256(rendered).hexdigest() 

126 

127 

128class _StorageClassModel(pydantic.BaseModel): 

129 """Model class used to validate storage class configuration.""" 

130 

131 pytype: str | None = None 

132 inheritsFrom: str | None = None 

133 components: dict[str, str] = pydantic.Field(default_factory=dict) 

134 derivedComponents: dict[str, str] = pydantic.Field(default_factory=dict) 

135 parameters: list[str] = pydantic.Field(default_factory=list) 

136 delegate: str | None = None 

137 converters: dict[str, str] = pydantic.Field(default_factory=dict) 

138 

139 

140class StorageClass: 

141 """Class describing how a label maps to a particular Python type. 

142 

143 Parameters 

144 ---------- 

145 name : `str` 

146 Name to use for this class. 

147 pytype : `type` or `str` 

148 Python type (or name of type) to associate with the `StorageClass`. 

149 components : `dict`, optional 

150 `dict` mapping name of a component to another `StorageClass`. 

151 derivedComponents : `dict`, optional 

152 `dict` mapping name of a derived component to another `StorageClass`. 

153 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

154 Parameters understood by this `StorageClass` that can control 

155 reading of data from datastores. 

156 delegate : `str`, optional 

157 Fully qualified name of class supporting assembly and disassembly 

158 of a `pytype` instance. 

159 converters : `dict` [`str`, `str`], optional 

160 Mapping of python type to function that can be called to convert 

161 that python type to the valid type of this storage class. 

162 """ 

163 

164 def __init__( 

165 self, 

166 name: str = "", 

167 pytype: type | str | None = None, 

168 components: dict[str, StorageClass] | None = None, 

169 derivedComponents: dict[str, StorageClass] | None = None, 

170 parameters: Sequence[str] | Set[str] | None = None, 

171 delegate: str | None = None, 

172 converters: dict[str, str] | None = None, 

173 ): 

174 # Merge converters with class defaults. 

175 self._converters = {} 

176 if converters: 

177 self._converters.update(converters) 

178 

179 # Version of converters where the python types have been 

180 # Do not try to import anything until needed. 

181 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

182 

183 self.name = name 

184 

185 if pytype is None: 

186 pytype = object 

187 

188 self._pytype: type | None 

189 if not isinstance(pytype, str): 

190 # Already have a type so store it and get the name 

191 self._pytypeName = get_full_type_name(pytype) 

192 self._pytype = pytype 

193 else: 

194 # Store the type name and defer loading of type 

195 self._pytypeName = pytype 

196 self._pytype = None 

197 

198 if components is not None: 

199 if len(components) == 1: 

200 raise ValueError( 

201 f"Composite storage class {name} is not allowed to have" 

202 f" only one component '{next(iter(components))}'." 

203 " Did you mean it to be a derived component?" 

204 ) 

205 self._components = components 

206 else: 

207 self._components = {} 

208 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

209 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

210 # if the delegate is not None also set it and clear the default 

211 # delegate 

212 self._delegate: type | None 

213 self._delegateClassName: str | None 

214 if delegate is not None: 

215 self._delegateClassName = delegate 

216 self._delegate = None 

217 elif components is not None: 

218 # We set a default delegate for composites so that a class is 

219 # guaranteed to support something if it is a composite. 

220 log.debug("Setting default delegate for %s", self.name) 

221 self._delegate = StorageClassDelegate 

222 self._delegateClassName = get_full_type_name(self._delegate) 

223 else: 

224 self._delegate = None 

225 self._delegateClassName = None 

226 

227 @property 

228 def components(self) -> Mapping[str, StorageClass]: 

229 """Return the components associated with this `StorageClass`.""" 

230 return self._components 

231 

232 @property 

233 def derivedComponents(self) -> Mapping[str, StorageClass]: 

234 """Return derived components associated with `StorageClass`.""" 

235 return self._derivedComponents 

236 

237 @property 

238 def converters(self) -> Mapping[str, str]: 

239 """Return the type converters supported by this `StorageClass`.""" 

240 return self._converters 

241 

242 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

243 """Return the type converters as python types.""" 

244 if self._converters_by_type is None: 

245 self._converters_by_type = {} 

246 

247 # Loop over list because the dict can be edited in loop. 

248 for candidate_type_str, converter_str in list(self.converters.items()): 

249 if hasattr(builtins, candidate_type_str): 

250 candidate_type = getattr(builtins, candidate_type_str) 

251 else: 

252 try: 

253 candidate_type = doImportType(candidate_type_str) 

254 except ImportError as e: 

255 log.warning( 

256 "Unable to import type %s associated with storage class %s (%s)", 

257 candidate_type_str, 

258 self.name, 

259 e, 

260 ) 

261 del self._converters[candidate_type_str] 

262 continue 

263 

264 if hasattr(builtins, converter_str): 

265 converter = getattr(builtins, converter_str) 

266 else: 

267 try: 

268 converter = doImportType(converter_str) 

269 except ImportError as e: 

270 log.warning( 

271 "Unable to import conversion function %s associated with storage class %s " 

272 "required to convert type %s (%s)", 

273 converter_str, 

274 self.name, 

275 candidate_type_str, 

276 e, 

277 ) 

278 del self._converters[candidate_type_str] 

279 continue 

280 if not callable(converter): 280 ↛ 286line 280 didn't jump to line 286 because the condition on line 280 was never true

281 # doImportType is annotated to return a Type but in actual 

282 # fact it can return Any except ModuleType because package 

283 # variables can be accessed. This make mypy believe it 

284 # is impossible for the return value to not be a callable 

285 # so we must ignore the warning. 

286 log.warning( # type: ignore 

287 "Conversion function %s associated with storage class " 

288 "%s to convert type %s is not a callable.", 

289 converter_str, 

290 self.name, 

291 candidate_type_str, 

292 ) 

293 del self._converters[candidate_type_str] 

294 continue 

295 self._converters_by_type[candidate_type] = converter 

296 return self._converters_by_type 

297 

298 @property 

299 def parameters(self) -> set[str]: 

300 """Return `set` of names of supported parameters.""" 

301 return set(self._parameters) 

302 

303 @property 

304 def pytype(self) -> type: 

305 """Return Python type associated with this `StorageClass`.""" 

306 if self._pytype is not None: 

307 return self._pytype 

308 

309 if hasattr(builtins, self._pytypeName): 

310 pytype = getattr(builtins, self._pytypeName) 

311 else: 

312 pytype = doImportType(self._pytypeName) 

313 self._pytype = pytype 

314 return self._pytype 

315 

316 @property 

317 def delegateClass(self) -> type | None: 

318 """Class to use to delegate type-specific actions.""" 

319 if self._delegate is not None: 

320 return self._delegate 

321 if self._delegateClassName is None: 

322 return None 

323 delegate_class = doImportType(self._delegateClassName) 

324 self._delegate = delegate_class 

325 return self._delegate 

326 

327 def allComponents(self) -> Mapping[str, StorageClass]: 

328 """Return all defined components. 

329 

330 This mapping includes all the derived and read/write components 

331 for the corresponding storage class. 

332 

333 Returns 

334 ------- 

335 comp : `dict` of [`str`, `StorageClass`] 

336 The component name to storage class mapping. 

337 """ 

338 return ChainMap(self._components, self._derivedComponents) 

339 

340 def delegate(self) -> StorageClassDelegate: 

341 """Return an instance of a storage class delegate. 

342 

343 Returns 

344 ------- 

345 delegate : `StorageClassDelegate` 

346 Instance of the delegate associated with this `StorageClass`. 

347 The delegate is constructed with this `StorageClass`. 

348 

349 Raises 

350 ------ 

351 TypeError 

352 This StorageClass has no associated delegate. 

353 """ 

354 cls = self.delegateClass 

355 if cls is None: 

356 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

357 return cls(storageClass=self) 

358 

359 def isComposite(self) -> bool: 

360 """Return Boolean indicating whether this is a composite or not. 

361 

362 Returns 

363 ------- 

364 isComposite : `bool` 

365 `True` if this `StorageClass` is a composite, `False` 

366 otherwise. 

367 """ 

368 if self.components: 

369 return True 

370 return False 

371 

372 def _lookupNames(self) -> tuple[LookupKey, ...]: 

373 """Keys to use when looking up this DatasetRef in a configuration. 

374 

375 The names are returned in order of priority. 

376 

377 Returns 

378 ------- 

379 names : `tuple` of `LookupKey` 

380 Tuple of a `LookupKey` using the `StorageClass` name. 

381 """ 

382 return (LookupKey(name=self.name),) 

383 

384 def knownParameters(self) -> set[str]: 

385 """Return set of all parameters known to this `StorageClass`. 

386 

387 The set includes parameters understood by components of a composite. 

388 

389 Returns 

390 ------- 

391 known : `set` 

392 All parameter keys of this `StorageClass` and the component 

393 storage classes. 

394 """ 

395 known = set(self._parameters) 

396 for sc in self.components.values(): 

397 known.update(sc.knownParameters()) 

398 return known 

399 

400 def validateParameters(self, parameters: Collection | None = None) -> None: 

401 """Check that the parameters are known to this `StorageClass`. 

402 

403 Does not check the values. 

404 

405 Parameters 

406 ---------- 

407 parameters : `~collections.abc.Collection`, optional 

408 Collection containing the parameters. Can be `dict`-like or 

409 `set`-like. The parameter values are not checked. 

410 If no parameters are supplied, always returns without error. 

411 

412 Raises 

413 ------ 

414 KeyError 

415 Some parameters are not understood by this `StorageClass`. 

416 """ 

417 # No parameters is always okay 

418 if not parameters: 

419 return 

420 

421 # Extract the important information into a set. Works for dict and 

422 # list. 

423 external = set(parameters) 

424 

425 diff = external - self.knownParameters() 

426 if diff: 

427 s = "s" if len(diff) > 1 else "" 

428 unknown = "', '".join(diff) 

429 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

430 

431 def filterParameters( 

432 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

433 ) -> Mapping[str, Any]: 

434 """Filter out parameters that are not known to this `StorageClass`. 

435 

436 Parameters 

437 ---------- 

438 parameters : `~collections.abc.Mapping`, optional 

439 Candidate parameters. Can be `None` if no parameters have 

440 been provided. 

441 subset : `~collections.abc.Collection`, optional 

442 Subset of supported parameters that the caller is interested 

443 in using. The subset must be known to the `StorageClass` 

444 if specified. If `None` the supplied parameters will all 

445 be checked, else only the keys in this set will be checked. 

446 

447 Returns 

448 ------- 

449 filtered : `~collections.abc.Mapping` 

450 Valid parameters. Empty `dict` if none are suitable. 

451 

452 Raises 

453 ------ 

454 ValueError 

455 Raised if the provided subset is not a subset of the supported 

456 parameters or if it is an empty set. 

457 """ 

458 if not parameters: 

459 return {} 

460 

461 known = self.knownParameters() 

462 

463 if subset is not None: 

464 if not subset: 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true

465 raise ValueError("Specified a parameter subset but it was empty") 

466 subset = set(subset) 

467 if not subset.issubset(known): 467 ↛ 468line 467 didn't jump to line 468 because the condition on line 467 was never true

468 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

469 wanted = subset 

470 else: 

471 wanted = known 

472 

473 return {k: parameters[k] for k in wanted if k in parameters} 

474 

475 def validateInstance(self, instance: Any) -> bool: 

476 """Check that the supplied Python object has the expected Python type. 

477 

478 Parameters 

479 ---------- 

480 instance : `object` 

481 Object to check. 

482 

483 Returns 

484 ------- 

485 isOk : `bool` 

486 True if the supplied instance object can be handled by this 

487 `StorageClass`, False otherwise. 

488 """ 

489 return isinstance(instance, self.pytype) 

490 

491 def is_type(self, other: type, compare_types: bool = False) -> bool: 

492 """Return Boolean indicating whether the supplied type matches 

493 the type in this `StorageClass`. 

494 

495 Parameters 

496 ---------- 

497 other : `type` 

498 The type to be checked. 

499 compare_types : `bool`, optional 

500 If `True` the python type will be used in the comparison 

501 if the type names do not match. This may trigger an import 

502 of code and so can be slower. 

503 

504 Returns 

505 ------- 

506 match : `bool` 

507 `True` if the types are equal. 

508 

509 Notes 

510 ----- 

511 If this `StorageClass` has not yet imported the Python type the 

512 check is done against the full type name, this prevents an attempt 

513 to import the type when it will likely not match. 

514 """ 

515 if self._pytype: 

516 return self._pytype is other 

517 

518 other_name = get_full_type_name(other) 

519 if self._pytypeName == other_name: 

520 return True 

521 

522 if compare_types: 

523 # Must protect against the import failing. 

524 try: 

525 return self.pytype is other 

526 except Exception: 

527 pass 

528 

529 return False 

530 

531 def can_convert(self, other: StorageClass) -> bool: 

532 """Return `True` if this storage class can convert python types 

533 in the other storage class. 

534 

535 Parameters 

536 ---------- 

537 other : `StorageClass` 

538 The storage class to check. 

539 

540 Returns 

541 ------- 

542 can : `bool` 

543 `True` if this storage class has a registered converter for 

544 the python type associated with the other storage class. That 

545 converter will convert the other python type to the one associated 

546 with this storage class. 

547 """ 

548 if other.name == self.name: 

549 # Identical storage classes are compatible. 

550 return True 

551 

552 # It may be that the storage class being compared is not 

553 # available because the python type can't be imported. In that 

554 # case conversion must be impossible. 

555 try: 

556 other_pytype = other.pytype 

557 except Exception: 

558 return False 

559 

560 # Or even this storage class itself can not have the type imported. 

561 try: 

562 self_pytype = self.pytype 

563 except Exception: 

564 return False 

565 

566 if issubclass(other_pytype, self_pytype): 

567 # Storage classes have different names but the same python type. 

568 return True 

569 

570 for candidate_type in self._get_converters_by_type(): 

571 if issubclass(other_pytype, candidate_type): 

572 return True 

573 return False 

574 

575 def coerce_type(self, incorrect: Any) -> Any: 

576 """Coerce the supplied incorrect instance to the python type 

577 associated with this `StorageClass`. 

578 

579 Parameters 

580 ---------- 

581 incorrect : `object` 

582 An object that might be the incorrect type. 

583 

584 Returns 

585 ------- 

586 correct : `object` 

587 An object that matches the python type of this `StorageClass`. 

588 Can be the same object as given. If `None`, `None` will be 

589 returned. 

590 

591 Raises 

592 ------ 

593 TypeError 

594 Raised if no conversion can be found. 

595 """ 

596 if incorrect is None: 

597 return None 

598 

599 # Possible this is the correct type already. 

600 if self.validateInstance(incorrect): 

601 return incorrect 

602 

603 # Check each registered converter. 

604 for candidate_type, converter in self._get_converters_by_type().items(): 

605 if isinstance(incorrect, candidate_type): 

606 try: 

607 return converter(incorrect) 

608 except Exception: 

609 log.error( 

610 "Converter %s failed to convert type %s", 

611 get_full_type_name(converter), 

612 get_full_type_name(incorrect), 

613 ) 

614 raise 

615 raise TypeError( 

616 "Type does not match and no valid converter found to convert" 

617 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

618 ) 

619 

620 def __eq__(self, other: Any) -> bool: 

621 """Equality checks name, pytype name, delegate name, and components.""" 

622 if not isinstance(other, StorageClass): 

623 return NotImplemented 

624 

625 if self.name != other.name: 

626 return False 

627 

628 # We must compare pytype and delegate by name since we do not want 

629 # to trigger an import of external module code here 

630 if self._delegateClassName != other._delegateClassName: 

631 return False 

632 if self._pytypeName != other._pytypeName: 

633 return False 

634 

635 # Ensure we have the same component keys in each 

636 if set(self.components.keys()) != set(other.components.keys()): 

637 return False 

638 

639 # Same parameters 

640 if self.parameters != other.parameters: 

641 return False 

642 

643 # Ensure that all the components have the same type 

644 return all(self.components[k] == other.components[k] for k in self.components) 

645 

646 def __hash__(self) -> int: 

647 return hash(self.name) 

648 

649 def __repr__(self) -> str: 

650 optionals: dict[str, Any] = {} 

651 if self._pytypeName != "object": 651 ↛ 653line 651 didn't jump to line 653 because the condition on line 651 was always true

652 optionals["pytype"] = self._pytypeName 

653 if self._delegateClassName is not None: 

654 optionals["delegate"] = self._delegateClassName 

655 if self._parameters: 655 ↛ 656line 655 didn't jump to line 656 because the condition on line 655 was never true

656 optionals["parameters"] = self._parameters 

657 if self.components: 

658 optionals["components"] = self.components 

659 if self.converters: 

660 optionals["converters"] = self.converters 

661 

662 # order is preserved in the dict 

663 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

664 

665 # Start with mandatory fields 

666 r = f"{self.__class__.__name__}({self.name!r}" 

667 if options: 667 ↛ 669line 667 didn't jump to line 669 because the condition on line 667 was always true

668 r = r + ", " + options 

669 r = r + ")" 

670 return r 

671 

672 def __str__(self) -> str: 

673 return self.name 

674 

675 

676class StorageClassFactory(metaclass=Singleton): 

677 """Factory for `StorageClass` instances. 

678 

679 This class is a singleton, with each instance sharing the pool of 

680 StorageClasses. Since code can not know whether it is the first 

681 time the instance has been created, the constructor takes no arguments. 

682 To populate the factory with storage classes, a call to 

683 `~StorageClassFactory.addFromConfig()` should be made. 

684 

685 Parameters 

686 ---------- 

687 config : `StorageClassConfig` or `str`, optional 

688 Load configuration. In a ButlerConfig` the relevant configuration 

689 is located in the ``storageClasses`` section. 

690 """ 

691 

692 def __init__(self, config: StorageClassConfig | str | None = None): 

693 self._storageClasses: dict[str, StorageClass] = {} 

694 self._lock = RLock() 

695 

696 # Always seed with the default config 

697 self.addFromConfig(StorageClassConfig()) 

698 

699 if config is not None: 699 ↛ 700line 699 didn't jump to line 700 because the condition on line 699 was never true

700 self.addFromConfig(config) 

701 

702 def __str__(self) -> str: 

703 """Return summary of factory. 

704 

705 Returns 

706 ------- 

707 summary : `str` 

708 Summary of the factory status. 

709 """ 

710 with self._lock: 

711 sep = "\n" 

712 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

713 

714StorageClasses 

715-------------- 

716{sep.join(f"{self._storageClasses[s]!r}" for s in sorted(self._storageClasses))} 

717""" 

718 

719 def __contains__(self, storageClassOrName: object) -> bool: 

720 with self._lock: 

721 if isinstance(storageClassOrName, str): 

722 return storageClassOrName in self._storageClasses 

723 elif isinstance(storageClassOrName, StorageClass): 

724 return storageClassOrName.name in self._storageClasses 

725 return False 

726 

727 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

728 """Add more `StorageClass` definitions from a config file. 

729 

730 Parameters 

731 ---------- 

732 config : `StorageClassConfig`, `Config` or `str` 

733 Storage class configuration. Can contain a ``storageClasses`` 

734 key if part of a global configuration. 

735 

736 Notes 

737 ----- 

738 Definitions derived from a configuration that has been seen before are 

739 reused. Registration itself, including the check that an existing 

740 definition is compatible, is always performed. 

741 """ 

742 cache_key = _derived_cache_key(config) 

743 if cache_key is not None and (derived := _derived_cache.get(cache_key)) is not None: 

744 with self._lock: 

745 for storageClass in derived.values(): 

746 self.registerStorageClass(storageClass) 

747 return 

748 

749 sconfig = StorageClassConfig(config) 

750 

751 # Since we can not assume that we will get definitions of 

752 # components or parents before their classes are defined 

753 # we have a helper function that we can call recursively 

754 # to extract definitions from the configuration. 

755 def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> StorageClass: 

756 # This might have already been processed through recursion, or 

757 # already present in the factory. 

758 if name not in _sconfig: 

759 return self.getStorageClass(name) 

760 try: 

761 model = _StorageClassModel.model_validate(_sconfig.pop(name)) 

762 except Exception as err: 

763 err.add_note(msg) 

764 raise 

765 components: dict[str, StorageClass] = {} 

766 derivedComponents: dict[str, StorageClass] = {} 

767 parameters: set[str] = set() 

768 delegate: str | None = None 

769 converters: dict[str, str] = {} 

770 if model.inheritsFrom is not None: 

771 base = processStorageClass(model.inheritsFrom, _sconfig, msg + f"; processing base of {name}") 

772 pytype = base._pytypeName 

773 components.update(base.components) 

774 derivedComponents.update(base.derivedComponents) 

775 parameters.update(base.parameters) 

776 delegate = base._delegateClassName 

777 converters.update(base.converters) 

778 if model.pytype is not None: 

779 pytype = model.pytype 

780 for k, v in model.components.items(): 

781 components[k] = processStorageClass( 

782 v, _sconfig, msg + f"; processing component {k} of {name}" 

783 ) 

784 for k, v in model.derivedComponents.items(): 

785 derivedComponents[k] = processStorageClass( 

786 v, _sconfig, msg + f"; processing derivedCmponent {k} of {name}" 

787 ) 

788 parameters.update(model.parameters) 

789 if model.delegate is not None: 

790 delegate = model.delegate 

791 converters.update(model.converters) 

792 result = StorageClass( 

793 name=name, 

794 pytype=pytype, 

795 components=components, 

796 derivedComponents=derivedComponents, 

797 parameters=parameters, 

798 delegate=delegate, 

799 converters=converters, 

800 ) 

801 self.registerStorageClass(result, msg=msg) 

802 return result 

803 

804 # In case there is a problem, construct a context message for any 

805 # error reporting. 

806 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

807 context = f"when adding definitions from {', '.join(files)}" if files else "" 

808 log.debug("Adding definitions from config %s", ", ".join(files)) 

809 

810 with self._lock: 

811 # Processing consumes entries from sconfig, so record the names 

812 # first in order to collect the results afterwards. 

813 names = list(sconfig.keys()) 

814 for name in names: 

815 processStorageClass(name, sconfig, context) 

816 

817 if cache_key is not None: 

818 _derived_cache[cache_key] = { 

819 name: self._storageClasses[name] for name in names if name in self._storageClasses 

820 } 

821 

822 def getStorageClass(self, storageClassName: str) -> StorageClass: 

823 """Get a StorageClass instance associated with the supplied name. 

824 

825 Parameters 

826 ---------- 

827 storageClassName : `str` 

828 Name of the storage class to retrieve. 

829 

830 Returns 

831 ------- 

832 instance : `StorageClass` 

833 Instance of the correct `StorageClass`. 

834 

835 Raises 

836 ------ 

837 KeyError 

838 The requested storage class name is not registered. 

839 """ 

840 with self._lock: 

841 return self._storageClasses[storageClassName] 

842 

843 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

844 """Find the storage class associated with this python type. 

845 

846 Parameters 

847 ---------- 

848 pytype : `type` 

849 The Python type to be matched. 

850 compare_types : `bool`, optional 

851 If `False`, the type will be checked against name of the python 

852 type. This comparison is always done first. If `True` and the 

853 string comparison failed, each candidate storage class will be 

854 forced to have its type imported. This can be significantly slower. 

855 

856 Returns 

857 ------- 

858 storageClass : `StorageClass` 

859 The matching storage class. 

860 

861 Raises 

862 ------ 

863 KeyError 

864 Raised if no match could be found. 

865 

866 Notes 

867 ----- 

868 It is possible for a python type to be associated with multiple 

869 storage classes. This method will currently return the first that 

870 matches. 

871 """ 

872 with self._lock: 

873 result = self._find_storage_class(pytype, False) 

874 if result: 

875 return result 

876 

877 if compare_types: 877 ↛ 884line 877 didn't jump to line 884 because the condition on line 877 was always true

878 # The fast comparison failed and we were asked to try the 

879 # variant that might involve code imports. 

880 result = self._find_storage_class(pytype, True) 

881 if result: 881 ↛ 882line 881 didn't jump to line 882 because the condition on line 881 was never true

882 return result 

883 

884 raise KeyError( 

885 f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}" 

886 ) 

887 

888 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

889 """Iterate through all storage classes to find a match. 

890 

891 Parameters 

892 ---------- 

893 pytype : `type` 

894 The Python type to be matched. 

895 compare_types : `bool`, optional 

896 Whether to use type name matching or explicit type matching. 

897 The latter can be slower. 

898 

899 Returns 

900 ------- 

901 storageClass : `StorageClass` or `None` 

902 The matching storage class, or `None` if no match was found. 

903 

904 Notes 

905 ----- 

906 Helper method for ``findStorageClass``. 

907 """ 

908 with self._lock: 

909 for storageClass in self._storageClasses.values(): 

910 if storageClass.is_type(pytype, compare_types=compare_types): 

911 return storageClass 

912 return None 

913 

914 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

915 """Store the `StorageClass` in the factory. 

916 

917 Will be indexed by `StorageClass.name` and will return instances 

918 of the supplied `StorageClass`. 

919 

920 Parameters 

921 ---------- 

922 storageClass : `StorageClass` 

923 Type of the Python `StorageClass` to register. 

924 msg : `str`, optional 

925 Additional message string to be included in any error message. 

926 

927 Raises 

928 ------ 

929 ValueError 

930 If a storage class has already been registered with 

931 that storage class name and the previous definition differs. 

932 """ 

933 with self._lock: 

934 if storageClass.name in self._storageClasses: 

935 existing = self.getStorageClass(storageClass.name) 

936 if existing != storageClass: 

937 errmsg = f" {msg}" if msg else "" 

938 raise ValueError( 

939 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

940 f"differs from current definition ({existing!r}){errmsg}" 

941 ) 

942 if type(existing) is StorageClass and type(storageClass) is not StorageClass: 942 ↛ 944line 942 didn't jump to line 944 because the condition on line 942 was never true