Coverage for python/lsst/daf/butler/remote_butler/_get.py: 0%

1from typing import Any 

2 

3from pydantic import AnyHttpUrl 

4 

5from lsst.resources import ResourcePath 

6from lsst.resources.http import HttpResourcePath 

7 

8from .._dataset_ref import DatasetRef 

9from .._location import Location 

10from ..datastore.cache_manager import AbstractDatastoreCacheManager, DatastoreDisabledCacheManager 

11from ..datastore.stored_file_info import StoredFileInfo 

12from ..datastores.file_datastore.get import ( 

13 DatasetLocationInformation, 

14 Mapping, 

15 generate_datastore_get_information, 

16 get_dataset_as_python_object_from_get_info, 

17) 

18from .authentication.interface import RemoteButlerAuthenticationProvider 

19from .server_models import FileAuthenticationMode, FileInfoPayload, FileInfoRecord 

20 

21 

22def get_dataset_as_python_object( 

23 registry_ref: DatasetRef, 

24 read_ref: DatasetRef, 

25 payload: FileInfoPayload, 

26 *, 

27 auth: RemoteButlerAuthenticationProvider, 

28 parameters: Mapping[str, Any] | None, 

29 cache_manager: AbstractDatastoreCacheManager | None = None, 

30) -> Any: 

31 """Retrieve an artifact from storage and return it as a Python object. 

32 

33 Parameters 

34 ---------- 

35 registry_ref : `DatasetRef` 

36 Metadata about this artifact using the dataset type definition from 

37 the repository, as returned by the server. This is the ref given to 

38 the `Formatter`, so that it never sees a read-time override, matching 

39 the behavior of `DirectButler`. It is never a component. 

40 read_ref : `DatasetRef` 

41 Metadata about this artifact using the `StorageClass` that the caller 

42 wants returned, and naming the component (if any) that they asked 

43 for. 

44 payload : `FileInfoPayload` 

45 Pre-processed information about each file associated with this 

46 artifact. 

47 auth : `RemoteButlerAuthenticationProvider` 

48 Provides authentication headers for HTTP service hosting the artifact 

49 files. 

50 parameters : `~collections.abc.Mapping` [`str`, `typing.Any`] 

51 `StorageClass` and `Formatter` parameters to be used when converting 

52 the artifact to a Python object. 

53 cache_manager : `AbstractDatastoreCacheManager` or `None`, optional 

54 Cache manager to use. If `None` the cache is disabled. 

55 

56 Returns 

57 ------- 

58 python_object : `typing.Any` 

59 The retrieved artifact, converted to a Python object. 

60 """ 

61 fileLocations = [_to_dataset_location_information(file_info, auth) for file_info in payload.file_info] 

62 

63 # The Formatter is constructed with the repository definition of the 

64 # dataset, while the component and read StorageClass requested by the 

65 # caller travel separately. DirectButler does the same in 

66 # ``FileDatastore._prepare_for_direct_get``. 

67 datastore_file_info = generate_datastore_get_information( 

68 fileLocations, 

69 registry_ref=registry_ref, 

70 read_ref=read_ref, 

71 parameters=parameters, 

72 ) 

73 if cache_manager is None: 

74 cache_manager = DatastoreDisabledCacheManager() 

75 return get_dataset_as_python_object_from_get_info( 

76 datastore_file_info, ref=read_ref, parameters=parameters, cache_manager=cache_manager 

77 ) 

78 

79 

80def convert_http_url_to_resource_path( 

81 url: AnyHttpUrl, auth: RemoteButlerAuthenticationProvider, auth_mode: FileAuthenticationMode 

82) -> ResourcePath: 

83 """Convert an HTTP URL to a ResourcePath instance with authentication 

84 headers attached. 

85 

86 Parameters 

87 ---------- 

88 url : `AnyHttpUrl` 

89 URL to convert. 

90 auth : `RemoteButlerAuthenticationProvider` 

91 Provides authentication headers for the URL. 

92 auth_mode : `FileAuthenticationMode` 

93 Specifies which authentication headers to use. 

94 """ 

95 if auth_mode == "none": 

96 headers = None 

97 elif auth_mode == "gafaelfawr": 

98 headers = auth.get_server_headers() 

99 elif auth_mode == "datastore": 

100 headers = auth.get_datastore_headers() 

101 else: 

102 raise ValueError(f"Unknown authentication type: '{auth_mode}'") 

103 

104 return HttpResourcePath.create_http_resource_path(str(url), extra_headers=headers) 

105 

106 

107def _to_dataset_location_information( 

108 file_info: FileInfoRecord, auth: RemoteButlerAuthenticationProvider 

109) -> DatasetLocationInformation: 

110 path = convert_http_url_to_resource_path(file_info.url, auth, auth_mode=file_info.auth) 

111 return (Location(None, path), StoredFileInfo.from_simple(file_info.datastoreRecords))