Coverage for python/lsst/daf/butler/remote_butler/_get.py: 0%

1from typing import Any 

2 

3from pydantic import AnyHttpUrl 

4 

5from lsst.resources import ResourcePath 

6from lsst.resources.http import HttpResourcePath 

7 

8from .._dataset_ref import DatasetRef 

9from .._location import Location 

10from ..datastore.cache_manager import AbstractDatastoreCacheManager, DatastoreDisabledCacheManager 

11from ..datastore.stored_file_info import StoredFileInfo 

12from ..datastores.file_datastore.get import ( 

13 DatasetLocationInformation, 

14 Mapping, 

15 generate_datastore_get_information, 

16 get_dataset_as_python_object_from_get_info, 

17) 

18from .authentication.interface import RemoteButlerAuthenticationProvider 

19from .server_models import FileAuthenticationMode, FileInfoPayload, FileInfoRecord 

20 

21 

22def get_dataset_as_python_object( 

23 written_ref: DatasetRef, 

24 read_ref: DatasetRef, 

25 payload: FileInfoPayload, 

26 *, 

27 auth: RemoteButlerAuthenticationProvider, 

28 parameters: Mapping[str, Any] | None, 

29 cache_manager: AbstractDatastoreCacheManager | None = None, 

30) -> Any: 

31 """Retrieve an artifact from storage and return it as a Python object. 

32 

33 Parameters 

34 ---------- 

35 written_ref : `DatasetRef` 

36 Metadata about this artifact using the `StorageClass` that the file 

37 was written with. This is the ref given to the `Formatter`, so that 

38 it always sees the write `StorageClass` regardless of any read-time 

39 override, matching the behavior of `DirectButler`. 

40 read_ref : `DatasetRef` 

41 Metadata about this artifact using the `StorageClass` that the caller 

42 wants returned. This differs from ``written_ref`` only in its 

43 `StorageClass` when a read-time override has been requested. The two 

44 refs always share the same dataset type name, so they either both 

45 refer to a component or both refer to a composite; any component 

46 override requested by the caller is applied before the read 

47 `StorageClass` override is derived. 

48 payload : `FileInfoPayload` 

49 Pre-processed information about each file associated with this 

50 artifact. 

51 auth : `RemoteButlerAuthenticationProvider` 

52 Provides authentication headers for HTTP service hosting the artifact 

53 files. 

54 parameters : `~collections.abc.Mapping` [`str`, `typing.Any`] 

55 `StorageClass` and `Formatter` parameters to be used when converting 

56 the artifact to a Python object. 

57 cache_manager : `AbstractDatastoreCacheManager` or `None`, optional 

58 Cache manager to use. If `None` the cache is disabled. 

59 

60 Returns 

61 ------- 

62 python_object : `typing.Any` 

63 The retrieved artifact, converted to a Python object. 

64 """ 

65 fileLocations = [_to_dataset_location_information(file_info, auth) for file_info in payload.file_info] 

66 

67 # The Formatter is constructed with the written ref, while the read 

68 # StorageClass override travels separately. DirectButler does the same in 

69 # ``FileDatastore._prepare_for_direct_get``. 

70 datastore_file_info = generate_datastore_get_information( 

71 fileLocations, 

72 ref=written_ref, 

73 readStorageClass=read_ref.datasetType.storageClass, 

74 parameters=parameters, 

75 ) 

76 if cache_manager is None: 

77 cache_manager = DatastoreDisabledCacheManager() 

78 return get_dataset_as_python_object_from_get_info( 

79 datastore_file_info, ref=read_ref, parameters=parameters, cache_manager=cache_manager 

80 ) 

81 

82 

83def convert_http_url_to_resource_path( 

84 url: AnyHttpUrl, auth: RemoteButlerAuthenticationProvider, auth_mode: FileAuthenticationMode 

85) -> ResourcePath: 

86 """Convert an HTTP URL to a ResourcePath instance with authentication 

87 headers attached. 

88 

89 Parameters 

90 ---------- 

91 url : `AnyHttpUrl` 

92 URL to convert. 

93 auth : `RemoteButlerAuthenticationProvider` 

94 Provides authentication headers for the URL. 

95 auth_mode : `FileAuthenticationMode` 

96 Specifies which authentication headers to use. 

97 """ 

98 if auth_mode == "none": 

99 headers = None 

100 elif auth_mode == "gafaelfawr": 

101 headers = auth.get_server_headers() 

102 elif auth_mode == "datastore": 

103 headers = auth.get_datastore_headers() 

104 else: 

105 raise ValueError(f"Unknown authentication type: '{auth_mode}'") 

106 

107 return HttpResourcePath.create_http_resource_path(str(url), extra_headers=headers) 

108 

109 

110def _to_dataset_location_information( 

111 file_info: FileInfoRecord, auth: RemoteButlerAuthenticationProvider 

112) -> DatasetLocationInformation: 

113 path = convert_http_url_to_resource_path(file_info.url, auth, auth_mode=file_info.auth) 

114 return (Location(None, path), StoredFileInfo.from_simple(file_info.datastoreRecords))