Coverage for python/lsst/daf/butler/tests/server.py: 0%
88 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-09-30 10:59 +0000
1import json
2import os
3from collections.abc import Iterator
4from contextlib import closing, contextmanager
5from dataclasses import dataclass
6from tempfile import TemporaryDirectory
8from fastapi import FastAPI, Request
9from fastapi.testclient import TestClient
11from lsst.daf.butler import Butler, ButlerConfig, Config, LabeledButlerFactory
12from lsst.daf.butler.remote_butler import RemoteButler
13from lsst.daf.butler.remote_butler._factory import RemoteButlerFactory
14from lsst.daf.butler.remote_butler.server import create_app
15from lsst.daf.butler.remote_butler.server._config import ButlerServerConfig, RepositoryConfig, mock_config
16from lsst.daf.butler.remote_butler.server._dependencies import (
17 auth_delegated_token_dependency,
18 butler_factory_dependency,
19 reset_dependency_caches,
20 user_name_dependency,
21)
22from lsst.resources import ResourcePath
23from lsst.resources.s3utils import clean_test_environment_for_s3, getS3Client
25from ..direct_butler import DirectButler
26from ._repo_template_cache import make_repo_for_test
27from .hybrid_butler import HybridButler
28from .postgresql import TemporaryPostgresInstance
29from .server_utils import add_auth_header_check_middleware
31try:
32 # moto v5
33 from moto import mock_aws # type: ignore
34except ImportError:
35 # moto v4 and earlier
36 from moto import mock_s3 as mock_aws # type: ignore
38__all__ = ("TEST_REPOSITORY_NAME", "TestServerInstance", "create_test_server")
41TEST_REPOSITORY_NAME = "testrepo"
44@dataclass(frozen=True)
45class TestServerInstance:
46 """Butler instances and other data associated with a temporary server
47 instance.
48 """
50 config_file_path: str
51 """Path to the Butler config file used by the server."""
52 client: TestClient
53 """httpx2 client connected to the temporary server."""
54 remote_butler: RemoteButler
55 """`RemoteButler` connected to the temporary server."""
56 remote_butler_without_error_propagation: RemoteButler
57 """`RemoteButler` connected to the temporary server.
59 By default, the TestClient instance raises any unhandled exceptions
60 from the server as if they had originated in the client to ease debugging.
61 However, this can make it appear that error propagation is working
62 correctly when in a real deployment the server exception would cause a 500
63 Internal Server Error. This instance of the butler is set up so that any
64 unhandled server exceptions do return a 500 status code."""
65 direct_butler: Butler
66 """`DirectButler` instance connected to the same repository as the
67 temporary server.
68 """
69 hybrid_butler: HybridButler
70 """`HybridButler` instance connected to the temporary server."""
71 app: FastAPI
72 """Butler server FastAPI app."""
75@contextmanager
76def create_test_server(
77 test_directory: str,
78 *,
79 postgres: TemporaryPostgresInstance | None = None,
80 server_config: ButlerServerConfig | None = None,
81) -> Iterator[TestServerInstance]:
82 """Create a temporary Butler server instance for testing.
84 Parameters
85 ----------
86 test_directory : `str`
87 Path to the ``tests/`` directory at the root of the repository,
88 containing Butler test configuration files.
89 postgres : `TemporaryPostgresInstance` | `None`
90 If provided, the Butler server will use this postgres database
91 instance. If no postgres instance is specified, the server will use a
92 a SQLite database.
93 server_config : `ButlerServerConfig`, optional
94 Configuration to use for the Butler server.
96 Returns
97 -------
98 instance : `TestServerInstance`
99 Object containing Butler instances connected to the server and
100 associated information.
101 """
102 # Set up a mock S3 environment using Moto. Moto also monkeypatches the
103 # `requests` library so that any HTTP requests to presigned S3 URLs get
104 # redirected to the mocked S3.
105 # Note that all files are stored in memory.
106 with clean_test_environment_for_s3():
107 with mock_aws():
108 base_config_path = os.path.join(test_directory, "config/basic/server.yaml")
109 # Create S3 buckets used for the datastore in server.yaml.
110 for bucket in ["mutable-bucket", "immutable-bucket"]:
111 getS3Client().create_bucket(Bucket=bucket)
113 config = Config(base_config_path)
114 if postgres is not None:
115 postgres.patch_butler_config(config)
117 with TemporaryDirectory() as root, mock_config(server_config) as server_config:
118 make_repo_for_test(root, config=config, forceConfigRoot=False)
119 config_file_path = os.path.join(root, "butler.yaml")
121 server_config.repositories = {
122 TEST_REPOSITORY_NAME: RepositoryConfig(
123 config_uri=config_file_path, authorized_groups=["*"]
124 )
125 }
126 reset_dependency_caches()
128 app = create_app()
129 if server_config.authentication == "rubin_science_platform":
130 add_auth_header_check_middleware(app)
131 _add_root_exception_handler(app)
132 # Override the server's Butler initialization to point at our
133 # test repo
134 server_butler_factory = LabeledButlerFactory({TEST_REPOSITORY_NAME: config_file_path})
135 # DirectButler has a dimension_record_cache object that
136 # maintains a complete set of dimension records for governor
137 # dimensions. These values change infrequently and are needed
138 # for almost every DirectButler operation, so the complete set
139 # is downloaded the first time a record is needed.
140 #
141 # On the server it would be expensive to do this for every
142 # request's new DirectButler instance, so normally these are
143 # loaded once, the first time a repository is accessed. This
144 # is a problem for unit tests because they typically manipulate
145 # instrument records etc during setup. So configure the
146 # factory to disable this preloading and re-fetch the records
147 # as needed.
148 server_butler_factory._preload_unsafe_direct_butler_caches = False
149 app.dependency_overrides[butler_factory_dependency] = lambda: server_butler_factory
150 # In an actual deployment, these headers would be provided by
151 # the Gafaelfawr ingress.
152 app.dependency_overrides[user_name_dependency] = lambda: "mock-username"
153 app.dependency_overrides[auth_delegated_token_dependency] = lambda: "mock-delegated-token"
155 direct_butler = Butler.from_config(config_file_path, writeable=True)
156 assert isinstance(direct_butler, DirectButler)
157 # Using TestClient in a context manager ensures that it uses
158 # the same async event loop for all requests -- otherwise it
159 # starts a new one on each request.
160 with TestClient(app) as client, direct_butler, closing(server_butler_factory):
161 remote_butler = _make_remote_butler(client)
162 hybrid_butler = HybridButler(remote_butler, direct_butler)
164 client_without_error_propagation = TestClient(app, raise_server_exceptions=False)
165 remote_butler_without_error_propagation = _make_remote_butler(
166 client_without_error_propagation
167 )
169 yield TestServerInstance(
170 config_file_path=config_file_path,
171 client=client,
172 direct_butler=direct_butler,
173 remote_butler=remote_butler,
174 remote_butler_without_error_propagation=remote_butler_without_error_propagation,
175 hybrid_butler=hybrid_butler,
176 app=app,
177 )
180def _make_remote_butler(client: TestClient) -> RemoteButler:
181 config_endpoint = f"https://test.example/api/butler/repo/{TEST_REPOSITORY_NAME}/butler.yaml"
182 config_json = client.get(config_endpoint).read()
183 config = Config(json.loads(config_json))
184 config.configFile = ResourcePath(config_endpoint)
185 butler_config = ButlerConfig(config)
186 remote_butler_factory = RemoteButlerFactory.create_factory_from_config(butler_config, client)
187 return remote_butler_factory.create_butler_for_access_token("fake-access-token")
190class UnhandledServerError(Exception):
191 """Raised for unhandled exceptions within the server that would result in a
192 500 Internal Server Error in a real deployment. This allows us to tell the
193 difference between exceptions being propagated intentionally, and those
194 just bubbling up implicitly from the server to the client.
196 The FastAPI TestClient by default passes unhandled exceptions up from the
197 server to the client. This is useful behavior for unit testing because it
198 gives you traceability from the test to the problem in the server code.
199 However, because RemoteButler is in some ways just a proxy for the
200 server-side Butler, we raise similar exceptions on the client and server
201 side. Thus the default TestClient behavior can mask missing error-handling
202 logic.
203 """
206def _add_root_exception_handler(app: FastAPI) -> None:
207 @app.exception_handler(Exception)
208 async def convert_exception_types(request: Request, exc: Exception) -> None:
209 raise UnhandledServerError("Unhandled server exception") from exc