Coverage for tests/test_consdbClient.py: 100%
170 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-19 03:44 -0700
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import json
23from typing import Any
25import pytest
26import responses
27from astropy.table import Table
28from requests import HTTPError, Response, Timeout
30from lsst.summit.utils import ConsDbClient, FlexibleMetadataInfo
31from lsst.summit.utils.consdbClient import (
32 DEFAULT_CONNECT_TIMEOUT,
33 DEFAULT_READ_TIMEOUT,
34 getCcdVisitTableForDay,
35)
38@pytest.fixture
39def client() -> ConsDbClient:
40 """Initialize client with a fake url
41 Requires mocking connection with @responses.activate decorator
42 """
43 return ConsDbClient("http://example.com/consdb")
46def test_table_name() -> None:
47 instrument = "latiss"
48 obs_type = "exposure"
49 assert (
50 ConsDbClient.compute_flexible_metadata_table_name(instrument, obs_type)
51 == "cdb_latiss.exposure_flexdata"
52 )
55@responses.activate
56def test_add_flexible_metadata_key(client: ConsDbClient) -> None:
57 instrument = "latiss"
58 obs_type = "exposure"
59 responses.post(
60 "http://example.com/consdb/flex/latiss/exposure/addkey",
61 json={
62 "message": "Key added to flexible metadata",
63 "key": "foo",
64 "instrument": "latiss",
65 "obs_type": "exposure",
66 },
67 match=[
68 responses.matchers.json_params_matcher({"key": "foo", "dtype": "bool", "doc": "bool key"}),
69 ],
70 )
71 responses.post(
72 "http://example.com/consdb/flex/latiss/exposure/addkey",
73 json={
74 "message": "Key added to flexible metadata",
75 "key": "bar",
76 "instrument": "latiss",
77 "obs_type": "exposure",
78 },
79 match=[
80 responses.matchers.json_params_matcher({"key": "bar", "dtype": "int", "doc": "int key"}),
81 ],
82 )
83 responses.post(
84 "http://example.com/consdb/flex/bad_instrument/exposure/addkey",
85 status=404,
86 json={"message": "Unknown instrument", "value": "bad_instrument", "valid": ["latiss"]},
87 )
88 responses.post(
89 "http://example.com/consdb/flex/latiss/bad_obs_type/addkey",
90 status=404,
91 json={"message": "Unknown observation type", "value": "bad_obs_type", "valid": ["exposure"]},
92 )
94 assert (
95 client.add_flexible_metadata_key(instrument, obs_type, "foo", "bool", "bool key").json()["key"]
96 == "foo"
97 )
98 assert (
99 client.add_flexible_metadata_key(instrument, obs_type, "bar", "int", "int key").json()["instrument"]
100 == "latiss"
101 )
102 with pytest.raises(HTTPError, match="404") as e:
103 client.add_flexible_metadata_key("bad_instrument", obs_type, "error", "int", "instrument error")
104 assert "Unknown instrument" in str(e.value.__notes__)
105 assert e.value.response is not None
106 json_data = e.value.response.json()
107 assert json_data["message"] == "Unknown instrument"
108 assert json_data["value"] == "bad_instrument"
109 assert json_data["valid"] == ["latiss"]
110 with pytest.raises(HTTPError, match="404"):
111 client.add_flexible_metadata_key(instrument, "bad_obs_type", "error", "int", "obs_type error")
114@responses.activate
115def test_get_flexible_metadata_keys(client: ConsDbClient) -> None:
116 description = {"foo": ["bool", "a", None, None], "bar": ["float", "b", "deg", "pos.eq.ra"]}
117 responses.get(
118 "http://example.com/consdb/flex/latiss/exposure/schema",
119 json=description,
120 )
121 instrument = "latiss"
122 obs_type = "exposure"
123 assert client.get_flexible_metadata_keys(instrument, obs_type) == {
124 "foo": FlexibleMetadataInfo("bool", "a"),
125 "bar": FlexibleMetadataInfo("float", "b", "deg", "pos.eq.ra"),
126 }
129@responses.activate
130def test_get_flexible_metadata(client: ConsDbClient) -> None:
131 results = {"bool_key": True, "int_key": 42, "float_key": 3.14159, "str_key": "foo"}
132 responses.get(
133 "http://example.com/consdb/flex/latiss/exposure/obs/271828",
134 json=results,
135 )
136 responses.get(
137 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=float_key", json={"float_key": 3.14159}
138 )
139 responses.get(
140 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=int_key&k=float_key",
141 json={"float_key": 3.14159, "int_key": 42},
142 )
143 instrument = "latiss"
144 obs_type = "exposure"
145 obs_id = 271828
146 assert client.get_flexible_metadata(instrument, obs_type, obs_id) == results
147 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["float_key"]) == {
148 "float_key": results["float_key"]
149 }
150 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["int_key", "float_key"]) == {
151 "int_key": results["int_key"],
152 "float_key": results["float_key"],
153 }
156@responses.activate
157def test_insert_flexible_metadata(client: ConsDbClient) -> None:
158 instrument = "latiss"
159 obs_type = "exposure"
160 with pytest.raises(ValueError):
161 client.insert_flexible_metadata(instrument, obs_type, 271828)
162 # TODO: more POST tests
165@responses.activate
166def test_schema(client: ConsDbClient) -> None:
167 description = {"foo": ("bool", "a"), "bar": ("int", "b")}
168 responses.get(
169 "http://example.com/consdb/schema/latiss/misc_table",
170 json=description,
171 )
172 instrument = "latiss"
173 table = "misc_table"
174 assert client.schema(instrument, table) == description
177@responses.activate
178@pytest.mark.parametrize(
179 "secret, redacted",
180 [
181 ("usdf:v987wefVMPz", "us***:v9***"),
182 ("u:v", "u***:v***"),
183 ("ulysses", "ul***"),
184 (":alberta94", "***:al***"),
185 ],
186)
187def test_clean_token_url_response(secret: str, redacted: str) -> None:
188 """Test tokens URL is cleaned when an error is thrown from requests
189 Use with pytest raises assert an error'
190 assert that url does not contain tokens
191 """
192 domain = "@usdf-fake.slackers.stanford.edu/consdb"
193 complex_client = ConsDbClient(f"https://{secret}{domain}")
195 obs_type = "exposure"
196 responses.post(
197 f"https://{secret}{domain}/flex/bad_instrument/exposure/addkey",
198 status=404,
199 )
200 with pytest.raises(HTTPError, match="404") as error:
201 complex_client.add_flexible_metadata_key(
202 "bad_instrument", obs_type, "error", "int", "instrument error"
203 )
205 url = error.value.args[0].split()[-1]
206 sanitized = f"https://{redacted}{domain}/flex/bad_instrument/exposure/addkey"
207 assert url == sanitized
210def test_client(client: ConsDbClient) -> None:
211 """Test ConsDbClient is initialized properly"""
212 assert "clean_url" in str(client.session.hooks["response"])
213 assert client.connect_timeout == DEFAULT_CONNECT_TIMEOUT
214 assert client.read_timeout == DEFAULT_READ_TIMEOUT
215 assert client.timeout == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
218def test_timeout_override() -> None:
219 """The timeouts are configurable, including disabling them entirely."""
220 tuned = ConsDbClient("http://example.com/consdb", connect_timeout=5, read_timeout=30)
221 assert tuned.timeout == (5, 30)
222 unbounded = ConsDbClient("http://example.com/consdb", connect_timeout=None, read_timeout=None)
223 assert unbounded.timeout == (None, None)
226def test_get_passes_timeout(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
227 """GET requests must carry a timeout so a stalled server cannot hang."""
228 captured: dict[str, Any] = {}
230 def fake_get(url: str, **kwargs: Any) -> Response:
231 captured.update(kwargs)
232 response = Response()
233 response.status_code = 200
234 response._content = b"{}"
235 return response
237 monkeypatch.setattr(client.session, "get", fake_get)
238 client.schema()
239 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
242def test_post_passes_timeout(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
243 """POST requests must carry a timeout so a stalled server cannot hang."""
244 captured: dict[str, Any] = {}
246 def fake_post(url: str, **kwargs: Any) -> Response:
247 captured.update(kwargs)
248 response = Response()
249 response.status_code = 200
250 response._content = b'{"message": "Data inserted"}'
251 return response
253 monkeypatch.setattr(client.session, "post", fake_post)
254 client.insert("latiss", "exposure", 271828, {"foo": 1})
255 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
258def test_timeout_propagates(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
259 """A timed-out request surfaces as requests.Timeout to the caller."""
261 def fake_get(url: str, **kwargs: Any) -> Response:
262 raise Timeout("timed out")
264 monkeypatch.setattr(client.session, "get", fake_get)
265 with pytest.raises(Timeout):
266 client.schema()
269@responses.activate
270def test_insert_obs_id(client: ConsDbClient) -> None:
271 """An integer obs_id targets the ``.../obs/{obs_id}`` endpoint."""
272 responses.post(
273 "http://example.com/consdb/insert/latiss/exposure/obs/271828",
274 json={"message": "Data inserted"},
275 match=[
276 responses.matchers.json_params_matcher(
277 {"table": "exposure", "obs_id": 271828, "values": {"foo": 1}}
278 ),
279 ],
280 )
281 assert client.insert("latiss", "exposure", 271828, {"foo": 1}).json()["message"] == "Data inserted"
284@responses.activate
285def test_insert_by_seq_num(client: ConsDbClient) -> None:
286 """A 2-tuple targets the ``.../{day_obs}/{seq_num}`` endpoint."""
287 responses.post(
288 "http://example.com/consdb/insert/latiss/exposure/by_seq_num/20240603/123",
289 json={"message": "Data inserted"},
290 match=[
291 responses.matchers.json_params_matcher({"table": "exposure", "values": {"foo": 1}}),
292 ],
293 )
294 assert (
295 client.insert("latiss", "exposure", (20240603, 123), {"foo": 1}).json()["message"] == "Data inserted"
296 )
299@responses.activate
300def test_insert_by_seq_num_detector(client: ConsDbClient) -> None:
301 """A 3-tuple targets the ``.../{day_obs}/{seq_num}/{detector}``
302 endpoint for per-detector (ccdexposure-level) tables."""
303 responses.post(
304 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94",
305 json={"message": "Data inserted"},
306 match=[
307 responses.matchers.json_params_matcher({"table": "ccdexposure", "values": {"foo": 1}}),
308 ],
309 )
310 assert (
311 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}).json()["message"]
312 == "Data inserted"
313 )
316@responses.activate
317def test_insert_allow_update(client: ConsDbClient) -> None:
318 """allow_update appends ``?u=1`` to upsert against the addressed key."""
319 responses.post(
320 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94?u=1",
321 json={"message": "Data inserted"},
322 match=[responses.matchers.query_param_matcher({"u": "1"})],
323 )
324 assert (
325 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}, allow_update=True).json()[
326 "message"
327 ]
328 == "Data inserted"
329 )
332def test_insert_bad_obs_id_tuple(client: ConsDbClient) -> None:
333 """A tuple that is not length 2 or 3 is rejected before any request."""
334 with pytest.raises(AssertionError, match="obs_id tuple"):
335 client.insert("latiss", "exposure", (20240603,), {"foo": 1}) # type: ignore[arg-type]
336 with pytest.raises(AssertionError, match="obs_id tuple"):
337 client.insert("latiss", "exposure", (20240603, 123, 94, 0), {"foo": 1}) # type: ignore[arg-type]
340def test_insert_no_values(client: ConsDbClient) -> None:
341 """Inserting with no values raises before any request."""
342 with pytest.raises(ValueError, match="No values to insert"):
343 client.insert("latiss", "exposure", 271828, {})
346@responses.activate
347def test_getCcdVisitTableForDay_dedupes_overlapping_columns(client: ConsDbClient) -> None:
348 """Columns already present in ccdvisit1_quicklook must not be re-selected.
350 ``cvq.*`` pulls in every column of ccdvisit1_quicklook. As ConsDB
351 denormalises identity columns (visit_id, detector, seq_num, ...) onto the
352 quicklook table, naming those again explicitly from the joined tables makes
353 the server return duplicate column names, which astropy refuses to build a
354 Table from (DM-55152). They must be dropped from the SELECT instead.
355 """
356 url = "http://example.com/consdb/query"
358 # First query is the LIMIT 0 schema probe: pretend the quicklook table has
359 # denormalised visit_id, detector and seq_num onto itself.
360 responses.post(
361 url,
362 json={"columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma"], "data": []},
363 )
364 # Second query is the real data query; its response only has to build
365 # cleanly, the behaviour under test is the SELECT clause that was sent.
366 responses.post(
367 url,
368 json={
369 "columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma", "band"],
370 "data": [[1, 100, 7, 5, 1.2, "r"]],
371 },
372 )
374 table = getCcdVisitTableForDay(client, 20240101)
375 assert isinstance(table, Table)
377 # Only inspect the SELECT clause; the WHERE clause legitimately references
378 # cv.visit_id etc. in its join conditions.
379 requestBody = responses.calls[1].request.body
380 assert isinstance(requestBody, (str, bytes))
381 sentQuery = json.loads(requestBody)["query"]
382 selectClause = sentQuery.split(" FROM ")[0]
383 # Columns already provided by cvq.* must not be re-selected explicitly...
384 assert "cvq.*" in selectClause
385 assert "cv.visit_id" not in selectClause
386 assert "cv.detector" not in selectClause
387 assert "v.seq_num" not in selectClause
388 # ...but columns the quicklook table lacks must still be pulled from visit1
389 for col in ("v.band", "v.exp_time", "v.day_obs", "v.img_type"):
390 assert col in selectClause
393@responses.activate
394def test_getCcdVisitTableForDay_keeps_columns_when_no_overlap(client: ConsDbClient) -> None:
395 """When the quicklook table shares no names, every extra is selected."""
396 url = "http://example.com/consdb/query"
397 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": []})
398 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": [[1, 1.2]]})
400 getCcdVisitTableForDay(client, 20240101)
402 requestBody = responses.calls[1].request.body
403 assert isinstance(requestBody, (str, bytes))
404 sentQuery = json.loads(requestBody)["query"]
405 selectClause = sentQuery.split(" FROM ")[0]
406 for col in ("cv.detector", "cv.visit_id", "v.band", "v.exp_time", "v.seq_num", "v.day_obs", "v.img_type"):
407 assert col in selectClause
410# TODO: more POST tests
411# client.insert_multiple(instrument, table, obs_dict, allow_update)
412# client.query(query)