Coverage for tests/test_consdbClient.py: 100%
164 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-08-29 10:16 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import json
24import pytest
25import responses
26from astropy.table import Table
27from requests import HTTPError, Response, Timeout
29from lsst.summit.utils import ConsDbClient, FlexibleMetadataInfo
30from lsst.summit.utils.consdbClient import (
31 DEFAULT_CONNECT_TIMEOUT,
32 DEFAULT_READ_TIMEOUT,
33 getCcdVisitTableForDay,
34)
37@pytest.fixture
38def client():
39 """Initialize client with a fake url
40 Requires mocking connection with @responses.activate decorator
41 """
42 return ConsDbClient("http://example.com/consdb")
45def test_table_name():
46 instrument = "latiss"
47 obs_type = "exposure"
48 assert (
49 ConsDbClient.compute_flexible_metadata_table_name(instrument, obs_type)
50 == "cdb_latiss.exposure_flexdata"
51 )
54@responses.activate
55def test_add_flexible_metadata_key(client):
56 instrument = "latiss"
57 obs_type = "exposure"
58 responses.post(
59 "http://example.com/consdb/flex/latiss/exposure/addkey",
60 json={
61 "message": "Key added to flexible metadata",
62 "key": "foo",
63 "instrument": "latiss",
64 "obs_type": "exposure",
65 },
66 match=[
67 responses.matchers.json_params_matcher({"key": "foo", "dtype": "bool", "doc": "bool key"}),
68 ],
69 )
70 responses.post(
71 "http://example.com/consdb/flex/latiss/exposure/addkey",
72 json={
73 "message": "Key added to flexible metadata",
74 "key": "bar",
75 "instrument": "latiss",
76 "obs_type": "exposure",
77 },
78 match=[
79 responses.matchers.json_params_matcher({"key": "bar", "dtype": "int", "doc": "int key"}),
80 ],
81 )
82 responses.post(
83 "http://example.com/consdb/flex/bad_instrument/exposure/addkey",
84 status=404,
85 json={"message": "Unknown instrument", "value": "bad_instrument", "valid": ["latiss"]},
86 )
87 responses.post(
88 "http://example.com/consdb/flex/latiss/bad_obs_type/addkey",
89 status=404,
90 json={"message": "Unknown observation type", "value": "bad_obs_type", "valid": ["exposure"]},
91 )
93 assert (
94 client.add_flexible_metadata_key(instrument, obs_type, "foo", "bool", "bool key").json()["key"]
95 == "foo"
96 )
97 assert (
98 client.add_flexible_metadata_key(instrument, obs_type, "bar", "int", "int key").json()["instrument"]
99 == "latiss"
100 )
101 with pytest.raises(HTTPError, match="404") as e:
102 client.add_flexible_metadata_key("bad_instrument", obs_type, "error", "int", "instrument error")
103 assert "Unknown instrument" in str(e.value.__notes__)
104 json_data = e.value.response.json()
105 assert json_data["message"] == "Unknown instrument"
106 assert json_data["value"] == "bad_instrument"
107 assert json_data["valid"] == ["latiss"]
108 with pytest.raises(HTTPError, match="404"):
109 client.add_flexible_metadata_key(instrument, "bad_obs_type", "error", "int", "obs_type error")
112@responses.activate
113def test_get_flexible_metadata_keys(client):
114 description = {"foo": ["bool", "a", None, None], "bar": ["float", "b", "deg", "pos.eq.ra"]}
115 responses.get(
116 "http://example.com/consdb/flex/latiss/exposure/schema",
117 json=description,
118 )
119 instrument = "latiss"
120 obs_type = "exposure"
121 assert client.get_flexible_metadata_keys(instrument, obs_type) == {
122 "foo": FlexibleMetadataInfo("bool", "a"),
123 "bar": FlexibleMetadataInfo("float", "b", "deg", "pos.eq.ra"),
124 }
127@responses.activate
128def test_get_flexible_metadata(client):
129 results = {"bool_key": True, "int_key": 42, "float_key": 3.14159, "str_key": "foo"}
130 responses.get(
131 "http://example.com/consdb/flex/latiss/exposure/obs/271828",
132 json=results,
133 )
134 responses.get(
135 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=float_key", json={"float_key": 3.14159}
136 )
137 responses.get(
138 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=int_key&k=float_key",
139 json={"float_key": 3.14159, "int_key": 42},
140 )
141 instrument = "latiss"
142 obs_type = "exposure"
143 obs_id = 271828
144 assert client.get_flexible_metadata(instrument, obs_type, obs_id) == results
145 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["float_key"]) == {
146 "float_key": results["float_key"]
147 }
148 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["int_key", "float_key"]) == {
149 "int_key": results["int_key"],
150 "float_key": results["float_key"],
151 }
154@responses.activate
155def test_insert_flexible_metadata(client):
156 instrument = "latiss"
157 obs_type = "exposure"
158 with pytest.raises(ValueError):
159 client.insert_flexible_metadata(instrument, obs_type, 271828)
160 # TODO: more POST tests
163@responses.activate
164def test_schema(client):
165 description = {"foo": ("bool", "a"), "bar": ("int", "b")}
166 responses.get(
167 "http://example.com/consdb/schema/latiss/misc_table",
168 json=description,
169 )
170 instrument = "latiss"
171 table = "misc_table"
172 assert client.schema(instrument, table) == description
175@responses.activate
176@pytest.mark.parametrize(
177 "secret, redacted",
178 [
179 ("usdf:v987wefVMPz", "us***:v9***"),
180 ("u:v", "u***:v***"),
181 ("ulysses", "ul***"),
182 (":alberta94", "***:al***"),
183 ],
184)
185def test_clean_token_url_response(secret, redacted):
186 """Test tokens URL is cleaned when an error is thrown from requests
187 Use with pytest raises assert an error'
188 assert that url does not contain tokens
189 """
190 domain = "@usdf-fake.slackers.stanford.edu/consdb"
191 complex_client = ConsDbClient(f"https://{secret}{domain}")
193 obs_type = "exposure"
194 responses.post(
195 f"https://{secret}{domain}/flex/bad_instrument/exposure/addkey",
196 status=404,
197 )
198 with pytest.raises(HTTPError, match="404") as error:
199 complex_client.add_flexible_metadata_key(
200 "bad_instrument", obs_type, "error", "int", "instrument error"
201 )
203 url = error.value.args[0].split()[-1]
204 sanitized = f"https://{redacted}{domain}/flex/bad_instrument/exposure/addkey"
205 assert url == sanitized
208def test_client(client):
209 """Test ConsDbClient is initialized properly"""
210 assert "clean_url" in str(client.session.hooks["response"])
211 assert client.connect_timeout == DEFAULT_CONNECT_TIMEOUT
212 assert client.read_timeout == DEFAULT_READ_TIMEOUT
213 assert client.timeout == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
216def test_timeout_override():
217 """The timeouts are configurable, including disabling them entirely."""
218 tuned = ConsDbClient("http://example.com/consdb", connect_timeout=5, read_timeout=30)
219 assert tuned.timeout == (5, 30)
220 unbounded = ConsDbClient("http://example.com/consdb", connect_timeout=None, read_timeout=None)
221 assert unbounded.timeout == (None, None)
224def test_get_passes_timeout(client, monkeypatch):
225 """GET requests must carry a timeout so a stalled server cannot hang."""
226 captured = {}
228 def fake_get(url, **kwargs):
229 captured.update(kwargs)
230 response = Response()
231 response.status_code = 200
232 response._content = b"{}"
233 return response
235 monkeypatch.setattr(client.session, "get", fake_get)
236 client.schema()
237 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
240def test_post_passes_timeout(client, monkeypatch):
241 """POST requests must carry a timeout so a stalled server cannot hang."""
242 captured = {}
244 def fake_post(url, **kwargs):
245 captured.update(kwargs)
246 response = Response()
247 response.status_code = 200
248 response._content = b'{"message": "Data inserted"}'
249 return response
251 monkeypatch.setattr(client.session, "post", fake_post)
252 client.insert("latiss", "exposure", 271828, {"foo": 1})
253 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
256def test_timeout_propagates(client, monkeypatch):
257 """A timed-out request surfaces as requests.Timeout to the caller."""
259 def fake_get(url, **kwargs):
260 raise Timeout("timed out")
262 monkeypatch.setattr(client.session, "get", fake_get)
263 with pytest.raises(Timeout):
264 client.schema()
267@responses.activate
268def test_insert_obs_id(client):
269 """An integer obs_id targets the ``.../obs/{obs_id}`` endpoint."""
270 responses.post(
271 "http://example.com/consdb/insert/latiss/exposure/obs/271828",
272 json={"message": "Data inserted"},
273 match=[
274 responses.matchers.json_params_matcher(
275 {"table": "exposure", "obs_id": 271828, "values": {"foo": 1}}
276 ),
277 ],
278 )
279 assert client.insert("latiss", "exposure", 271828, {"foo": 1}).json()["message"] == "Data inserted"
282@responses.activate
283def test_insert_by_seq_num(client):
284 """A 2-tuple targets the ``.../{day_obs}/{seq_num}`` endpoint."""
285 responses.post(
286 "http://example.com/consdb/insert/latiss/exposure/by_seq_num/20240603/123",
287 json={"message": "Data inserted"},
288 match=[
289 responses.matchers.json_params_matcher({"table": "exposure", "values": {"foo": 1}}),
290 ],
291 )
292 assert (
293 client.insert("latiss", "exposure", (20240603, 123), {"foo": 1}).json()["message"] == "Data inserted"
294 )
297@responses.activate
298def test_insert_by_seq_num_detector(client):
299 """A 3-tuple targets the ``.../{day_obs}/{seq_num}/{detector}``
300 endpoint for per-detector (ccdexposure-level) tables."""
301 responses.post(
302 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94",
303 json={"message": "Data inserted"},
304 match=[
305 responses.matchers.json_params_matcher({"table": "ccdexposure", "values": {"foo": 1}}),
306 ],
307 )
308 assert (
309 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}).json()["message"]
310 == "Data inserted"
311 )
314@responses.activate
315def test_insert_allow_update(client):
316 """allow_update appends ``?u=1`` to upsert against the addressed key."""
317 responses.post(
318 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94?u=1",
319 json={"message": "Data inserted"},
320 match=[responses.matchers.query_param_matcher({"u": "1"})],
321 )
322 assert (
323 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}, allow_update=True).json()[
324 "message"
325 ]
326 == "Data inserted"
327 )
330def test_insert_bad_obs_id_tuple(client):
331 """A tuple that is not length 2 or 3 is rejected before any request."""
332 with pytest.raises(AssertionError, match="obs_id tuple"):
333 client.insert("latiss", "exposure", (20240603,), {"foo": 1})
334 with pytest.raises(AssertionError, match="obs_id tuple"):
335 client.insert("latiss", "exposure", (20240603, 123, 94, 0), {"foo": 1})
338def test_insert_no_values(client):
339 """Inserting with no values raises before any request."""
340 with pytest.raises(ValueError, match="No values to insert"):
341 client.insert("latiss", "exposure", 271828, {})
344@responses.activate
345def test_getCcdVisitTableForDay_dedupes_overlapping_columns(client):
346 """Columns already present in ccdvisit1_quicklook must not be re-selected.
348 ``cvq.*`` pulls in every column of ccdvisit1_quicklook. As ConsDB
349 denormalises identity columns (visit_id, detector, seq_num, ...) onto the
350 quicklook table, naming those again explicitly from the joined tables makes
351 the server return duplicate column names, which astropy refuses to build a
352 Table from (DM-55152). They must be dropped from the SELECT instead.
353 """
354 url = "http://example.com/consdb/query"
356 # First query is the LIMIT 0 schema probe: pretend the quicklook table has
357 # denormalised visit_id, detector and seq_num onto itself.
358 responses.post(
359 url,
360 json={"columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma"], "data": []},
361 )
362 # Second query is the real data query; its response only has to build
363 # cleanly, the behaviour under test is the SELECT clause that was sent.
364 responses.post(
365 url,
366 json={
367 "columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma", "band"],
368 "data": [[1, 100, 7, 5, 1.2, "r"]],
369 },
370 )
372 table = getCcdVisitTableForDay(client, 20240101)
373 assert isinstance(table, Table)
375 # Only inspect the SELECT clause; the WHERE clause legitimately references
376 # cv.visit_id etc. in its join conditions.
377 sentQuery = json.loads(responses.calls[1].request.body)["query"]
378 selectClause = sentQuery.split(" FROM ")[0]
379 # Columns already provided by cvq.* must not be re-selected explicitly...
380 assert "cvq.*" in selectClause
381 assert "cv.visit_id" not in selectClause
382 assert "cv.detector" not in selectClause
383 assert "v.seq_num" not in selectClause
384 # ...but columns the quicklook table lacks must still be pulled from visit1
385 for col in ("v.band", "v.exp_time", "v.day_obs", "v.img_type"):
386 assert col in selectClause
389@responses.activate
390def test_getCcdVisitTableForDay_keeps_columns_when_no_overlap(client):
391 """When the quicklook table shares no names, every extra is selected."""
392 url = "http://example.com/consdb/query"
393 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": []})
394 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": [[1, 1.2]]})
396 getCcdVisitTableForDay(client, 20240101)
398 sentQuery = json.loads(responses.calls[1].request.body)["query"]
399 selectClause = sentQuery.split(" FROM ")[0]
400 for col in ("cv.detector", "cv.visit_id", "v.band", "v.exp_time", "v.seq_num", "v.day_obs", "v.img_type"):
401 assert col in selectClause
404# TODO: more POST tests
405# client.insert_multiple(instrument, table, obs_dict, allow_update)
406# client.query(query)