Coverage for tests/test_consdbClient.py: 29%

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import json 

23 

24import pytest 

25import responses 

26from astropy.table import Table 

27from requests import HTTPError 

28 

29from lsst.summit.utils import ConsDbClient, FlexibleMetadataInfo 

30from lsst.summit.utils.consdbClient import getCcdVisitTableForDay 

31 

32 

33@pytest.fixture 

34def client(): 

35 """Initialize client with a fake url 

36 Requires mocking connection with @responses.activate decorator 

37 """ 

38 return ConsDbClient("http://example.com/consdb") 

39 

40 

41def test_table_name(): 

42 instrument = "latiss" 

43 obs_type = "exposure" 

44 assert ( 

45 ConsDbClient.compute_flexible_metadata_table_name(instrument, obs_type) 

46 == "cdb_latiss.exposure_flexdata" 

47 ) 

48 

49 

50@responses.activate 

51def test_add_flexible_metadata_key(client): 

52 instrument = "latiss" 

53 obs_type = "exposure" 

54 responses.post( 

55 "http://example.com/consdb/flex/latiss/exposure/addkey", 

56 json={ 

57 "message": "Key added to flexible metadata", 

58 "key": "foo", 

59 "instrument": "latiss", 

60 "obs_type": "exposure", 

61 }, 

62 match=[ 

63 responses.matchers.json_params_matcher({"key": "foo", "dtype": "bool", "doc": "bool key"}), 

64 ], 

65 ) 

66 responses.post( 

67 "http://example.com/consdb/flex/latiss/exposure/addkey", 

68 json={ 

69 "message": "Key added to flexible metadata", 

70 "key": "bar", 

71 "instrument": "latiss", 

72 "obs_type": "exposure", 

73 }, 

74 match=[ 

75 responses.matchers.json_params_matcher({"key": "bar", "dtype": "int", "doc": "int key"}), 

76 ], 

77 ) 

78 responses.post( 

79 "http://example.com/consdb/flex/bad_instrument/exposure/addkey", 

80 status=404, 

81 json={"message": "Unknown instrument", "value": "bad_instrument", "valid": ["latiss"]}, 

82 ) 

83 responses.post( 

84 "http://example.com/consdb/flex/latiss/bad_obs_type/addkey", 

85 status=404, 

86 json={"message": "Unknown observation type", "value": "bad_obs_type", "valid": ["exposure"]}, 

87 ) 

88 

89 assert ( 

90 client.add_flexible_metadata_key(instrument, obs_type, "foo", "bool", "bool key").json()["key"] 

91 == "foo" 

92 ) 

93 assert ( 

94 client.add_flexible_metadata_key(instrument, obs_type, "bar", "int", "int key").json()["instrument"] 

95 == "latiss" 

96 ) 

97 with pytest.raises(HTTPError, match="404") as e: 

98 client.add_flexible_metadata_key("bad_instrument", obs_type, "error", "int", "instrument error") 

99 assert "Unknown instrument" in str(e.value.__notes__) 

100 json_data = e.value.response.json() 

101 assert json_data["message"] == "Unknown instrument" 

102 assert json_data["value"] == "bad_instrument" 

103 assert json_data["valid"] == ["latiss"] 

104 with pytest.raises(HTTPError, match="404"): 

105 client.add_flexible_metadata_key(instrument, "bad_obs_type", "error", "int", "obs_type error") 

106 

107 

108@responses.activate 

109def test_get_flexible_metadata_keys(client): 

110 description = {"foo": ["bool", "a", None, None], "bar": ["float", "b", "deg", "pos.eq.ra"]} 

111 responses.get( 

112 "http://example.com/consdb/flex/latiss/exposure/schema", 

113 json=description, 

114 ) 

115 instrument = "latiss" 

116 obs_type = "exposure" 

117 assert client.get_flexible_metadata_keys(instrument, obs_type) == { 

118 "foo": FlexibleMetadataInfo("bool", "a"), 

119 "bar": FlexibleMetadataInfo("float", "b", "deg", "pos.eq.ra"), 

120 } 

121 

122 

123@responses.activate 

124def test_get_flexible_metadata(client): 

125 results = {"bool_key": True, "int_key": 42, "float_key": 3.14159, "str_key": "foo"} 

126 responses.get( 

127 "http://example.com/consdb/flex/latiss/exposure/obs/271828", 

128 json=results, 

129 ) 

130 responses.get( 

131 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=float_key", json={"float_key": 3.14159} 

132 ) 

133 responses.get( 

134 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=int_key&k=float_key", 

135 json={"float_key": 3.14159, "int_key": 42}, 

136 ) 

137 instrument = "latiss" 

138 obs_type = "exposure" 

139 obs_id = 271828 

140 assert client.get_flexible_metadata(instrument, obs_type, obs_id) == results 

141 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["float_key"]) == { 

142 "float_key": results["float_key"] 

143 } 

144 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["int_key", "float_key"]) == { 

145 "int_key": results["int_key"], 

146 "float_key": results["float_key"], 

147 } 

148 

149 

150@responses.activate 

151def test_insert_flexible_metadata(client): 

152 instrument = "latiss" 

153 obs_type = "exposure" 

154 with pytest.raises(ValueError): 

155 client.insert_flexible_metadata(instrument, obs_type, 271828) 

156 # TODO: more POST tests 

157 

158 

159@responses.activate 

160def test_schema(client): 

161 description = {"foo": ("bool", "a"), "bar": ("int", "b")} 

162 responses.get( 

163 "http://example.com/consdb/schema/latiss/misc_table", 

164 json=description, 

165 ) 

166 instrument = "latiss" 

167 table = "misc_table" 

168 assert client.schema(instrument, table) == description 

169 

170 

171@responses.activate 

172@pytest.mark.parametrize( 

173 "secret, redacted", 

174 [ 

175 ("usdf:v987wefVMPz", "us***:v9***"), 

176 ("u:v", "u***:v***"), 

177 ("ulysses", "ul***"), 

178 (":alberta94", "***:al***"), 

179 ], 

180) 

181def test_clean_token_url_response(secret, redacted): 

182 """Test tokens URL is cleaned when an error is thrown from requests 

183 Use with pytest raises assert an error' 

184 assert that url does not contain tokens 

185 """ 

186 domain = "@usdf-fake.slackers.stanford.edu/consdb" 

187 complex_client = ConsDbClient(f"https://{secret}{domain}") 

188 

189 obs_type = "exposure" 

190 responses.post( 

191 f"https://{secret}{domain}/flex/bad_instrument/exposure/addkey", 

192 status=404, 

193 ) 

194 with pytest.raises(HTTPError, match="404") as error: 

195 complex_client.add_flexible_metadata_key( 

196 "bad_instrument", obs_type, "error", "int", "instrument error" 

197 ) 

198 

199 url = error.value.args[0].split()[-1] 

200 sanitized = f"https://{redacted}{domain}/flex/bad_instrument/exposure/addkey" 

201 assert url == sanitized 

202 

203 

204def test_client(client): 

205 """Test ConsDbClient is initialized properly""" 

206 assert "clean_url" in str(client.session.hooks["response"]) 

207 

208 

209@responses.activate 

210def test_insert_obs_id(client): 

211 """An integer obs_id targets the ``.../obs/{obs_id}`` endpoint.""" 

212 responses.post( 

213 "http://example.com/consdb/insert/latiss/exposure/obs/271828", 

214 json={"message": "Data inserted"}, 

215 match=[ 

216 responses.matchers.json_params_matcher( 

217 {"table": "exposure", "obs_id": 271828, "values": {"foo": 1}} 

218 ), 

219 ], 

220 ) 

221 assert client.insert("latiss", "exposure", 271828, {"foo": 1}).json()["message"] == "Data inserted" 

222 

223 

224@responses.activate 

225def test_insert_by_seq_num(client): 

226 """A 2-tuple targets the ``.../{day_obs}/{seq_num}`` endpoint.""" 

227 responses.post( 

228 "http://example.com/consdb/insert/latiss/exposure/by_seq_num/20240603/123", 

229 json={"message": "Data inserted"}, 

230 match=[ 

231 responses.matchers.json_params_matcher({"table": "exposure", "values": {"foo": 1}}), 

232 ], 

233 ) 

234 assert ( 

235 client.insert("latiss", "exposure", (20240603, 123), {"foo": 1}).json()["message"] == "Data inserted" 

236 ) 

237 

238 

239@responses.activate 

240def test_insert_by_seq_num_detector(client): 

241 """A 3-tuple targets the ``.../{day_obs}/{seq_num}/{detector}`` 

242 endpoint for per-detector (ccdexposure-level) tables.""" 

243 responses.post( 

244 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94", 

245 json={"message": "Data inserted"}, 

246 match=[ 

247 responses.matchers.json_params_matcher({"table": "ccdexposure", "values": {"foo": 1}}), 

248 ], 

249 ) 

250 assert ( 

251 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}).json()["message"] 

252 == "Data inserted" 

253 ) 

254 

255 

256@responses.activate 

257def test_insert_allow_update(client): 

258 """allow_update appends ``?u=1`` to upsert against the addressed key.""" 

259 responses.post( 

260 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94?u=1", 

261 json={"message": "Data inserted"}, 

262 match=[responses.matchers.query_param_matcher({"u": "1"})], 

263 ) 

264 assert ( 

265 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}, allow_update=True).json()[ 

266 "message" 

267 ] 

268 == "Data inserted" 

269 ) 

270 

271 

272def test_insert_bad_obs_id_tuple(client): 

273 """A tuple that is not length 2 or 3 is rejected before any request.""" 

274 with pytest.raises(AssertionError, match="obs_id tuple"): 

275 client.insert("latiss", "exposure", (20240603,), {"foo": 1}) 

276 with pytest.raises(AssertionError, match="obs_id tuple"): 

277 client.insert("latiss", "exposure", (20240603, 123, 94, 0), {"foo": 1}) 

278 

279 

280def test_insert_no_values(client): 

281 """Inserting with no values raises before any request.""" 

282 with pytest.raises(ValueError, match="No values to insert"): 

283 client.insert("latiss", "exposure", 271828, {}) 

284 

285 

286@responses.activate 

287def test_getCcdVisitTableForDay_dedupes_overlapping_columns(client): 

288 """Columns already present in ccdvisit1_quicklook must not be re-selected. 

289 

290 ``cvq.*`` pulls in every column of ccdvisit1_quicklook. As ConsDB 

291 denormalises identity columns (visit_id, detector, seq_num, ...) onto the 

292 quicklook table, naming those again explicitly from the joined tables makes 

293 the server return duplicate column names, which astropy refuses to build a 

294 Table from (DM-55152). They must be dropped from the SELECT instead. 

295 """ 

296 url = "http://example.com/consdb/query" 

297 

298 # First query is the LIMIT 0 schema probe: pretend the quicklook table has 

299 # denormalised visit_id, detector and seq_num onto itself. 

300 responses.post( 

301 url, 

302 json={"columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma"], "data": []}, 

303 ) 

304 # Second query is the real data query; its response only has to build 

305 # cleanly, the behaviour under test is the SELECT clause that was sent. 

306 responses.post( 

307 url, 

308 json={ 

309 "columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma", "band"], 

310 "data": [[1, 100, 7, 5, 1.2, "r"]], 

311 }, 

312 ) 

313 

314 table = getCcdVisitTableForDay(client, 20240101) 

315 assert isinstance(table, Table) 

316 

317 # Only inspect the SELECT clause; the WHERE clause legitimately references 

318 # cv.visit_id etc. in its join conditions. 

319 sentQuery = json.loads(responses.calls[1].request.body)["query"] 

320 selectClause = sentQuery.split(" FROM ")[0] 

321 # Columns already provided by cvq.* must not be re-selected explicitly... 

322 assert "cvq.*" in selectClause 

323 assert "cv.visit_id" not in selectClause 

324 assert "cv.detector" not in selectClause 

325 assert "v.seq_num" not in selectClause 

326 # ...but columns the quicklook table lacks must still be pulled from visit1 

327 for col in ("v.band", "v.exp_time", "v.day_obs", "v.img_type"): 

328 assert col in selectClause 

329 

330 

331@responses.activate 

332def test_getCcdVisitTableForDay_keeps_columns_when_no_overlap(client): 

333 """When the quicklook table shares no names, every extra is selected.""" 

334 url = "http://example.com/consdb/query" 

335 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": []}) 

336 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": [[1, 1.2]]}) 

337 

338 getCcdVisitTableForDay(client, 20240101) 

339 

340 sentQuery = json.loads(responses.calls[1].request.body)["query"] 

341 selectClause = sentQuery.split(" FROM ")[0] 

342 for col in ("cv.detector", "cv.visit_id", "v.band", "v.exp_time", "v.seq_num", "v.day_obs", "v.img_type"): 

343 assert col in selectClause 

344 

345 

346# TODO: more POST tests 

347# client.insert_multiple(instrument, table, obs_dict, allow_update) 

348# client.query(query)