Coverage for python/lsst/summit/utils/consdbClient.py: 68%
195 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-17 10:13 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24from collections.abc import Mapping
25from dataclasses import dataclass
26from typing import Any
27from urllib.parse import quote, urlparse
29import numpy as np
30import requests
31from astropy.table import Column, Table
33__all__ = ["ConsDbClient", "FlexibleMetadataInfo", "getCcdVisitTableForDay", "getWideQuicklookTableForDay"]
36logger = logging.getLogger(__name__)
38# Timeouts in seconds applied to every request made by ConsDbClient.
39# The read timeout is generous because some queries are genuinely slow.
40DEFAULT_CONNECT_TIMEOUT: float = 10.0
41DEFAULT_READ_TIMEOUT: float = 600.0
44def _urljoin(*args: str) -> str:
45 """Join parts of a URL with slashes.
47 Does not do any quoting. Mostly to remove a level of list-making.
49 Parameters
50 ----------
51 *args : `str`
52 Each parameter is a URL part.
54 Returns
55 -------
56 url : `str`
57 The joined URL.
58 """
59 return "/".join(args)
62def _check_status(r: requests.Response) -> None:
63 """Check the status of an HTTP response and raise if an error.
65 Adds additional response information to the raise_for_status exception.
67 Parameters
68 ----------
69 r : `requests.Response`
70 The response to check.
72 Raises
73 ------
74 requests.HTTPError
75 Raised if a non-successful status is returned.
76 """
77 try:
78 r.raise_for_status()
79 except requests.HTTPError as e:
80 try:
81 # Use ``r`` rather than ``e.response``: raise_for_status always
82 # attaches this response, but its type is Response | None.
83 json_data = r.json()
84 e.add_note(str(json_data))
85 if "message" in json_data: 85 ↛ 89line 85 didn't jump to line 89 because the condition on line 85 was always true
86 e.add_note(f"\n\n{json_data['message']}")
87 except requests.JSONDecodeError:
88 pass
89 raise e
92def clean_url(resp: requests.Response, *args: Any, **kwargs: Any) -> requests.Response:
93 """Parse url from response and remove netloc portion.
95 Set new url in response and return response
97 Parameters
98 ----------
99 resp : `requests.Response`
100 The response that could contain a URL with tokens
101 """
102 url = urlparse(resp.url)
103 short_user = f"{url.username[:2]}***" if url.username is not None else ""
104 short_pass = f":{url.password[:2]}***" if url.password is not None else ""
105 netloc = f"{short_user}{short_pass}@{url.hostname}"
106 resp.url = url._replace(netloc=netloc).geturl()
107 return resp
110@dataclass
111class FlexibleMetadataInfo:
112 """Description of a flexible metadata value.
114 Parameters
115 ----------
116 dtype : `str`
117 Data type of the flexible metadata value.
118 One of ``bool``, ``int``, ``float``, or ``str``.
119 doc : `str`
120 Documentation string.
121 unit : `str`, optional
122 Unit of the value.
123 ucd : `str`, optional
124 IVOA Unified Content Descriptor
125 (https://www.ivoa.net/documents/UCD1+/).
126 """
128 dtype: str
129 doc: str
130 unit: str | None = None
131 ucd: str | None = None
134class ConsDbClient:
135 """A client library for accessing the Consolidated Database.
137 This library provides a basic interface for using flexible metadata
138 (key/value pairs associated with observation ids from an observation
139 type table), determining the schema of ConsDB tables, querying the
140 ConsDB using a general SQL SELECT statement, and inserting into
141 ConsDB tables.
143 Parameters
144 ----------
145 url : `str` | `None`
146 Base URL of the Web service, defaults to the value of environment
147 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query
148 service).
149 token : `str` | `None`
150 Authentication token for the RSP. The token must begin with "gt-".
151 connect_timeout : `float` | `None`
152 Seconds to wait for the connection to be established, defaults to
153 `DEFAULT_CONNECT_TIMEOUT`.
154 read_timeout : `float` | `None`
155 Seconds to wait between bytes sent by the server, defaults to
156 `DEFAULT_READ_TIMEOUT`. Pass `None` to wait indefinitely, which is
157 rarely what you want.
159 Notes
160 -----
161 This client is a thin layer over the publish/query Web service, which
162 avoids having a dependency on database drivers.
164 It enforces the return of query results as Astropy Tables.
165 """
167 def __init__(
168 self,
169 url: str | None = None,
170 token: str | None = None,
171 connect_timeout: float | None = DEFAULT_CONNECT_TIMEOUT,
172 read_timeout: float | None = DEFAULT_READ_TIMEOUT,
173 ):
174 self.session = requests.Session()
175 self.session.hooks["response"].append(clean_url)
176 self.connect_timeout = connect_timeout
177 self.read_timeout = read_timeout
179 if token is not None: 179 ↛ 180line 179 didn't jump to line 180 because the condition on line 179 was never true
180 if not token.startswith("gt-"):
181 raise ValueError("token must start with `gt-`.")
183 self.session.headers.update({"Authorization": f"Bearer {token}"})
185 if url is None: 185 ↛ 186line 185 didn't jump to line 186 because the condition on line 185 was never true
186 self.url = os.environ["LSST_CONSDB_PQ_URL"]
187 else:
188 self.url = url
189 self.url = self.url.rstrip("/")
191 @property
192 def timeout(self) -> tuple[float | None, float | None]:
193 """The ``(connect, read)`` timeout pair passed to requests
194 (`tuple` [ `float` | `None`, `float` | `None` ], read-only).
195 """
196 return (self.connect_timeout, self.read_timeout)
198 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any:
199 """Submit GET requests to the server.
201 Parameters
202 ----------
203 url : `str`
204 URL to GET.
205 query : `dict` [`str`, `str` | `list` [`str`]], optional
206 Query parameters to attach to the URL.
208 Raises
209 ------
210 requests.RequestException
211 Raised if any kind of connection error occurs.
212 requests.Timeout
213 Raised if the request exceeds the client timeout.
214 requests.HTTPError
215 Raised if a non-successful status is returned.
216 requests.JSONDecodeError
217 Raised if the result does not decode as JSON.
219 Returns
220 -------
221 result : `Any`
222 Result of decoding the Web service result content as JSON.
223 """
224 logger.debug(f"GET {url}")
225 response = self.session.get(url, params=query, timeout=self.timeout)
226 _check_status(response)
227 return response.json()
229 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response:
230 """Submit POST requests to the server.
232 Parameters
233 ----------
234 url : `str`
235 URL to POST.
236 data : `dict` [`str`, `Any`]
237 Key/value pairs of data to POST.
239 Raises
240 ------
241 requests.RequestException
242 Raised if any kind of connection error occurs.
243 requests.Timeout
244 Raised if the request exceeds the client timeout.
245 requests.HTTPError
246 Raised if a non-successful status is returned.
248 Returns
249 -------
250 result : `requests.Response`
251 The raw Web service result object.
252 """
253 logger.debug(f"POST {url}: {data}")
254 response = self.session.post(url, json=data, timeout=self.timeout)
255 _check_status(response)
256 return response
258 @staticmethod
259 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str:
260 """Compute the name of a flexible metadata table.
262 Each instrument and observation type made with that instrument can
263 have a flexible metadata table. This function is useful when
264 issuing SQL queries, and it avoids a round-trip to the server.
266 Parameters
267 ----------
268 instrument : `str`
269 Name of the instrument (e.g. ``LATISS``).
270 obs_type : `str`
271 Name of the observation type (e.g. ``Exposure``).
273 Returns
274 -------
275 table_name : `str`
276 Name of the appropriate flexible metadata table.
277 """
278 return f"cdb_{instrument}.{obs_type}_flexdata"
280 @staticmethod
281 def compute_fixed_metadata_namespace(instrument: str) -> str:
282 """Compute the namespace for a fixed metadata table.
284 Each instrument has its own namespace in the ConsDB.
285 This function is useful when issuing SQL queries, and it avoids a
286 round-trip to the server.
288 Parameters
289 ----------
290 instrument : `str`
291 Name of the instrument (e.g. ``LATISS``).
293 Returns
294 -------
295 namespace_name : `str`
296 Name of the appropriate namespace
297 """
298 return f"cdb_{instrument}"
300 def add_flexible_metadata_key(
301 self,
302 instrument: str,
303 obs_type: str,
304 key: str,
305 dtype: str,
306 doc: str,
307 unit: str | None = None,
308 ucd: str | None = None,
309 ) -> requests.Response:
310 """Add a key to a flexible metadata table.
312 Parameters
313 ----------
314 instrument : `str`
315 Name of the instrument (e.g. ``LATISS``).
316 obs_type : `str`
317 Name of the observation type (e.g. ``Exposure``).
318 key : `str`
319 Name of the key to be added (must not already exist).
320 dtype : `str`
321 One of ``bool``, ``int``, ``float``, or ``str``.
322 doc : `str`
323 Documentation string for the key.
324 unit : `str`, optional
325 Unit for the value. Should be from the IVOA
326 (https://www.ivoa.net/documents/VOUnits/) or astropy.
327 ucd : `str`, optional
328 IVOA Unified Content Descriptor
329 (https://www.ivoa.net/documents/UCD1+/).
331 Returns
332 -------
333 response : `requests.Response`
334 HTTP response from the server, with 200 status for success.
336 Raises
337 ------
338 requests.RequestException
339 Raised if any kind of connection error occurs.
340 requests.HTTPError
341 Raised if a non-successful status is returned.
342 """
343 data = {"key": key, "dtype": dtype, "doc": doc}
344 if unit is not None: 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true
345 data["unit"] = unit
346 if ucd is not None: 346 ↛ 347line 346 didn't jump to line 347 because the condition on line 346 was never true
347 data["ucd"] = ucd
348 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey")
349 return self._handle_post(url, data)
351 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]:
352 """Retrieve the valid keys for a flexible metadata table.
354 Parameters
355 ----------
356 instrument : `str`
357 Name of the instrument (e.g. ``LATISS``).
358 obs_type : `str`
359 Name of the observation type (e.g. ``Exposure``).
361 Returns
362 -------
363 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ]
364 Dict of keys and information values.
366 Raises
367 ------
368 requests.RequestException
369 Raised if any kind of connection error occurs.
370 requests.HTTPError
371 Raised if a non-successful status is returned.
372 """
373 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema")
374 result = self._handle_get(url)
375 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()}
377 def get_flexible_metadata(
378 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None
379 ) -> dict[str, Any]:
380 """Get the flexible metadata for an observation.
382 Parameters
383 ----------
384 instrument : `str`
385 Name of the instrument (e.g. ``LATISS``).
386 obs_type : `str`
387 Name of the observation type (e.g. ``Exposure``).
388 obs_id : `int`
389 Unique observation id.
390 keys : `list` [ `str` ], optional
391 List of keys to be retrieved; all if not specified.
393 Returns
394 -------
395 result_dict : `dict` [ `str`, `Any` ]
396 Dictionary of key/value pairs for the observation.
398 Raises
399 ------
400 requests.RequestException
401 Raised if any kind of connection error occurs.
402 requests.HTTPError
403 Raised if a non-successful status is returned.
404 """
405 url = _urljoin(
406 self.url,
407 "flex",
408 quote(instrument),
409 quote(obs_type),
410 "obs",
411 quote(str(obs_id)),
412 )
413 return self._handle_get(url, {"k": keys} if keys else None)
415 def get_all_metadata(
416 self, instrument: str, obs_type: str, obs_id: int, flex: bool = False
417 ) -> dict[str, Any]:
418 """Get all metadata for an observation.
420 Parameters
421 ----------
422 instrument : `str`
423 Name of the instrument (e.g. ``LATISS``).
424 obs_type : `str`
425 Name of the observation type (e.g. ``Exposure``).
426 obs_id : `int`
427 Unique observation id.
428 flex : `bool`
429 Include flexible metadata.
431 Returns
432 -------
433 result_dict : `dict` [ `str`, `Any` ]
434 Dictionary of key/value pairs for the observation.
436 Raises
437 ------
438 requests.RequestException
439 Raised if any kind of connection error occurs.
440 requests.HTTPError
441 Raised if a non-successful status is returned.
442 """
443 url = _urljoin(
444 self.url,
445 "query",
446 quote(instrument),
447 quote(obs_type),
448 "obs",
449 quote(str(obs_id)),
450 )
451 return self._handle_get(url, {"flex": "1"} if flex else None)
453 def insert_flexible_metadata(
454 self,
455 instrument: str,
456 obs_type: str,
457 obs_id: int,
458 values: dict[str, Any] | None = None,
459 *,
460 allow_update: bool = False,
461 **kwargs: Any,
462 ) -> requests.Response:
463 """Set flexible metadata values for an observation.
465 Parameters
466 ----------
467 instrument : `str`
468 Name of the instrument (e.g. ``LATISS``).
469 obs_type : `str`
470 Name of the observation type (e.g. ``Exposure``).
471 obs_id : `int`
472 Unique observation id.
473 values : `dict` [ `str`, `Any` ], optional
474 Dictionary of key/value pairs to add for the observation.
475 allow_update : `bool`, optional
476 If ``True``, allow replacement of values of existing keys.
477 **kwargs : `dict`
478 Additional key/value pairs, overriding ``values``.
480 Returns
481 -------
482 response : `requests.Response`
483 HTTP response from the server, with 200 status for success.
485 Raises
486 ------
487 ValueError
488 Raised if no values are provided in ``values`` or kwargs.
489 requests.RequestException
490 Raised if any kind of connection error occurs.
491 requests.HTTPError
492 Raised if a non-successful status is returned.
493 """
494 if values: 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true
495 values.update(kwargs)
496 else:
497 values = kwargs
498 if not values: 498 ↛ 500line 498 didn't jump to line 500 because the condition on line 498 was always true
499 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}")
500 data = {"values": values}
501 url = _urljoin(
502 self.url,
503 "flex",
504 quote(instrument),
505 quote(obs_type),
506 "obs",
507 quote(str(obs_id)),
508 )
509 if allow_update:
510 url += "?u=1"
511 return self._handle_post(url, data)
513 def insert(
514 self,
515 instrument: str,
516 table: str,
517 obs_id: tuple[int, int] | tuple[int, int, int] | int,
518 values: Mapping[str, Any],
519 *,
520 allow_update: bool = False,
521 **kwargs: Any,
522 ) -> requests.Response:
523 """Insert values into a single ConsDB fixed metadata table.
525 Parameters
526 ----------
527 instrument : `str`
528 Name of the instrument (e.g. ``LATISS``).
529 table : `str`
530 Name of the table to insert into.
531 obs_id : `tuple`[`int`, `int`] | `tuple`[`int`, `int`, `int`] | `int`
532 How to address the row to insert:
534 - `int`: the natural observation id (e.g. ``exposure_id``,
535 ``ccdexposure_id``), targeting the ``.../obs/{obs_id}``
536 endpoint.
537 - 2-tuple ``(day_obs, seq_num)``: targeting the
538 ``.../by_seq_num/{day_obs}/{seq_num}`` endpoint, for
539 exposure-level tables.
540 - 3-tuple ``(day_obs, seq_num, detector)``: targeting the
541 ``.../by_seq_num/{day_obs}/{seq_num}/{detector}`` endpoint,
542 for ccdexposure-level (per-detector) tables.
543 values : `Mapping` [ `str`, `Any` ]
544 Dictionary-like mapping of column/value pairs to add for the
545 observation.
546 allow_update : `bool`, optional
547 If ``True``, allow replacement of values of existing columns.
548 **kwargs : `dict`
549 Additional column/value pairs, overriding ``values``.
551 Returns
552 -------
553 response : `requests.Response`
554 HTTP response from the server, with 200 status for success.
556 Raises
557 ------
558 ValueError
559 Raised if no values are provided in ``values`` or kwargs.
560 requests.RequestException
561 Raised if any kind of connection error occurs.
562 requests.HTTPError
563 Raised if a non-successful status is returned.
564 """
565 # Build a new merged dict to avoid mutating the incoming Mapping.
566 merged_values: dict[str, Any] = {**(dict(values) if values else {}), **kwargs}
567 if not merged_values:
568 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}")
570 data: dict[str, Any]
571 if isinstance(obs_id, tuple):
572 assert len(obs_id) in (
573 2,
574 3,
575 ), f"obs_id tuple must be (day_obs, seq_num) or (day_obs, seq_num, detector); got {obs_id!r}"
577 data = {"table": table, "values": merged_values}
578 url = _urljoin(
579 self.url,
580 "insert",
581 quote(instrument),
582 quote(table),
583 "by_seq_num",
584 # Two segments for exposure-level tables, three when a
585 # detector is supplied for ccdexposure-level tables.
586 *(quote(str(part)) for part in obs_id),
587 )
588 else:
589 data = {"table": table, "obs_id": obs_id, "values": merged_values}
590 url = _urljoin(
591 self.url,
592 "insert",
593 quote(instrument),
594 quote(table),
595 "obs",
596 quote(str(obs_id)),
597 )
598 if allow_update:
599 url += "?u=1"
600 return self._handle_post(url, data)
602 def insert_multiple(
603 self,
604 instrument: str,
605 table: str,
606 obs_dict: dict[int, dict[str, Any]],
607 *,
608 allow_update: bool = False,
609 ) -> requests.Response:
610 """Insert values into a single ConsDB fixed metadata table.
612 Parameters
613 ----------
614 instrument : `str`
615 Name of the instrument (e.g. ``LATISS``).
616 table : `str`
617 Name of the table to insert into.
618 obs_dict : `dict` [ `int`, `dict` [ `str`, `Any` ] ]
619 Dictionary of observation ids, each with a dictionary of
620 column/value pairs to add for each observation.
621 allow_update : `bool`, optional
622 If ``True``, allow replacement of values of existing columns.
624 Returns
625 -------
626 response : `requests.Response`
627 HTTP response from the server, with 200 status for success.
629 Raises
630 ------
631 ValueError
632 Raised if no values are provided in ``obs_dict``.
633 requests.RequestException
634 Raised if any kind of connection error occurs.
635 requests.HTTPError
636 Raised if a non-successful status is returned.
637 """
638 if not obs_dict:
639 raise ValueError(f"No values to insert for {instrument} {table}")
640 data = {"table": table, "obs_dict": obs_dict}
641 url = _urljoin(
642 self.url,
643 "insert",
644 quote(instrument),
645 quote(table),
646 )
647 if allow_update:
648 url += "?u=1"
649 return self._handle_post(url, data)
651 def query(self, query: str) -> Table:
652 """Query the ConsDB database.
654 Parameters
655 ----------
656 query : `str`
657 A SQL query (currently) to the database.
659 Returns
660 -------
661 result : `Table`
662 An ``astropy.Table`` containing the query results.
664 Raises
665 ------
666 requests.RequestException
667 Raised if any kind of connection error occurs.
668 requests.HTTPError
669 Raised if a non-successful status is returned.
671 Notes
672 -----
673 This is a very general query interface because it is expected that
674 a wide variety of types of queries will be needed. If some types prove
675 to be common, syntactic sugar could be added to make them simpler.
676 """
677 url = _urljoin(self.url, "query")
678 data = {"query": query}
679 result = self._handle_post(url, data).json()
681 columns = result.get("columns", [])
682 if not columns: 682 ↛ 684line 682 didn't jump to line 684 because the condition on line 682 was never true
683 # No result columns
684 return Table(rows=[])
686 rows = result.get("data", [])
687 if not rows:
688 # No result rows
689 return Table(names=columns)
691 return Table(rows=rows, names=columns)
693 def schema(
694 self, instrument: str | None = None, table: str | None = None
695 ) -> dict[str, tuple[str, str]] | list[str]:
696 """Retrieve information about ConsDB.
698 If ``instrument`` and ``table`` are given, return the schema of a
699 fixed metadata table in ConsDB.
701 If only ``instrument`` is given, return the names of all tables
702 for that instrument.
704 If no arguments are given, return the names of all instruments.
706 Parameters
707 ----------
708 instrument : `str`, optional
709 Name of the instrument (e.g. ``LATISS``).
710 table : `str`, optional
711 Name of the table to insert into.
713 Returns
714 -------
715 info : `list` [ `str` ] or `dict` [ `str`, `tuple` [ `str`, `str` ] ]
716 A list of instrument strings or table names, or else a dict of
717 columns with values that are tuples containing a data type string
718 and a documentation string.
720 Raises
721 ------
722 ValueError
723 Raised if only ``table`` is given.
724 requests.RequestException
725 Raised if any kind of connection error occurs.
726 requests.HTTPError
727 Raised if a non-successful status is returned.
729 Notes
730 -----
731 Fixed metadata data types may use the full database vocabulary,
732 unlike flexible metadata data types.
733 """
734 if instrument is None:
735 if table is not None: 735 ↛ 736line 735 didn't jump to line 736 because the condition on line 735 was never true
736 raise ValueError("Must specify instrument if table is given")
737 url = _urljoin(self.url, "schema")
738 elif table is None: 738 ↛ 739line 738 didn't jump to line 739 because the condition on line 738 was never true
739 url = _urljoin(self.url, "schema", quote(instrument))
740 else:
741 url = _urljoin(self.url, "schema", quote(instrument), quote(table))
742 result = self._handle_get(url)
743 if instrument is not None and table is not None:
744 return {key: (str(value[0]), str(value[1])) for key, value in result.items()}
745 else:
746 return [str(value) for value in result]
749def getCcdVisitTableForDay(
750 client: ConsDbClient,
751 dayObs: int,
752 visitTableItems: list[str] | None = None,
753 detectors: list[int] | None = None,
754 withZeropoint: bool = False,
755) -> Table:
756 """Get the ccdvisit1_quicklook table for a given dayObs.
758 Parameters
759 ----------
760 client : `ConsDbClient`
761 The ConsDbClient to use.
762 dayObs : `int`
763 The dayObs to query for.
764 visitTableItems : `list` of `str`, optional
765 Additional items from the visit1 table to include.
766 detectors : `list` of `int`, optional
767 If given, only return rows for these detectors.
768 withZeropoint : `bool`, optional
769 If ``True``, only return rows with a non-null zeropoint.
771 Returns
772 -------
773 table : `astropy.table.Table`
774 The resulting table.
775 """
776 # ``cvq.*`` already pulls in every column of ccdvisit1_quicklook, so any
777 # column we also name explicitly from ccdvisit1/visit1 has to be dropped
778 # from the SELECT when the quicklook table itself already defines it.
779 # ConsDB has been denormalising identity columns (visit_id, detector,
780 # seq_num, ...) onto the quicklook tables, and selecting such a column
781 # twice makes the server return duplicate column names, which astropy then
782 # refuses to build a Table from. getWideQuicklookTableForDay does the same
783 # dedup for the visit1 join.
784 cvqCols = set(client.query("SELECT * FROM cdb_LSSTCam.ccdvisit1_quicklook LIMIT 0").colnames)
786 visitItems = ["band", "exp_time", "seq_num", "day_obs", "img_type"]
787 if visitTableItems: 787 ↛ 788line 787 didn't jump to line 788 because the condition on line 787 was never true
788 visitItems += visitTableItems
790 selectClauses = ["cvq.*"]
791 claimed = set(cvqCols) # cvq.* already provides all of these names
792 for col in ("detector", "visit_id"):
793 if col not in claimed:
794 selectClauses.append(f"cv.{col}")
795 claimed.add(col)
796 for col in visitItems:
797 if col not in claimed:
798 selectClauses.append(f"v.{col}")
799 claimed.add(col)
801 query = (
802 f"SELECT {', '.join(selectClauses)} "
803 "FROM cdb_LSSTCam.ccdvisit1_quicklook as cvq, "
804 "cdb_LSSTCam.ccdvisit1 as cv, "
805 "cdb_LSSTCam.visit1 as v "
806 )
807 where = f"WHERE cvq.ccdvisit_id=cv.ccdvisit_id and cv.visit_id=v.visit_id and v.day_obs={dayObs}"
808 if detectors: 808 ↛ 809line 808 didn't jump to line 809 because the condition on line 808 was never true
809 where += f" and cv.detector in ({','.join([str(d) for d in detectors])})"
810 if withZeropoint: 810 ↛ 811line 810 didn't jump to line 811 because the condition on line 810 was never true
811 where += " and cvq.zero_point is not null"
813 table = client.query(query + where)
814 return table
817def columnsEqual(a: Column, b: Column) -> bool:
818 """Check if two columns are equal, taking masks into account.
820 Parameters
821 ----------
822 a : `Column`
823 First column to compare.
824 b : `Column`
825 Second column to compare.
827 Returns
828 -------
829 equal : `bool`
830 True if the columns are equal, False otherwise.
831 """
832 aArr = np.asanyarray(a)
833 bArr = np.asanyarray(b)
834 if aArr.shape != bArr.shape:
835 return False
837 aMask = getattr(a, "mask", None)
838 bMask = getattr(b, "mask", None)
840 if aMask is None and bMask is None:
841 return bool(np.all(aArr == bArr))
843 if aMask is None:
844 aMask = np.zeros(aArr.shape, dtype=bool)
845 if bMask is None:
846 bMask = np.zeros(bArr.shape, dtype=bool)
848 aMaskArr = np.asanyarray(aMask)
849 bMaskArr = np.asanyarray(bMask)
851 if np.any(aMaskArr ^ bMaskArr):
852 return False # one masked where the other isn't
854 present = ~aMaskArr
855 return bool(np.all(aArr[present] == bArr[present]))
858def getWideQuicklookTableForDay(client: ConsDbClient, dayObs: int) -> Table:
859 """Get a wide quicklook table for a given dayObs.
861 Joins all columns from the visit1 table to the visit1_quicklook table. Note
862 that the visit1 table already contains all the columns from the exposure
863 table, and is just keyed by the exposure_id instead of the visit_id.
865 Parameters
866 ----------
867 client : `ConsDbClient`
868 The ConsDbClient to use.
869 dayObs : `int`
870 The dayObs to query for.
872 Returns
873 -------
874 table : `astropy.table.Table`
875 The resulting wide quicklook table.
876 """
877 vqCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1_quicklook LIMIT 0").colnames)
878 vCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1 LIMIT 0").colnames)
880 vOnlyCols = vCols - vqCols # exclude visit_id and all duplicates
882 selectClauses = ["vq.*"] + [f"v.{col}" for col in sorted(vOnlyCols)]
884 query = f"""
885 SELECT {', '.join(selectClauses)}
886 FROM cdb_LSSTCam.visit1_quicklook vq
887 INNER JOIN cdb_LSSTCam.visit1 v USING (visit_id)
888 WHERE vq.day_obs = {dayObs}
889 """
890 return client.query(query)