Coverage for python/lsst/summit/utils/consdbClient.py: 68%
195 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-26 10:53 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24from collections.abc import Mapping
25from dataclasses import dataclass
26from typing import Any
27from urllib.parse import quote, urlparse
29import numpy as np
30import requests
31from astropy.table import Column, Table
33__all__ = ["ConsDbClient", "FlexibleMetadataInfo", "getCcdVisitTableForDay", "getWideQuicklookTableForDay"]
36logger = logging.getLogger(__name__)
38# Timeouts in seconds applied to every request made by ConsDbClient.
39# The read timeout is generous because some queries are genuinely slow.
40DEFAULT_CONNECT_TIMEOUT: float = 10.0
41DEFAULT_READ_TIMEOUT: float = 600.0
44def _urljoin(*args: str) -> str:
45 """Join parts of a URL with slashes.
47 Does not do any quoting. Mostly to remove a level of list-making.
49 Parameters
50 ----------
51 *args : `str`
52 Each parameter is a URL part.
54 Returns
55 -------
56 url : `str`
57 The joined URL.
58 """
59 return "/".join(args)
62def _check_status(r: requests.Response) -> None:
63 """Check the status of an HTTP response and raise if an error.
65 Adds additional response information to the raise_for_status exception.
67 Parameters
68 ----------
69 r : `requests.Response`
70 The response to check.
72 Raises
73 ------
74 requests.HTTPError
75 Raised if a non-successful status is returned.
76 """
77 try:
78 r.raise_for_status()
79 except requests.HTTPError as e:
80 try:
81 json_data = e.response.json()
82 e.add_note(str(json_data))
83 if "message" in json_data: 83 ↛ 87line 83 didn't jump to line 87 because the condition on line 83 was always true
84 e.add_note(f"\n\n{json_data['message']}")
85 except requests.JSONDecodeError:
86 pass
87 raise e
90def clean_url(resp: requests.Response, *args, **kwargs) -> requests.Response:
91 """Parse url from response and remove netloc portion.
93 Set new url in response and return response
95 Parameters
96 ----------
97 resp : `requests.Response`
98 The response that could contain a URL with tokens
99 """
100 url = urlparse(resp.url)
101 short_user = f"{url.username[:2]}***" if url.username is not None else ""
102 short_pass = f":{url.password[:2]}***" if url.password is not None else ""
103 netloc = f"{short_user}{short_pass}@{url.hostname}"
104 resp.url = url._replace(netloc=netloc).geturl()
105 return resp
108@dataclass
109class FlexibleMetadataInfo:
110 """Description of a flexible metadata value.
112 Parameters
113 ----------
114 dtype : `str`
115 Data type of the flexible metadata value.
116 One of ``bool``, ``int``, ``float``, or ``str``.
117 doc : `str`
118 Documentation string.
119 unit : `str`, optional
120 Unit of the value.
121 ucd : `str`, optional
122 IVOA Unified Content Descriptor
123 (https://www.ivoa.net/documents/UCD1+/).
124 """
126 dtype: str
127 doc: str
128 unit: str | None = None
129 ucd: str | None = None
132class ConsDbClient:
133 """A client library for accessing the Consolidated Database.
135 This library provides a basic interface for using flexible metadata
136 (key/value pairs associated with observation ids from an observation
137 type table), determining the schema of ConsDB tables, querying the
138 ConsDB using a general SQL SELECT statement, and inserting into
139 ConsDB tables.
141 Parameters
142 ----------
143 url : `str` | `None`
144 Base URL of the Web service, defaults to the value of environment
145 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query
146 service).
147 token : `str` | `None`
148 Authentication token for the RSP. The token must begin with "gt-".
149 connect_timeout : `float` | `None`
150 Seconds to wait for the connection to be established, defaults to
151 `DEFAULT_CONNECT_TIMEOUT`.
152 read_timeout : `float` | `None`
153 Seconds to wait between bytes sent by the server, defaults to
154 `DEFAULT_READ_TIMEOUT`. Pass `None` to wait indefinitely, which is
155 rarely what you want.
157 Notes
158 -----
159 This client is a thin layer over the publish/query Web service, which
160 avoids having a dependency on database drivers.
162 It enforces the return of query results as Astropy Tables.
163 """
165 def __init__(
166 self,
167 url: str | None = None,
168 token: str | None = None,
169 connect_timeout: float | None = DEFAULT_CONNECT_TIMEOUT,
170 read_timeout: float | None = DEFAULT_READ_TIMEOUT,
171 ):
172 self.session = requests.Session()
173 self.session.hooks["response"].append(clean_url)
174 self.connect_timeout = connect_timeout
175 self.read_timeout = read_timeout
177 if token is not None: 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 if not token.startswith("gt-"):
179 raise ValueError("token must start with `gt-`.")
181 self.session.headers.update({"Authorization": f"Bearer {token}"})
183 if url is None: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 self.url = os.environ["LSST_CONSDB_PQ_URL"]
185 else:
186 self.url = url
187 self.url = self.url.rstrip("/")
189 @property
190 def timeout(self) -> tuple[float | None, float | None]:
191 """The ``(connect, read)`` timeout pair passed to requests
192 (`tuple` [ `float` | `None`, `float` | `None` ], read-only).
193 """
194 return (self.connect_timeout, self.read_timeout)
196 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any:
197 """Submit GET requests to the server.
199 Parameters
200 ----------
201 url : `str`
202 URL to GET.
203 query : `dict` [`str`, `str` | `list` [`str`]], optional
204 Query parameters to attach to the URL.
206 Raises
207 ------
208 requests.RequestException
209 Raised if any kind of connection error occurs.
210 requests.Timeout
211 Raised if the request exceeds the client timeout.
212 requests.HTTPError
213 Raised if a non-successful status is returned.
214 requests.JSONDecodeError
215 Raised if the result does not decode as JSON.
217 Returns
218 -------
219 result : `Any`
220 Result of decoding the Web service result content as JSON.
221 """
222 logger.debug(f"GET {url}")
223 response = self.session.get(url, params=query, timeout=self.timeout)
224 _check_status(response)
225 return response.json()
227 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response:
228 """Submit POST requests to the server.
230 Parameters
231 ----------
232 url : `str`
233 URL to POST.
234 data : `dict` [`str`, `Any`]
235 Key/value pairs of data to POST.
237 Raises
238 ------
239 requests.RequestException
240 Raised if any kind of connection error occurs.
241 requests.Timeout
242 Raised if the request exceeds the client timeout.
243 requests.HTTPError
244 Raised if a non-successful status is returned.
246 Returns
247 -------
248 result : `requests.Response`
249 The raw Web service result object.
250 """
251 logger.debug(f"POST {url}: {data}")
252 response = self.session.post(url, json=data, timeout=self.timeout)
253 _check_status(response)
254 return response
256 @staticmethod
257 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str:
258 """Compute the name of a flexible metadata table.
260 Each instrument and observation type made with that instrument can
261 have a flexible metadata table. This function is useful when
262 issuing SQL queries, and it avoids a round-trip to the server.
264 Parameters
265 ----------
266 instrument : `str`
267 Name of the instrument (e.g. ``LATISS``).
268 obs_type : `str`
269 Name of the observation type (e.g. ``Exposure``).
271 Returns
272 -------
273 table_name : `str`
274 Name of the appropriate flexible metadata table.
275 """
276 return f"cdb_{instrument}.{obs_type}_flexdata"
278 @staticmethod
279 def compute_fixed_metadata_namespace(instrument: str) -> str:
280 """Compute the namespace for a fixed metadata table.
282 Each instrument has its own namespace in the ConsDB.
283 This function is useful when issuing SQL queries, and it avoids a
284 round-trip to the server.
286 Parameters
287 ----------
288 instrument : `str`
289 Name of the instrument (e.g. ``LATISS``).
291 Returns
292 -------
293 namespace_name : `str`
294 Name of the appropriate namespace
295 """
296 return f"cdb_{instrument}"
298 def add_flexible_metadata_key(
299 self,
300 instrument: str,
301 obs_type: str,
302 key: str,
303 dtype: str,
304 doc: str,
305 unit: str | None = None,
306 ucd: str | None = None,
307 ) -> requests.Response:
308 """Add a key to a flexible metadata table.
310 Parameters
311 ----------
312 instrument : `str`
313 Name of the instrument (e.g. ``LATISS``).
314 obs_type : `str`
315 Name of the observation type (e.g. ``Exposure``).
316 key : `str`
317 Name of the key to be added (must not already exist).
318 dtype : `str`
319 One of ``bool``, ``int``, ``float``, or ``str``.
320 doc : `str`
321 Documentation string for the key.
322 unit : `str`, optional
323 Unit for the value. Should be from the IVOA
324 (https://www.ivoa.net/documents/VOUnits/) or astropy.
325 ucd : `str`, optional
326 IVOA Unified Content Descriptor
327 (https://www.ivoa.net/documents/UCD1+/).
329 Returns
330 -------
331 response : `requests.Response`
332 HTTP response from the server, with 200 status for success.
334 Raises
335 ------
336 requests.RequestException
337 Raised if any kind of connection error occurs.
338 requests.HTTPError
339 Raised if a non-successful status is returned.
340 """
341 data = {"key": key, "dtype": dtype, "doc": doc}
342 if unit is not None: 342 ↛ 343line 342 didn't jump to line 343 because the condition on line 342 was never true
343 data["unit"] = unit
344 if ucd is not None: 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true
345 data["ucd"] = ucd
346 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey")
347 return self._handle_post(url, data)
349 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]:
350 """Retrieve the valid keys for a flexible metadata table.
352 Parameters
353 ----------
354 instrument : `str`
355 Name of the instrument (e.g. ``LATISS``).
356 obs_type : `str`
357 Name of the observation type (e.g. ``Exposure``).
359 Returns
360 -------
361 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ]
362 Dict of keys and information values.
364 Raises
365 ------
366 requests.RequestException
367 Raised if any kind of connection error occurs.
368 requests.HTTPError
369 Raised if a non-successful status is returned.
370 """
371 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema")
372 result = self._handle_get(url)
373 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()}
375 def get_flexible_metadata(
376 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None
377 ) -> dict[str, Any]:
378 """Get the flexible metadata for an observation.
380 Parameters
381 ----------
382 instrument : `str`
383 Name of the instrument (e.g. ``LATISS``).
384 obs_type : `str`
385 Name of the observation type (e.g. ``Exposure``).
386 obs_id : `int`
387 Unique observation id.
388 keys : `list` [ `str` ], optional
389 List of keys to be retrieved; all if not specified.
391 Returns
392 -------
393 result_dict : `dict` [ `str`, `Any` ]
394 Dictionary of key/value pairs for the observation.
396 Raises
397 ------
398 requests.RequestException
399 Raised if any kind of connection error occurs.
400 requests.HTTPError
401 Raised if a non-successful status is returned.
402 """
403 url = _urljoin(
404 self.url,
405 "flex",
406 quote(instrument),
407 quote(obs_type),
408 "obs",
409 quote(str(obs_id)),
410 )
411 return self._handle_get(url, {"k": keys} if keys else None)
413 def get_all_metadata(
414 self, instrument: str, obs_type: str, obs_id: int, flex: bool = False
415 ) -> dict[str, Any]:
416 """Get all metadata for an observation.
418 Parameters
419 ----------
420 instrument : `str`
421 Name of the instrument (e.g. ``LATISS``).
422 obs_type : `str`
423 Name of the observation type (e.g. ``Exposure``).
424 obs_id : `int`
425 Unique observation id.
426 flex : `bool`
427 Include flexible metadata.
429 Returns
430 -------
431 result_dict : `dict` [ `str`, `Any` ]
432 Dictionary of key/value pairs for the observation.
434 Raises
435 ------
436 requests.RequestException
437 Raised if any kind of connection error occurs.
438 requests.HTTPError
439 Raised if a non-successful status is returned.
440 """
441 url = _urljoin(
442 self.url,
443 "query",
444 quote(instrument),
445 quote(obs_type),
446 "obs",
447 quote(str(obs_id)),
448 )
449 return self._handle_get(url, {"flex": "1"} if flex else None)
451 def insert_flexible_metadata(
452 self,
453 instrument: str,
454 obs_type: str,
455 obs_id: int,
456 values: dict[str, Any] | None = None,
457 *,
458 allow_update: bool = False,
459 **kwargs,
460 ) -> requests.Response:
461 """Set flexible metadata values for an observation.
463 Parameters
464 ----------
465 instrument : `str`
466 Name of the instrument (e.g. ``LATISS``).
467 obs_type : `str`
468 Name of the observation type (e.g. ``Exposure``).
469 obs_id : `int`
470 Unique observation id.
471 values : `dict` [ `str`, `Any` ], optional
472 Dictionary of key/value pairs to add for the observation.
473 allow_update : `bool`, optional
474 If ``True``, allow replacement of values of existing keys.
475 **kwargs : `dict`
476 Additional key/value pairs, overriding ``values``.
478 Returns
479 -------
480 response : `requests.Response`
481 HTTP response from the server, with 200 status for success.
483 Raises
484 ------
485 ValueError
486 Raised if no values are provided in ``values`` or kwargs.
487 requests.RequestException
488 Raised if any kind of connection error occurs.
489 requests.HTTPError
490 Raised if a non-successful status is returned.
491 """
492 if values: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true
493 values.update(kwargs)
494 else:
495 values = kwargs
496 if not values: 496 ↛ 498line 496 didn't jump to line 498 because the condition on line 496 was always true
497 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}")
498 data = {"values": values}
499 url = _urljoin(
500 self.url,
501 "flex",
502 quote(instrument),
503 quote(obs_type),
504 "obs",
505 quote(str(obs_id)),
506 )
507 if allow_update:
508 url += "?u=1"
509 return self._handle_post(url, data)
511 def insert(
512 self,
513 instrument: str,
514 table: str,
515 obs_id: tuple[int, int] | tuple[int, int, int] | int,
516 values: Mapping[str, Any],
517 *,
518 allow_update: bool = False,
519 **kwargs,
520 ) -> requests.Response:
521 """Insert values into a single ConsDB fixed metadata table.
523 Parameters
524 ----------
525 instrument : `str`
526 Name of the instrument (e.g. ``LATISS``).
527 table : `str`
528 Name of the table to insert into.
529 obs_id : `tuple`[`int`, `int`] | `tuple`[`int`, `int`, `int`] | `int`
530 How to address the row to insert:
532 - `int`: the natural observation id (e.g. ``exposure_id``,
533 ``ccdexposure_id``), targeting the ``.../obs/{obs_id}``
534 endpoint.
535 - 2-tuple ``(day_obs, seq_num)``: targeting the
536 ``.../by_seq_num/{day_obs}/{seq_num}`` endpoint, for
537 exposure-level tables.
538 - 3-tuple ``(day_obs, seq_num, detector)``: targeting the
539 ``.../by_seq_num/{day_obs}/{seq_num}/{detector}`` endpoint,
540 for ccdexposure-level (per-detector) tables.
541 values : `Mapping` [ `str`, `Any` ]
542 Dictionary-like mapping of column/value pairs to add for the
543 observation.
544 allow_update : `bool`, optional
545 If ``True``, allow replacement of values of existing columns.
546 **kwargs : `dict`
547 Additional column/value pairs, overriding ``values``.
549 Returns
550 -------
551 response : `requests.Response`
552 HTTP response from the server, with 200 status for success.
554 Raises
555 ------
556 ValueError
557 Raised if no values are provided in ``values`` or kwargs.
558 requests.RequestException
559 Raised if any kind of connection error occurs.
560 requests.HTTPError
561 Raised if a non-successful status is returned.
562 """
563 # Build a new merged dict to avoid mutating the incoming Mapping.
564 merged_values: dict[str, Any] = {**(dict(values) if values else {}), **kwargs}
565 if not merged_values:
566 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}")
568 data: dict[str, Any]
569 if isinstance(obs_id, tuple):
570 assert len(obs_id) in (
571 2,
572 3,
573 ), f"obs_id tuple must be (day_obs, seq_num) or (day_obs, seq_num, detector); got {obs_id!r}"
575 data = {"table": table, "values": merged_values}
576 url = _urljoin(
577 self.url,
578 "insert",
579 quote(instrument),
580 quote(table),
581 "by_seq_num",
582 # Two segments for exposure-level tables, three when a
583 # detector is supplied for ccdexposure-level tables.
584 *(quote(str(part)) for part in obs_id),
585 )
586 else:
587 data = {"table": table, "obs_id": obs_id, "values": merged_values}
588 url = _urljoin(
589 self.url,
590 "insert",
591 quote(instrument),
592 quote(table),
593 "obs",
594 quote(str(obs_id)),
595 )
596 if allow_update:
597 url += "?u=1"
598 return self._handle_post(url, data)
600 def insert_multiple(
601 self,
602 instrument: str,
603 table: str,
604 obs_dict: dict[int, dict[str, Any]],
605 *,
606 allow_update=False,
607 ) -> requests.Response:
608 """Insert values into a single ConsDB fixed metadata table.
610 Parameters
611 ----------
612 instrument : `str`
613 Name of the instrument (e.g. ``LATISS``).
614 table : `str`
615 Name of the table to insert into.
616 obs_dict : `dict` [ `int`, `dict` [ `str`, `Any` ] ]
617 Dictionary of observation ids, each with a dictionary of
618 column/value pairs to add for each observation.
619 allow_update : `bool`, optional
620 If ``True``, allow replacement of values of existing columns.
622 Returns
623 -------
624 response : `requests.Response`
625 HTTP response from the server, with 200 status for success.
627 Raises
628 ------
629 ValueError
630 Raised if no values are provided in ``obs_dict``.
631 requests.RequestException
632 Raised if any kind of connection error occurs.
633 requests.HTTPError
634 Raised if a non-successful status is returned.
635 """
636 if not obs_dict:
637 raise ValueError(f"No values to insert for {instrument} {table}")
638 data = {"table": table, "obs_dict": obs_dict}
639 url = _urljoin(
640 self.url,
641 "insert",
642 quote(instrument),
643 quote(table),
644 )
645 if allow_update:
646 url += "?u=1"
647 return self._handle_post(url, data)
649 def query(self, query: str) -> Table:
650 """Query the ConsDB database.
652 Parameters
653 ----------
654 query : `str`
655 A SQL query (currently) to the database.
657 Returns
658 -------
659 result : `Table`
660 An ``astropy.Table`` containing the query results.
662 Raises
663 ------
664 requests.RequestException
665 Raised if any kind of connection error occurs.
666 requests.HTTPError
667 Raised if a non-successful status is returned.
669 Notes
670 -----
671 This is a very general query interface because it is expected that
672 a wide variety of types of queries will be needed. If some types prove
673 to be common, syntactic sugar could be added to make them simpler.
674 """
675 url = _urljoin(self.url, "query")
676 data = {"query": query}
677 result = self._handle_post(url, data).json()
679 columns = result.get("columns", [])
680 if not columns: 680 ↛ 682line 680 didn't jump to line 682 because the condition on line 680 was never true
681 # No result columns
682 return Table(rows=[])
684 rows = result.get("data", [])
685 if not rows:
686 # No result rows
687 return Table(names=columns)
689 return Table(rows=rows, names=columns)
691 def schema(
692 self, instrument: str | None = None, table: str | None = None
693 ) -> dict[str, tuple[str, str]] | list[str]:
694 """Retrieve information about ConsDB.
696 If ``instrument`` and ``table`` are given, return the schema of a
697 fixed metadata table in ConsDB.
699 If only ``instrument`` is given, return the names of all tables
700 for that instrument.
702 If no arguments are given, return the names of all instruments.
704 Parameters
705 ----------
706 instrument : `str`, optional
707 Name of the instrument (e.g. ``LATISS``).
708 table : `str`, optional
709 Name of the table to insert into.
711 Returns
712 -------
713 info : `list` [ `str` ] or `dict` [ `str`, `tuple` [ `str`, `str` ] ]
714 A list of instrument strings or table names, or else a dict of
715 columns with values that are tuples containing a data type string
716 and a documentation string.
718 Raises
719 ------
720 ValueError
721 Raised if only ``table`` is given.
722 requests.RequestException
723 Raised if any kind of connection error occurs.
724 requests.HTTPError
725 Raised if a non-successful status is returned.
727 Notes
728 -----
729 Fixed metadata data types may use the full database vocabulary,
730 unlike flexible metadata data types.
731 """
732 if instrument is None:
733 if table is not None: 733 ↛ 734line 733 didn't jump to line 734 because the condition on line 733 was never true
734 raise ValueError("Must specify instrument if table is given")
735 url = _urljoin(self.url, "schema")
736 elif table is None: 736 ↛ 737line 736 didn't jump to line 737 because the condition on line 736 was never true
737 url = _urljoin(self.url, "schema", quote(instrument))
738 else:
739 url = _urljoin(self.url, "schema", quote(instrument), quote(table))
740 result = self._handle_get(url)
741 if instrument is not None and table is not None:
742 return {key: (str(value[0]), str(value[1])) for key, value in result.items()}
743 else:
744 return [str(value) for value in result]
747def getCcdVisitTableForDay(
748 client: ConsDbClient,
749 dayObs: int,
750 visitTableItems: list[str] | None = None,
751 detectors: list[int] | None = None,
752 withZeropoint: bool = False,
753) -> Table:
754 """Get the ccdvisit1_quicklook table for a given dayObs.
756 Parameters
757 ----------
758 client : `ConsDbClient`
759 The ConsDbClient to use.
760 dayObs : `int`
761 The dayObs to query for.
762 visitTableItems : `list` of `str`, optional
763 Additional items from the visit1 table to include.
764 detectors : `list` of `int`, optional
765 If given, only return rows for these detectors.
766 withZeropoint : `bool`, optional
767 If ``True``, only return rows with a non-null zeropoint.
769 Returns
770 -------
771 table : `astropy.table.Table`
772 The resulting table.
773 """
774 # ``cvq.*`` already pulls in every column of ccdvisit1_quicklook, so any
775 # column we also name explicitly from ccdvisit1/visit1 has to be dropped
776 # from the SELECT when the quicklook table itself already defines it.
777 # ConsDB has been denormalising identity columns (visit_id, detector,
778 # seq_num, ...) onto the quicklook tables, and selecting such a column
779 # twice makes the server return duplicate column names, which astropy then
780 # refuses to build a Table from. getWideQuicklookTableForDay does the same
781 # dedup for the visit1 join.
782 cvqCols = set(client.query("SELECT * FROM cdb_LSSTCam.ccdvisit1_quicklook LIMIT 0").colnames)
784 visitItems = ["band", "exp_time", "seq_num", "day_obs", "img_type"]
785 if visitTableItems: 785 ↛ 786line 785 didn't jump to line 786 because the condition on line 785 was never true
786 visitItems += visitTableItems
788 selectClauses = ["cvq.*"]
789 claimed = set(cvqCols) # cvq.* already provides all of these names
790 for col in ("detector", "visit_id"):
791 if col not in claimed:
792 selectClauses.append(f"cv.{col}")
793 claimed.add(col)
794 for col in visitItems:
795 if col not in claimed:
796 selectClauses.append(f"v.{col}")
797 claimed.add(col)
799 query = (
800 f"SELECT {', '.join(selectClauses)} "
801 "FROM cdb_LSSTCam.ccdvisit1_quicklook as cvq, "
802 "cdb_LSSTCam.ccdvisit1 as cv, "
803 "cdb_LSSTCam.visit1 as v "
804 )
805 where = f"WHERE cvq.ccdvisit_id=cv.ccdvisit_id and cv.visit_id=v.visit_id and v.day_obs={dayObs}"
806 if detectors: 806 ↛ 807line 806 didn't jump to line 807 because the condition on line 806 was never true
807 where += f" and cv.detector in ({','.join([str(d) for d in detectors])})"
808 if withZeropoint: 808 ↛ 809line 808 didn't jump to line 809 because the condition on line 808 was never true
809 where += " and cvq.zero_point is not null"
811 table = client.query(query + where)
812 return table
815def columnsEqual(a: Column, b: Column) -> bool:
816 """Check if two columns are equal, taking masks into account.
818 Parameters
819 ----------
820 a : `Column`
821 First column to compare.
822 b : `Column`
823 Second column to compare.
825 Returns
826 -------
827 equal : `bool`
828 True if the columns are equal, False otherwise.
829 """
830 aArr = np.asanyarray(a)
831 bArr = np.asanyarray(b)
832 if aArr.shape != bArr.shape:
833 return False
835 aMask = getattr(a, "mask", None)
836 bMask = getattr(b, "mask", None)
838 if aMask is None and bMask is None:
839 return bool(np.all(aArr == bArr))
841 if aMask is None:
842 aMask = np.zeros(aArr.shape, dtype=bool)
843 if bMask is None:
844 bMask = np.zeros(bArr.shape, dtype=bool)
846 aMaskArr = np.asanyarray(aMask)
847 bMaskArr = np.asanyarray(bMask)
849 if np.any(aMaskArr ^ bMaskArr):
850 return False # one masked where the other isn't
852 present = ~aMaskArr
853 return bool(np.all(aArr[present] == bArr[present]))
856def getWideQuicklookTableForDay(client: ConsDbClient, dayObs: int) -> Table:
857 """Get a wide quicklook table for a given dayObs.
859 Joins all columns from the visit1 table to the visit1_quicklook table. Note
860 that the visit1 table already contains all the columns from the exposure
861 table, and is just keyed by the exposure_id instead of the visit_id.
863 Parameters
864 ----------
865 client : `ConsDbClient`
866 The ConsDbClient to use.
867 dayObs : `int`
868 The dayObs to query for.
870 Returns
871 -------
872 table : `astropy.table.Table`
873 The resulting wide quicklook table.
874 """
875 vqCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1_quicklook LIMIT 0").colnames)
876 vCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1 LIMIT 0").colnames)
878 vOnlyCols = vCols - vqCols # exclude visit_id and all duplicates
880 selectClauses = ["vq.*"] + [f"v.{col}" for col in sorted(vOnlyCols)]
882 query = f"""
883 SELECT {', '.join(selectClauses)}
884 FROM cdb_LSSTCam.visit1_quicklook vq
885 INNER JOIN cdb_LSSTCam.visit1 v USING (visit_id)
886 WHERE vq.day_obs = {dayObs}
887 """
888 return client.query(query)