Coverage for python/lsst/summit/utils/consdbClient.py: 16%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22import logging

23import os

24from collections.abc import Mapping

25from dataclasses import dataclass

26from typing import Any

27from urllib.parse import quote, urlparse

29import numpy as np

30import requests

31from astropy.table import Column, Table

33__all__ = ["ConsDbClient", "FlexibleMetadataInfo", "getCcdVisitTableForDay", "getWideQuicklookTableForDay"]

36logger = logging.getLogger(__name__)

39def _urljoin(*args: str) -> str:

40 """Join parts of a URL with slashes.

42 Does not do any quoting. Mostly to remove a level of list-making.

44 Parameters

45 ----------

46 *args : `str`

47 Each parameter is a URL part.

49 Returns

50 -------

51 url : `str`

52 The joined URL.

53 """

54 return "/".join(args)

57def _check_status(r: requests.Response) -> None:

58 """Check the status of an HTTP response and raise if an error.

60 Adds additional response information to the raise_for_status exception.

62 Parameters

63 ----------

64 r : `requests.Response`

65 The response to check.

67 Raises

68 ------

69 requests.HTTPError

70 Raised if a non-successful status is returned.

71 """

72 try:

73 r.raise_for_status()

74 except requests.HTTPError as e:

75 try:

76 json_data = e.response.json()

77 e.add_note(str(json_data))

78 if "message" in json_data:

79 e.add_note(f"\n\n{json_data['message']}")

80 except requests.JSONDecodeError:

81 pass

82 raise e

85def clean_url(resp: requests.Response, *args, **kwargs) -> requests.Response:

86 """Parse url from response and remove netloc portion.

88 Set new url in response and return response

90 Parameters

91 ----------

92 resp : `requests.Response`

93 The response that could contain a URL with tokens

94 """

95 url = urlparse(resp.url)

96 short_user = f"{url.username[:2]}***" if url.username is not None else ""

97 short_pass = f":{url.password[:2]}***" if url.password is not None else ""

98 netloc = f"{short_user}{short_pass}@{url.hostname}"

99 resp.url = url._replace(netloc=netloc).geturl()

100 return resp

101

102

103@dataclass

104class FlexibleMetadataInfo:

105 """Description of a flexible metadata value.

106

107 Parameters

108 ----------

109 dtype : `str`

110 Data type of the flexible metadata value.

111 One of ``bool``, ``int``, ``float``, or ``str``.

112 doc : `str`

113 Documentation string.

114 unit : `str`, optional

115 Unit of the value.

116 ucd : `str`, optional

117 IVOA Unified Content Descriptor

118 (https://www.ivoa.net/documents/UCD1+/).

119 """

120

121 dtype: str

122 doc: str

123 unit: str | None = None

124 ucd: str | None = None

125

126

127class ConsDbClient:

128 """A client library for accessing the Consolidated Database.

129

130 This library provides a basic interface for using flexible metadata

131 (key/value pairs associated with observation ids from an observation

132 type table), determining the schema of ConsDB tables, querying the

133 ConsDB using a general SQL SELECT statement, and inserting into

134 ConsDB tables.

135

136 Parameters

137 ----------

138 url : `str`, optional

139 Base URL of the Web service, defaults to the value of environment

140 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query

141 service).

142 token : `str`, optional

143 Authentication token for the RSP. The token must begin with "gt-".

144

145 Notes

146 -----

147 This client is a thin layer over the publish/query Web service, which

148 avoids having a dependency on database drivers.

149

150 It enforces the return of query results as Astropy Tables.

151 """

152

153 def __init__(self, url: str | None = None, token: str | None = None):

154 self.session = requests.Session()

155 self.session.hooks["response"].append(clean_url)

156

157 if token is not None:

158 if not token.startswith("gt-"):

159 raise ValueError("token must start with `gt-`.")

160

161 self.session.headers.update({"Authorization": f"Bearer {token}"})

162

163 if url is None:

164 self.url = os.environ["LSST_CONSDB_PQ_URL"]

165 else:

166 self.url = url

167 self.url = self.url.rstrip("/")

168

169 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any:

170 """Submit GET requests to the server.

171

172 Parameters

173 ----------

174 url : `str`

175 URL to GET.

176 query : `dict` [`str`, `str` | `list` [`str`]], optional

177 Query parameters to attach to the URL.

178

179 Raises

180 ------

181 requests.RequestException

182 Raised if any kind of connection error occurs.

183 requests.HTTPError

184 Raised if a non-successful status is returned.

185 requests.JSONDecodeError

186 Raised if the result does not decode as JSON.

187

188 Returns

189 -------

190 result : `Any`

191 Result of decoding the Web service result content as JSON.

192 """

193 logger.debug(f"GET {url}")

194 response = self.session.get(url, params=query)

195 _check_status(response)

196 return response.json()

197

198 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response:

199 """Submit POST requests to the server.

200

201 Parameters

202 ----------

203 url : `str`

204 URL to POST.

205 data : `dict` [`str`, `Any`]

206 Key/value pairs of data to POST.

207

208 Raises

209 ------

210 requests.RequestException

211 Raised if any kind of connection error occurs.

212 requests.HTTPError

213 Raised if a non-successful status is returned.

214

215 Returns

216 -------

217 result : `requests.Response`

218 The raw Web service result object.

219 """

220 logger.debug(f"POST {url}: {data}")

221 response = self.session.post(url, json=data)

222 _check_status(response)

223 return response

224

225 @staticmethod

226 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str:

227 """Compute the name of a flexible metadata table.

228

229 Each instrument and observation type made with that instrument can

230 have a flexible metadata table. This function is useful when

231 issuing SQL queries, and it avoids a round-trip to the server.

232

233 Parameters

234 ----------

235 instrument : `str`

236 Name of the instrument (e.g. ``LATISS``).

237 obs_type : `str`

238 Name of the observation type (e.g. ``Exposure``).

239

240 Returns

241 -------

242 table_name : `str`

243 Name of the appropriate flexible metadata table.

244 """

245 return f"cdb_{instrument}.{obs_type}_flexdata"

246

247 @staticmethod

248 def compute_fixed_metadata_namespace(instrument: str) -> str:

249 """Compute the namespace for a fixed metadata table.

250

251 Each instrument has its own namespace in the ConsDB.

252 This function is useful when issuing SQL queries, and it avoids a

253 round-trip to the server.

254

255 Parameters

256 ----------

257 instrument : `str`

258 Name of the instrument (e.g. ``LATISS``).

259

260 Returns

261 -------

262 namespace_name : `str`

263 Name of the appropriate namespace

264 """

265 return f"cdb_{instrument}"

266

267 def add_flexible_metadata_key(

268 self,

269 instrument: str,

270 obs_type: str,

271 key: str,

272 dtype: str,

273 doc: str,

274 unit: str | None = None,

275 ucd: str | None = None,

276 ) -> requests.Response:

277 """Add a key to a flexible metadata table.

278

279 Parameters

280 ----------

281 instrument : `str`

282 Name of the instrument (e.g. ``LATISS``).

283 obs_type : `str`

284 Name of the observation type (e.g. ``Exposure``).

285 key : `str`

286 Name of the key to be added (must not already exist).

287 dtype : `str`

288 One of ``bool``, ``int``, ``float``, or ``str``.

289 doc : `str`

290 Documentation string for the key.

291 unit : `str`, optional

292 Unit for the value. Should be from the IVOA

293 (https://www.ivoa.net/documents/VOUnits/) or astropy.

294 ucd : `str`, optional

295 IVOA Unified Content Descriptor

296 (https://www.ivoa.net/documents/UCD1+/).

297

298 Returns

299 -------

300 response : `requests.Response`

301 HTTP response from the server, with 200 status for success.

302

303 Raises

304 ------

305 requests.RequestException

306 Raised if any kind of connection error occurs.

307 requests.HTTPError

308 Raised if a non-successful status is returned.

309 """

310 data = {"key": key, "dtype": dtype, "doc": doc}

311 if unit is not None:

312 data["unit"] = unit

313 if ucd is not None:

314 data["ucd"] = ucd

315 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey")

316 return self._handle_post(url, data)

317

318 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]:

319 """Retrieve the valid keys for a flexible metadata table.

320

321 Parameters

322 ----------

323 instrument : `str`

324 Name of the instrument (e.g. ``LATISS``).

325 obs_type : `str`

326 Name of the observation type (e.g. ``Exposure``).

327

328 Returns

329 -------

330 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ]

331 Dict of keys and information values.

332

333 Raises

334 ------

335 requests.RequestException

336 Raised if any kind of connection error occurs.

337 requests.HTTPError

338 Raised if a non-successful status is returned.

339 """

340 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema")

341 result = self._handle_get(url)

342 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()}

343

344 def get_flexible_metadata(

345 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None

346 ) -> dict[str, Any]:

347 """Get the flexible metadata for an observation.

348

349 Parameters

350 ----------

351 instrument : `str`

352 Name of the instrument (e.g. ``LATISS``).

353 obs_type : `str`

354 Name of the observation type (e.g. ``Exposure``).

355 obs_id : `int`

356 Unique observation id.

357 keys : `list` [ `str` ], optional

358 List of keys to be retrieved; all if not specified.

359

360 Returns

361 -------

362 result_dict : `dict` [ `str`, `Any` ]

363 Dictionary of key/value pairs for the observation.

364

365 Raises

366 ------

367 requests.RequestException

368 Raised if any kind of connection error occurs.

369 requests.HTTPError

370 Raised if a non-successful status is returned.

371 """

372 url = _urljoin(

373 self.url,

374 "flex",

375 quote(instrument),

376 quote(obs_type),

377 "obs",

378 quote(str(obs_id)),

379 )

380 return self._handle_get(url, {"k": keys} if keys else None)

381

382 def get_all_metadata(

383 self, instrument: str, obs_type: str, obs_id: int, flex: bool = False

384 ) -> dict[str, Any]:

385 """Get all metadata for an observation.

386

387 Parameters

388 ----------

389 instrument : `str`

390 Name of the instrument (e.g. ``LATISS``).

391 obs_type : `str`

392 Name of the observation type (e.g. ``Exposure``).

393 obs_id : `int`

394 Unique observation id.

395 flex : `bool`

396 Include flexible metadata.

397

398 Returns

399 -------

400 result_dict : `dict` [ `str`, `Any` ]

401 Dictionary of key/value pairs for the observation.

402

403 Raises

404 ------

405 requests.RequestException

406 Raised if any kind of connection error occurs.

407 requests.HTTPError

408 Raised if a non-successful status is returned.

409 """

410 url = _urljoin(

411 self.url,

412 "query",

413 quote(instrument),

414 quote(obs_type),

415 "obs",

416 quote(str(obs_id)),

417 )

418 return self._handle_get(url, {"flex": "1"} if flex else None)

419

420 def insert_flexible_metadata(

421 self,

422 instrument: str,

423 obs_type: str,

424 obs_id: int,

425 values: dict[str, Any] | None = None,

426 *,

427 allow_update: bool = False,

428 **kwargs,

429 ) -> requests.Response:

430 """Set flexible metadata values for an observation.

431

432 Parameters

433 ----------

434 instrument : `str`

435 Name of the instrument (e.g. ``LATISS``).

436 obs_type : `str`

437 Name of the observation type (e.g. ``Exposure``).

438 obs_id : `int`

439 Unique observation id.

440 values : `dict` [ `str`, `Any` ], optional

441 Dictionary of key/value pairs to add for the observation.

442 allow_update : `bool`, optional

443 If ``True``, allow replacement of values of existing keys.

444 **kwargs : `dict`

445 Additional key/value pairs, overriding ``values``.

446

447 Returns

448 -------

449 response : `requests.Response`

450 HTTP response from the server, with 200 status for success.

451

452 Raises

453 ------

454 ValueError

455 Raised if no values are provided in ``values`` or kwargs.

456 requests.RequestException

457 Raised if any kind of connection error occurs.

458 requests.HTTPError

459 Raised if a non-successful status is returned.

460 """

461 if values:

462 values.update(kwargs)

463 else:

464 values = kwargs

465 if not values:

466 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}")

467 data = {"values": values}

468 url = _urljoin(

469 self.url,

470 "flex",

471 quote(instrument),

472 quote(obs_type),

473 "obs",

474 quote(str(obs_id)),

475 )

476 if allow_update:

477 url += "?u=1"

478 return self._handle_post(url, data)

479

480 def insert(

481 self,

482 instrument: str,

483 table: str,

484 obs_id: tuple[int, int] | tuple[int, int, int] | int,

485 values: Mapping[str, Any],

486 *,

487 allow_update: bool = False,

488 **kwargs,

489 ) -> requests.Response:

490 """Insert values into a single ConsDB fixed metadata table.

491

492 Parameters

493 ----------

494 instrument : `str`

495 Name of the instrument (e.g. ``LATISS``).

496 table : `str`

497 Name of the table to insert into.

498 obs_id : `tuple`[`int`, `int`] | `tuple`[`int`, `int`, `int`] | `int`

499 How to address the row to insert:

500

501 - `int`: the natural observation id (e.g. ``exposure_id``,

502 ``ccdexposure_id``), targeting the ``.../obs/{obs_id}``

503 endpoint.

504 - 2-tuple ``(day_obs, seq_num)``: targeting the

505 ``.../by_seq_num/{day_obs}/{seq_num}`` endpoint, for

506 exposure-level tables.

507 - 3-tuple ``(day_obs, seq_num, detector)``: targeting the

508 ``.../by_seq_num/{day_obs}/{seq_num}/{detector}`` endpoint,

509 for ccdexposure-level (per-detector) tables.

510 values : `Mapping` [ `str`, `Any` ]

511 Dictionary-like mapping of column/value pairs to add for the

512 observation.

513 allow_update : `bool`, optional

514 If ``True``, allow replacement of values of existing columns.

515 **kwargs : `dict`

516 Additional column/value pairs, overriding ``values``.

517

518 Returns

519 -------

520 response : `requests.Response`

521 HTTP response from the server, with 200 status for success.

522

523 Raises

524 ------

525 ValueError

526 Raised if no values are provided in ``values`` or kwargs.

527 requests.RequestException

528 Raised if any kind of connection error occurs.

529 requests.HTTPError

530 Raised if a non-successful status is returned.

531 """

532 # Build a new merged dict to avoid mutating the incoming Mapping.

533 merged_values: dict[str, Any] = {**(dict(values) if values else {}), **kwargs}

534 if not merged_values:

535 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}")

536

537 data: dict[str, Any]

538 if isinstance(obs_id, tuple):

539 assert len(obs_id) in (

540 2,

541 3,

542 ), f"obs_id tuple must be (day_obs, seq_num) or (day_obs, seq_num, detector); got {obs_id!r}"

543

544 data = {"table": table, "values": merged_values}

545 url = _urljoin(

546 self.url,

547 "insert",

548 quote(instrument),

549 quote(table),

550 "by_seq_num",

551 # Two segments for exposure-level tables, three when a

552 # detector is supplied for ccdexposure-level tables.

553 *(quote(str(part)) for part in obs_id),

554 )

555 else:

556 data = {"table": table, "obs_id": obs_id, "values": merged_values}

557 url = _urljoin(

558 self.url,

559 "insert",

560 quote(instrument),

561 quote(table),

562 "obs",

563 quote(str(obs_id)),

564 )

565 if allow_update:

566 url += "?u=1"

567 return self._handle_post(url, data)

568

569 def insert_multiple(

570 self,

571 instrument: str,

572 table: str,

573 obs_dict: dict[int, dict[str, Any]],

574 *,

575 allow_update=False,

576 ) -> requests.Response:

577 """Insert values into a single ConsDB fixed metadata table.

578

579 Parameters

580 ----------

581 instrument : `str`

582 Name of the instrument (e.g. ``LATISS``).

583 table : `str`

584 Name of the table to insert into.

585 obs_dict : `dict` [ `int`, `dict` [ `str`, `Any` ] ]

586 Dictionary of observation ids, each with a dictionary of

587 column/value pairs to add for each observation.

588 allow_update : `bool`, optional

589 If ``True``, allow replacement of values of existing columns.

590

591 Returns

592 -------

593 response : `requests.Response`

594 HTTP response from the server, with 200 status for success.

595

596 Raises

597 ------

598 ValueError

599 Raised if no values are provided in ``obs_dict``.

600 requests.RequestException

601 Raised if any kind of connection error occurs.

602 requests.HTTPError

603 Raised if a non-successful status is returned.

604 """

605 if not obs_dict:

606 raise ValueError(f"No values to insert for {instrument} {table}")

607 data = {"table": table, "obs_dict": obs_dict}

608 url = _urljoin(

609 self.url,

610 "insert",

611 quote(instrument),

612 quote(table),

613 )

614 if allow_update:

615 url += "?u=1"

616 return self._handle_post(url, data)

617

618 def query(self, query: str) -> Table:

619 """Query the ConsDB database.

620

621 Parameters

622 ----------

623 query : `str`

624 A SQL query (currently) to the database.

625

626 Returns

627 -------

628 result : `Table`

629 An ``astropy.Table`` containing the query results.

630

631 Raises

632 ------

633 requests.RequestException

634 Raised if any kind of connection error occurs.

635 requests.HTTPError

636 Raised if a non-successful status is returned.

637

638 Notes

639 -----

640 This is a very general query interface because it is expected that

641 a wide variety of types of queries will be needed. If some types prove

642 to be common, syntactic sugar could be added to make them simpler.

643 """

644 url = _urljoin(self.url, "query")

645 data = {"query": query}

646 result = self._handle_post(url, data).json()

647

648 columns = result.get("columns", [])

649 if not columns:

650 # No result columns

651 return Table(rows=[])

652

653 rows = result.get("data", [])

654 if not rows:

655 # No result rows

656 return Table(names=columns)

657

658 return Table(rows=rows, names=columns)

659

660 def schema(

661 self, instrument: str | None = None, table: str | None = None

662 ) -> dict[str, tuple[str, str]] | list[str]:

663 """Retrieve information about ConsDB.

664

665 If ``instrument`` and ``table`` are given, return the schema of a

666 fixed metadata table in ConsDB.

667

668 If only ``instrument`` is given, return the names of all tables

669 for that instrument.

670

671 If no arguments are given, return the names of all instruments.

672

673 Parameters

674 ----------

675 instrument : `str`, optional

676 Name of the instrument (e.g. ``LATISS``).

677 table : `str`, optional

678 Name of the table to insert into.

679

680 Returns

681 -------

682 info : `list` [ `str` ] or `dict` [ `str`, `tuple` [ `str`, `str` ] ]

683 A list of instrument strings or table names, or else a dict of

684 columns with values that are tuples containing a data type string

685 and a documentation string.

686

687 Raises

688 ------

689 ValueError

690 Raised if only ``table`` is given.

691 requests.RequestException

692 Raised if any kind of connection error occurs.

693 requests.HTTPError

694 Raised if a non-successful status is returned.

695

696 Notes

697 -----

698 Fixed metadata data types may use the full database vocabulary,

699 unlike flexible metadata data types.

700 """

701 if instrument is None:

702 if table is not None:

703 raise ValueError("Must specify instrument if table is given")

704 url = _urljoin(self.url, "schema")

705 elif table is None:

706 url = _urljoin(self.url, "schema", quote(instrument))

707 else:

708 url = _urljoin(self.url, "schema", quote(instrument), quote(table))

709 result = self._handle_get(url)

710 if instrument is not None and table is not None:

711 return {key: (str(value[0]), str(value[1])) for key, value in result.items()}

712 else:

713 return [str(value) for value in result]

714

715

716def getCcdVisitTableForDay(

717 client: ConsDbClient,

718 dayObs: int,

719 visitTableItems: list[str] | None = None,

720 detectors: list[int] | None = None,

721 withZeropoint: bool = False,

722) -> Table:

723 """Get the ccdvisit1_quicklook table for a given dayObs.

724

725 Parameters

726 ----------

727 client : `ConsDbClient`

728 The ConsDbClient to use.

729 dayObs : `int`

730 The dayObs to query for.

731 visitTableItems : `list` of `str`, optional

732 Additional items from the visit1 table to include.

733 detectors : `list` of `int`, optional

734 If given, only return rows for these detectors.

735 withZeropoint : `bool`, optional

736 If ``True``, only return rows with a non-null zeropoint.

737

738 Returns

739 -------

740 table : `astropy.table.Table`

741 The resulting table.

742 """

743 # ``cvq.*`` already pulls in every column of ccdvisit1_quicklook, so any

744 # column we also name explicitly from ccdvisit1/visit1 has to be dropped

745 # from the SELECT when the quicklook table itself already defines it.

746 # ConsDB has been denormalising identity columns (visit_id, detector,

747 # seq_num, ...) onto the quicklook tables, and selecting such a column

748 # twice makes the server return duplicate column names, which astropy then

749 # refuses to build a Table from. getWideQuicklookTableForDay does the same

750 # dedup for the visit1 join.

751 cvqCols = set(client.query("SELECT * FROM cdb_LSSTCam.ccdvisit1_quicklook LIMIT 0").colnames)

752

753 visitItems = ["band", "exp_time", "seq_num", "day_obs", "img_type"]

754 if visitTableItems:

755 visitItems += visitTableItems

756

757 selectClauses = ["cvq.*"]

758 claimed = set(cvqCols) # cvq.* already provides all of these names

759 for col in ("detector", "visit_id"):

760 if col not in claimed:

761 selectClauses.append(f"cv.{col}")

762 claimed.add(col)

763 for col in visitItems:

764 if col not in claimed:

765 selectClauses.append(f"v.{col}")

766 claimed.add(col)

767

768 query = (

769 f"SELECT {', '.join(selectClauses)} "

770 "FROM cdb_LSSTCam.ccdvisit1_quicklook as cvq, "

771 "cdb_LSSTCam.ccdvisit1 as cv, "

772 "cdb_LSSTCam.visit1 as v "

773 )

774 where = f"WHERE cvq.ccdvisit_id=cv.ccdvisit_id and cv.visit_id=v.visit_id and v.day_obs={dayObs}"

775 if detectors:

776 where += f" and cv.detector in ({','.join([str(d) for d in detectors])})"

777 if withZeropoint:

778 where += " and cvq.zero_point is not null"

779

780 table = client.query(query + where)

781 return table

782

783

784def columnsEqual(a: Column, b: Column) -> bool:

785 """Check if two columns are equal, taking masks into account.

786

787 Parameters

788 ----------

789 a : `Column`

790 First column to compare.

791 b : `Column`

792 Second column to compare.

793

794 Returns

795 -------

796 equal : `bool`

797 True if the columns are equal, False otherwise.

798 """

799 aArr = np.asanyarray(a)

800 bArr = np.asanyarray(b)

801 if aArr.shape != bArr.shape:

802 return False

803

804 aMask = getattr(a, "mask", None)

805 bMask = getattr(b, "mask", None)

806

807 if aMask is None and bMask is None:

808 return bool(np.all(aArr == bArr))

809

810 if aMask is None:

811 aMask = np.zeros(aArr.shape, dtype=bool)

812 if bMask is None:

813 bMask = np.zeros(bArr.shape, dtype=bool)

814

815 aMaskArr = np.asanyarray(aMask)

816 bMaskArr = np.asanyarray(bMask)

817

818 if np.any(aMaskArr ^ bMaskArr):

819 return False # one masked where the other isn't

820

821 present = ~aMaskArr

822 return bool(np.all(aArr[present] == bArr[present]))

823

824

825def getWideQuicklookTableForDay(client: ConsDbClient, dayObs: int) -> Table:

826 """Get a wide quicklook table for a given dayObs.

827

828 Joins all columns from the visit1 table to the visit1_quicklook table. Note

829 that the visit1 table already contains all the columns from the exposure

830 table, and is just keyed by the exposure_id instead of the visit_id.

831

832 Parameters

833 ----------

834 client : `ConsDbClient`

835 The ConsDbClient to use.

836 dayObs : `int`

837 The dayObs to query for.

838

839 Returns

840 -------

841 table : `astropy.table.Table`

842 The resulting wide quicklook table.

843 """

844 vqCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1_quicklook LIMIT 0").colnames)

845 vCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1 LIMIT 0").colnames)

846

847 vOnlyCols = vCols - vqCols # exclude visit_id and all duplicates

848

849 selectClauses = ["vq.*"] + [f"v.{col}" for col in sorted(vOnlyCols)]

850

851 query = f"""

852 SELECT {', '.join(selectClauses)}

853 FROM cdb_LSSTCam.visit1_quicklook vq

854 INNER JOIN cdb_LSSTCam.visit1 v USING (visit_id)

855 WHERE vq.day_obs = {dayObs}

856 """

857 return client.query(query)