Coverage for tests/test_butlerUtils.py: 21%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22import copy

23import datetime

24import os

25import random

26import unittest

27from typing import Any, Iterable

29import lsst.daf.butler as dafButler

30import lsst.utils.tests

31from lsst.daf.butler import DatasetRef, DimensionRecord

32from lsst.daf.butler.direct_butler import DirectButler

33from lsst.resources import ResourcePath

34from lsst.summit.utils.butlerUtils import removeDataProduct # noqa: F401

35from lsst.summit.utils.butlerUtils import (

36 RECENT_DAY,

37 _assureDict,

38 _dayobs_present,

39 _expid_present,

40 _get_dayobs_key,

41 _get_expid_key,

42 _get_seqnum_key,

43 _seqnum_present,

44 fillDataId,

45 getDatasetRefForDataId,

46 getDayObs,

47 getDayObsSeqNumFromExposureId,

48 getDaysWithData,

49 getExpId,

50 getExpIdFromDayObsSeqNum,

51 getExpRecord,

52 getExpRecordFromDataId,

53 getLatissDefaultCollections,

54 getLatissOnSkyDataIds,

55 getMostRecentDataId,

56 getMostRecentDayObs,

57 getSeqNum,

58 getSeqNumsForDayObs,

59 getSite,

60 makeDefaultLatissButler,

61 sanitizeDayObs,

62 sortRecordsByDayObsThenSeqNum,

63 updateDataId,

64 updateDataIdOrDataCord,

65)

68class ButlerUtilsTestCase(lsst.utils.tests.TestCase):

69 """A test case for testing sky position offsets for exposures."""

71 def setUp(self) -> None:

72 # this also functions as test_makeDefaultLatissButler(), but we may as

73 # well catch the butler once it's made so it can be reused if needed,

74 # given how hard it is to made it robustly

76 # butler stuff

77 try:

78 if getSite() == "jenkins": 78 ↛ 80line 78 didn't jump to line 80 because the condition on line 78 was always true

79 raise unittest.SkipTest("Skip running butler-driven tests in Jenkins.")

80 butler = makeDefaultLatissButler()

81 except FileNotFoundError:

82 raise unittest.SkipTest("Skipping tests that require the LATISS butler repo.")

83 # fillDataId() and getLatissOnSkyDataIds() use DirectButler-only APIs,

84 # so pin the concrete type rather than fail deep inside them.

85 assert isinstance(butler, DirectButler)

86 self.butler = butler

88 # dict-like dataIds

89 self.rawDataId = getMostRecentDataId(self.butler)

90 self.fullId = fillDataId(self.butler, self.rawDataId)

91 self.assertIn("exposure", self.fullId)

92 self.assertIn("day_obs", self.fullId)

93 self.assertIn("seq_num", self.fullId)

94 self.expIdOnly = {"exposure": self.fullId["exposure"], "detector": 0}

95 self.dayObsSeqNumIdOnly = {

96 "day_obs": getDayObs(self.fullId),

97 "seq_num": getSeqNum(self.fullId),

98 "detector": 0,

99 }

100

101 # expRecords

102 self.expRecordNoDetector = getExpRecordFromDataId(self.butler, self.rawDataId)

103 self.assertIsInstance(self.expRecordNoDetector, dafButler.DimensionRecord)

104 self.assertFalse(hasattr(self.expRecordNoDetector, "detector"))

105 self.assertFalse("detector" in self.expRecordNoDetector.dataId)

106 # just a crosscheck on the above to make sure other things are correct

107 self.assertTrue(hasattr(self.expRecordNoDetector, "instrument"))

108

109 # data coordinates

110 # popping here because butler.registry.expandDataId cannot have

111 # day_obs or seq_num present right now

112 rawDataIdNoDayObSeqNum = _assureDict(self.rawDataId)

113 if dayObsKey := _get_dayobs_key(rawDataIdNoDayObSeqNum):

114 rawDataIdNoDayObSeqNum.pop(dayObsKey)

115 if seqNumKey := _get_seqnum_key(rawDataIdNoDayObSeqNum):

116 rawDataIdNoDayObSeqNum.pop(seqNumKey)

117 self.rawDataIdNoDayObSeqNum = rawDataIdNoDayObSeqNum

118 self.dataCoordMinimal = self.butler.registry.expandDataId(self.rawDataIdNoDayObSeqNum, detector=0)

119 self.assertIsInstance(self.dataCoordMinimal, dafButler.DataCoordinate)

120

121 def test_getLatissDefaultCollections(self) -> None:

122 defaultCollections = getLatissDefaultCollections()

123 self.assertTrue(defaultCollections is not None)

124 self.assertTrue(defaultCollections != [])

125 self.assertTrue(len(defaultCollections) >= 1)

126

127 def test_RECENT_DAY(self) -> None:

128 todayInt = int(datetime.date.today().strftime("%Y%m%d"))

129 self.assertTrue(RECENT_DAY <= todayInt) # in the past

130 self.assertTrue(RECENT_DAY >= 20200101) # not too far in the past

131

132 # check that the value of RECENT_DAY is before the end of the data.

133 daysWithData = getDaysWithData(self.butler)

134 self.assertLessEqual(RECENT_DAY, max(daysWithData))

135

136 # no test here, but print a warning if it hasn't been updated recently

137 recentDay_datetime = datetime.datetime.strptime(str(RECENT_DAY), "%Y%m%d")

138 now = datetime.datetime.today()

139 timeSinceUpdate = now - recentDay_datetime

140 if timeSinceUpdate.days > 100: # TODO:

141 print(

142 f"RECENT_DAY is now {timeSinceUpdate.days} days in the past. "

143 "You might want to consider updating this to speed up butler queries."

144 )

145

146 def test_sanitizeDayObs(self) -> None:

147 dayObs: str | int = "2020-01-02"

148 self.assertEqual(sanitizeDayObs(dayObs), 20200102)

149 dayObs = 20210201

150 self.assertEqual(sanitizeDayObs(dayObs), dayObs)

151

152 with self.assertRaises(ValueError):

153 sanitizeDayObs(1.234) # type: ignore[arg-type]

154 sanitizeDayObs("Febuary 29th, 1970")

155

156 def test_getMostRecentDayObs(self) -> None:

157 # just a basic sanity check here as we can't know the value,

158 # but at least check something is returned, and is plausible

159 recentDay = getMostRecentDayObs(self.butler)

160 self.assertIsInstance(recentDay, int)

161 self.assertTrue(recentDay >= RECENT_DAY)

162 # some test data might be set a millennium in the future, i.e.

163 # the year wouldd be 2XXX+1000, so set to y4k just in case

164 self.assertTrue(recentDay < 40000000)

165

166 def test_getSeqNumsForDayObs(self) -> None:

167 emptyDay = 19990101

168 seqnums = getSeqNumsForDayObs(self.butler, emptyDay)

169 self.assertIsInstance(seqnums, Iterable)

170 self.assertEqual(len(list(seqnums)), 0)

171

172 recentDay = getMostRecentDayObs(self.butler)

173 seqnums = getSeqNumsForDayObs(self.butler, recentDay)

174 self.assertIsInstance(seqnums, Iterable)

175 self.assertTrue(len(list(seqnums)) >= 1)

176

177 def test_getMostRecentDataId(self) -> None:

178 # we can't know the values, but it should always return something

179 # and the dict and int forms should always have certain keys and agree

180 dataId = getMostRecentDataId(self.butler)

181 self.assertIsInstance(dataId, dict)

182 self.assertIn("day_obs", dataId)

183 self.assertIn("seq_num", dataId)

184 self.assertTrue("exposure" in dataId or "exposure.id" in dataId)

185

186 def test_getDatasetRefForDataId(self) -> None:

187 dRef = getDatasetRefForDataId(self.butler, "raw", self.rawDataId)

188 self.assertIsInstance(dRef, DatasetRef)

189

190 dRef = getDatasetRefForDataId(self.butler, "raw", self.rawDataIdNoDayObSeqNum)

191 self.assertIsInstance(dRef, DatasetRef)

192 dRef = getDatasetRefForDataId(self.butler, "raw", self.dataCoordMinimal)

193 self.assertIsInstance(dRef, DatasetRef)

194

195 def test__dayobs_present(self) -> None:

196 goods = [{"day_obs": 123}, {"exposure.day_obs": 234}, {"day_obs": 345, "otherkey": -1}]

197 bads = [{"different_key": 123}]

198 for good in goods:

199 self.assertTrue(_dayobs_present(good))

200 for bad in bads:

201 self.assertFalse(_dayobs_present(bad))

202

203 def test__seqnum_present(self) -> None:

204 goods = [{"seq_num": 123}, {"exposure.seq_num": 234}, {"seq_num": 345, "otherkey": -1}]

205 bads = [{"different_key": 123}]

206 for good in goods:

207 self.assertTrue(_seqnum_present(good))

208 for bad in bads:

209 self.assertFalse(_seqnum_present(bad))

210

211 def test__expid_present(self) -> None:

212 goods = [{"exposure": 123}, {"exposure.id": 234}, {"exposure.id": 345, "otherkey": -1}]

213 bads = [{"different_key": 123}]

214 for good in goods:

215 self.assertTrue(_expid_present(good))

216 for bad in bads:

217 self.assertFalse(_expid_present(bad))

218

219 def test_getDayObs(self) -> None:

220 dayVal = 98765

221 goods = [{"day_obs": dayVal}, {"exposure.day_obs": dayVal}, {"day_obs": dayVal, "otherkey": -1}]

222 bads = [{"different_key": 123}]

223 for good in goods:

224 self.assertTrue(getDayObs(good) == dayVal)

225 for bad in bads:

226 self.assertTrue(getDayObs(bad) is None)

227

228 def test_getSeqNum(self) -> None:

229 seqVal = 12345

230 goods = [{"seq_num": seqVal}, {"exposure.seq_num": seqVal}, {"seq_num": seqVal, "otherkey": -1}]

231 bads = [{"different_key": 123}]

232 for good in goods:

233 self.assertTrue(getSeqNum(good) == seqVal)

234 for bad in bads:

235 self.assertTrue(getSeqNum(bad) is None)

236

237 def test_getExpId(self) -> None:

238 expIdVal = 12345

239 goods = [{"exposure": expIdVal}, {"exposure.id": expIdVal}, {"exposure": expIdVal, "otherkey": -1}]

240 bads = [{"different_key": 123}]

241 for good in goods:

242 self.assertTrue(getExpId(good) == expIdVal)

243 for bad in bads:

244 self.assertTrue(getExpId(bad) is None)

245

246 def test_datasetExists(self) -> None:

247 self.assertTrue(self.butler.exists("raw", self.rawDataId))

248 self.assertTrue(self.butler.exists("raw", self.expIdOnly))

249 self.assertTrue(self.butler.exists("raw", self.dayObsSeqNumIdOnly))

250 return

251

252 def test_sortRecordsByDayObsThenSeqNum(self) -> None:

253 where = "exposure.day_obs=dayObs"

254 recordResults = self.butler.registry.queryDimensionRecords(

255 "exposure", where=where, bind={"dayObs": RECENT_DAY}

256 )

257 expRecords = list(recordResults)

258 self.assertGreaterEqual(len(expRecords), 1) # just ensure we're not doing a no-op test

259 random.shuffle(expRecords) # they are often already in order, so make sure they're not

260 sortedIds = sortRecordsByDayObsThenSeqNum(expRecords)

261 for i, _id in enumerate(sortedIds[:-1]):

262 self.assertTrue(_id.seq_num < sortedIds[i + 1].seq_num)

263

264 # Check that ambiguous sorts raise as expected

265 with self.assertRaises(ValueError):

266 recordResults = self.butler.registry.queryDimensionRecords(

267 "exposure", where=where, bind={"dayObs": RECENT_DAY}

268 )

269 expRecords = list(recordResults)

270 self.assertGreaterEqual(len(expRecords), 1) # just ensure we're not doing a no-op test

271 expRecords.append(expRecords[0]) # add a duplicate

272 sortedIds = sortRecordsByDayObsThenSeqNum(expRecords)

273 return

274

275 def test_getDaysWithData(self) -> None:

276 days = getDaysWithData(self.butler)

277 self.assertTrue(len(days) >= 0)

278 self.assertIsInstance(days[0], int)

279 return

280

281 def test_getExpIdFromDayObsSeqNum(self) -> None:

282 expId = getExpIdFromDayObsSeqNum(self.butler, self.dayObsSeqNumIdOnly)

283 self.assertTrue(_expid_present(expId))

284 return

285

286 def test_updateDataIdOrDataCord(self) -> None:

287 updateVals = {"testKey": "testValue"}

288

289 ids: list[dafButler.DataId | DimensionRecord] = [

290 self.rawDataId,

291 self.expRecordNoDetector,

292 self.dataCoordMinimal,

293 ]

294 for originalId in ids:

295 testId = updateDataIdOrDataCord(originalId, **updateVals)

296 for k, v in updateVals.items():

297 self.assertTrue(testId[k] == v)

298 return

299

300 def test_fillDataId(self) -> None:

301 self.assertFalse(_dayobs_present(self.expIdOnly))

302 self.assertFalse(_seqnum_present(self.expIdOnly))

303

304 fullId = fillDataId(self.butler, self.expIdOnly)

305 self.assertTrue(_dayobs_present(fullId))

306 self.assertTrue(_seqnum_present(fullId))

307

308 ids: list[dafButler.DataId | DimensionRecord] = [

309 self.rawDataId,

310 self.expRecordNoDetector,

311 self.dataCoordMinimal,

312 ]

313 for dataId in ids:

314 fullId = fillDataId(self.butler, dataId)

315 self.assertTrue(_dayobs_present(fullId))

316 self.assertTrue(_seqnum_present(fullId))

317 self.assertTrue(_expid_present(fullId))

318 return

319

320 def test_getExpRecordFromDataId(self) -> None:

321 record = getExpRecordFromDataId(self.butler, self.rawDataId)

322 self.assertIsInstance(record, dafButler.DimensionRecord)

323 return

324

325 def test_getDayObsSeqNumFromExposureId(self) -> None:

326 dayObsSeqNum = getDayObsSeqNumFromExposureId(self.butler, self.expIdOnly)

327 self.assertTrue(_dayobs_present(dayObsSeqNum))

328 self.assertTrue(_seqnum_present(dayObsSeqNum))

329 return

330

331 def test_removeDataProduct(self) -> None:

332 # Can't think of an easy or safe test for this

333 return

334

335 def test_getLatissOnSkyDataIds(self) -> None:

336 # This is very slow, consider removing as it's the least import of all

337 # the util functions. However, restricting it to only the most recent

338 # day does help a lot, so probably OK like that, and should speed up

339 # with middleware improvements in the future, and we should ensure

340 # that they don't break this, so inclined to leave for now

341 dayToUse = getDaysWithData(self.butler)[-1]

342 # the most recent day with data might only be biases or flats so make

343 # sure to override the default of skipping biases, darks & flats

344 skipTypes = ()

345 ids = getLatissOnSkyDataIds(self.butler, skipTypes=skipTypes, startDate=dayToUse, endDate=dayToUse)

346 self.assertTrue(len(ids) > 0)

347 self.assertTrue(ids[0] is not None)

348

349 ids = getLatissOnSkyDataIds(

350 self.butler, skipTypes=skipTypes, startDate=dayToUse, endDate=dayToUse, full=True

351 )

352 self.assertTrue(len(ids) > 0)

353 self.assertTrue(ids[0] is not None)

354 testId = ids[0]

355 self.assertTrue(_dayobs_present(testId))

356 self.assertTrue(_seqnum_present(testId))

357 self.assertTrue(_expid_present(testId))

358 return

359

360 def test__assureDict(self) -> None:

361 items: list[dafButler.DataId | DimensionRecord] = [

362 self.rawDataId,

363 self.fullId,

364 self.expIdOnly,

365 self.expRecordNoDetector,

366 self.dataCoordMinimal,

367 self.rawDataIdNoDayObSeqNum,

368 ]

369 for item in items:

370 testId = _assureDict(item)

371 self.assertIsInstance(testId, dict)

372 return

373

374 def test__get_dayobs_key(self) -> None:

375 dataId = {"a_random_key": 321, "exposure.day_obs": 20200312, "z_random_key": "abc"}

376 self.assertTrue(_get_dayobs_key(dataId) == "exposure.day_obs")

377 dataId = {"day_obs": 20200312}

378 self.assertTrue(_get_dayobs_key(dataId) == "day_obs")

379 dataId = {"missing": 20200312}

380 self.assertTrue(_get_dayobs_key(dataId) is None)

381 return

382

383 def test__get_seqnum_key(self) -> None:

384 dataId = {"a_random_key": 321, "exposure.seq_num": 123, "z_random_key": "abc"}

385 self.assertTrue(_get_seqnum_key(dataId) == "exposure.seq_num")

386 dataId = {"seq_num": 123}

387 self.assertTrue(_get_seqnum_key(dataId) == "seq_num")

388 dataId = {"missing": 123}

389 self.assertTrue(_get_seqnum_key(dataId) is None)

390 return

391

392 def test__get_expid_key(self) -> None:

393 dataId = {"a_random_key": 321, "exposure.id": 123, "z_random_key": "abc"}

394 self.assertTrue(_get_expid_key(dataId) == "exposure.id")

395 dataId = {"a_random_key": 321, "exposure": 123, "z_random_key": "abc"}

396 self.assertTrue(_get_expid_key(dataId) == "exposure")

397 dataId = {"missing": 123}

398 self.assertTrue(_get_expid_key(dataId) is None)

399 return

400

401 def test_updateDataId(self) -> None:

402 # check with a dataCoordinate

403 dataId: Any = copy.copy(self.expRecordNoDetector.dataId)

404 self.assertTrue("detector" not in dataId)

405 dataId = updateDataId(dataId, detector=123)

406 self.assertTrue("detector" in dataId)

407 self.assertEqual(dataId["detector"], 123)

408

409 # check with a dict

410 self.assertIsInstance(self.rawDataId, dict)

411 dataId = copy.copy(self.rawDataId)

412 dataId.pop("detector")

413 self.assertTrue("detector" not in dataId)

414 dataId = updateDataId(dataId, detector=321)

415 self.assertTrue("detector" in dataId)

416 self.assertEqual(dataId["detector"], 321)

417

418 def test_getExpRecord(self) -> None:

419 expId = self.expIdOnly["exposure"]

420 dayObs = self.dayObsSeqNumIdOnly["day_obs"]

421 seqNum = self.dayObsSeqNumIdOnly["seq_num"]

422 assert seqNum is not None

423

424 recordByExpId = getExpRecord(self.butler, "LATISS", expId=expId)

425 self.assertIsInstance(recordByExpId, dafButler.DimensionRecord)

426

427 recordByDayObsSeqNum = getExpRecord(self.butler, "LATISS", dayObs=dayObs, seqNum=seqNum)

428 self.assertIsInstance(recordByDayObsSeqNum, dafButler.DimensionRecord)

429 self.assertEqual(recordByExpId, recordByDayObsSeqNum)

430

431 with self.assertRaises(ValueError):

432 # because we need dayObs too, so immediate raise due to bad args

433 _ = getExpRecord(self.butler, "LATISS", seqNum=seqNum)

434

435 with self.assertRaises(RuntimeError):

436 # (dayObs, seqNum) no longer matches the expId, so there are no

437 # results, which is a RuntimeError

438 _ = getExpRecord(self.butler, "LATISS", expId=expId, dayObs=dayObs, seqNum=seqNum + 1)

439

440

441class ButlerInitTestCase(lsst.utils.tests.TestCase):

442 """Separately test whether we can make a butler with the env var set

443 and that the expected error type is raised and passed through when it is

444 not, as this is relied upon to correctly skip tests when butler init is

445 not possible.

446 """

447

448 def test_dafButlerRaiseTypes(self) -> None:

449 # If DAF_BUTLER_REPOSITORY_INDEX is not set *at all* then

450 # using an instrument label raises a FileNotFoundError

451 with unittest.mock.patch.dict("os.environ"):

452 if "DAF_BUTLER_REPOSITORY_INDEX" in os.environ: # can't del unless it's already there 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true

453 del os.environ["DAF_BUTLER_REPOSITORY_INDEX"]

454 with self.assertRaises(FileNotFoundError):

455 dafButler.Butler("LATISS") # type: ignore[abstract]

456

457 # If DAF_BUTLER_REPOSITORY_INDEX is present but is just an empty

458 # string then using a label raises a RuntimeError

459 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_REPOSITORY_INDEX": ""}):

460 with self.assertRaises(FileNotFoundError):

461 dafButler.Butler("LATISS") # type: ignore[abstract]

462

463 # If DAF_BUTLER_REPOSITORY_INDEX _is_ set, we can't rely on any given

464 # camera existing, but we can check that we get the expected error

465 # when trying to init an instrument which definitely won't be defined.

466 if os.getenv("DAF_BUTLER_REPOSITORY_INDEX"): 466 ↛ 467line 466 didn't jump to line 467 because the condition on line 466 was never true

467 with self.assertRaises(FileNotFoundError):

468 dafButler.Butler("NotAValidCameraName") # type: ignore[abstract]

469

470 def test_makeDefaultLatissButlerRaiseTypes(self) -> None:

471 """makeDefaultLatissButler unifies the mixed exception types from

472 butler inits, so test all available possibilities here.

473 """

474 if getSite() == "jenkins": 474 ↛ 476line 474 didn't jump to line 476 because the condition on line 474 was always true

475 raise unittest.SkipTest("Skip running butler-driven tests in Jenkins.")

476 with unittest.mock.patch.dict("os.environ"):

477 if "DAF_BUTLER_REPOSITORY_INDEX" in os.environ: # can't del unless it's already there

478 del os.environ["DAF_BUTLER_REPOSITORY_INDEX"]

479 with self.assertRaises(FileNotFoundError):

480 makeDefaultLatissButler()

481

482 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_REPOSITORY_INDEX": ""}):

483 with self.assertRaises(FileNotFoundError):

484 makeDefaultLatissButler()

485

486 fakeFile = "/path/to/a/file/which/does/not_exist.yaml"

487 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_REPOSITORY_INDEX": fakeFile}):

488 with self.assertRaises(FileNotFoundError):

489 makeDefaultLatissButler()

490

491 def test_DAF_BUTLER_REPOSITORY_INDEX_value(self) -> None:

492 # If DAF_BUTLER_REPOSITORY_INDEX is truthy then we expect it to point

493 # to an actual file

494 repoFile = os.getenv("DAF_BUTLER_REPOSITORY_INDEX")

495 if repoFile: 495 ↛ 496line 495 didn't jump to line 496 because the condition on line 495 was never true

496 self.assertTrue(ResourcePath(repoFile).exists())

497

498

499class TestMemory(lsst.utils.tests.MemoryTestCase):

500 pass

501

502

503def setup_module(module: object) -> None:

504 lsst.utils.tests.init()

505

506

507if __name__ == "__main__": 507 ↛ 508line 507 didn't jump to line 508 because the condition on line 507 was never true

508 lsst.utils.tests.init()

509 unittest.main()