Coverage for tests/test_efdUtils.py: 92%

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Test cases for utils.""" 

23 

24import asyncio 

25import datetime 

26import unittest 

27from typing import Any 

28 

29import astropy 

30import pandas as pd 

31from astropy.time import Time 

32from utils import getVcr 

33 

34import lsst.utils.tests 

35from lsst.summit.utils.dateTime import ( 

36 astropyToEfdTimestamp, 

37 efdTimestampToAstropy, 

38 getDayObsEndTime, 

39 getDayObsStartTime, 

40) 

41from lsst.summit.utils.efdUtils import ( 

42 clipDataToEvent, 

43 getEfdData, 

44 getMostRecentRowWithDataBefore, 

45 getTopics, 

46 makeEfdClient, 

47) 

48from lsst.summit.utils.tmaUtils import TMAEvent, TMAState 

49 

50HAS_EFD_CLIENT = True 

51try: 

52 import lsst_efd_client 

53except ImportError: 

54 HAS_EFD_CLIENT = False 

55 

56vcr = getVcr() 

57 

58 

59@unittest.skipIf(not HAS_EFD_CLIENT, "No EFD client available") 

60@vcr.use_cassette() 

61class EfdUtilsTestCase(lsst.utils.tests.TestCase): 

62 # class attributes populated in setUpClass 

63 client: Any 

64 dayObs: int 

65 axisTopic: str 

66 timeSeriesTopic: str 

67 event: TMAEvent 

68 

69 @classmethod 

70 @vcr.use_cassette() 

71 def setUpClass(cls) -> None: 

72 try: 

73 cls.client = makeEfdClient(testing=True) 

74 except RuntimeError: 

75 raise unittest.SkipTest("Could not instantiate an EFD client") 

76 cls.dayObs = 20230531 

77 # get a sample expRecord here to test expRecordToTimespan 

78 cls.axisTopic = "lsst.sal.MTMount.logevent_azimuthMotionState" 

79 cls.timeSeriesTopic = "lsst.sal.MTMount.azimuth" 

80 cls.event = TMAEvent( 

81 dayObs=20230531, 

82 seqNum=27, 

83 type=TMAState.TRACKING, 

84 endReason=TMAState.SLEWING, 

85 duration=0.47125244140625, 

86 begin=Time(1685578353.2265284, scale="utc", format="unix"), 

87 end=Time(1685578353.6977808, scale="utc", format="unix"), 

88 blockInfos=None, 

89 version=0, 

90 _startRow=254, 

91 _endRow=255, 

92 ) 

93 

94 @vcr.use_cassette() 

95 def tearDown(self) -> None: 

96 loop = asyncio.get_event_loop() 

97 if self.client.influx_client is not None: 97 ↛ exitline 97 didn't return from function 'tearDown' because the condition on line 97 was always true

98 loop.run_until_complete(self.client.influx_client.close()) 

99 

100 @vcr.use_cassette() 

101 def test_makeEfdClient(self) -> None: 

102 self.assertIsInstance(self.client, lsst_efd_client.efd_helper.EfdClient) 

103 

104 @vcr.use_cassette() 

105 def test_getTopics(self) -> None: 

106 topics = getTopics(self.client, "lsst.sal.MTMount*") 

107 self.assertIsInstance(topics, list) 

108 self.assertGreater(len(topics), 0) 

109 

110 topics = getTopics(self.client, "*fake.topics.does.not.exist*") 

111 self.assertIsInstance(topics, list) 

112 self.assertEqual(len(topics), 0) 

113 

114 # check we can find the mount with a preceding wildcard 

115 topics = getTopics(self.client, "*mTmoUnt*") 

116 self.assertIsInstance(topics, list) 

117 self.assertGreater(len(topics), 0) 

118 

119 # check it fails if we don't allow case insensitivity 

120 topics = getTopics(self.client, "*mTmoUnt*", caseSensitive=True) 

121 self.assertIsInstance(topics, list) 

122 self.assertEqual(len(topics), 0) 

123 

124 @vcr.use_cassette() 

125 def test_getEfdData(self) -> None: 

126 dayStart = getDayObsStartTime(self.dayObs) 

127 dayEnd = getDayObsEndTime(self.dayObs) 

128 oneDay = datetime.timedelta(hours=24) 

129 # twelveHours = datetime.timedelta(hours=12) 

130 

131 # test the dayObs interface 

132 dayObsData = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs) 

133 self.assertIsInstance(dayObsData, pd.DataFrame) 

134 

135 # test the starttime interface 

136 dayStartData = getEfdData(self.client, self.axisTopic, begin=dayStart, timespan=oneDay) 

137 self.assertIsInstance(dayStartData, pd.DataFrame) 

138 

139 # check they're equal 

140 self.assertTrue(dayObsData.equals(dayStartData)) 

141 

142 # test the starttime interface with an endtime 

143 dayEnd = getDayObsEndTime(self.dayObs) 

144 dayStartEndData = getEfdData(self.client, self.axisTopic, begin=dayStart, end=dayEnd) 

145 self.assertTrue(dayObsData.equals(dayStartEndData)) 

146 

147 # test event 

148 # note that here we're going to clip to an event and pad things, so 

149 # we want to use the timeSeriesTopic not the states, so that there's 

150 # plenty of rows to test the padding is actually working 

151 eventData = getEfdData(self.client, self.timeSeriesTopic, event=self.event) 

152 self.assertIsInstance(dayObsData, pd.DataFrame) 

153 

154 # test padding options 

155 padded = getEfdData(self.client, self.timeSeriesTopic, event=self.event, prePadding=1, postPadding=2) 

156 self.assertGreater(len(padded), len(eventData)) 

157 startTimeDiff = efdTimestampToAstropy(eventData.iloc[0]["private_efdStamp"]) - efdTimestampToAstropy( 

158 padded.iloc[0]["private_efdStamp"] 

159 ) 

160 endTimeDiff = efdTimestampToAstropy(padded.iloc[-1]["private_efdStamp"]) - efdTimestampToAstropy( 

161 eventData.iloc[-1]["private_efdStamp"] 

162 ) 

163 

164 self.assertGreater(startTimeDiff.sec, 0) 

165 self.assertLess(startTimeDiff.sec, 1.1) # padding isn't super exact, so give a little wiggle room 

166 self.assertGreater(endTimeDiff.sec, 0) 

167 self.assertLess(endTimeDiff.sec, 2.1) # padding isn't super exact, so give a little wiggle room 

168 

169 with self.assertRaises(ValueError): 

170 # not enough info to constrain 

171 _ = getEfdData(self.client, self.axisTopic) 

172 # dayObs supplied and a start time is not allowed 

173 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, begin=dayStart) 

174 # dayObs supplied and a stop time is not allowed 

175 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, end=dayEnd) 

176 # dayObs supplied and timespan is not allowed 

177 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, timespan=oneDay) 

178 # being alone is not allowed 

179 _ = getEfdData(self.client, self.axisTopic, begin=self.dayObs) 

180 # good query, except the topic doesn't exist 

181 _ = getEfdData(self.client, "badTopic", begin=dayStart, end=dayEnd) 

182 

183 @vcr.use_cassette() 

184 def test_raiseIfTopicNotInSchema(self) -> None: 

185 dayStart = getDayObsStartTime(self.dayObs) 

186 dayEnd = getDayObsEndTime(self.dayObs) 

187 

188 badTopic = "lsst.sal.nonExistentTopic" 

189 # test this does not raise 

190 _ = getEfdData(self.client, badTopic, begin=dayStart, end=dayEnd, raiseIfTopicNotInSchema=False) 

191 

192 with self.assertRaises(ValueError): 

193 # test this does raise, as raiseIfTopicNotInSchema defaults to True 

194 _ = getEfdData(self.client, badTopic, begin=dayStart, end=dayEnd) 

195 

196 @vcr.use_cassette() 

197 def test_getMostRecentRowWithDataBefore(self) -> None: 

198 time = Time(1687845854.736784, scale="utc", format="unix") 

199 rowData = getMostRecentRowWithDataBefore( 

200 self.client, "lsst.sal.MTM1M3.logevent_forceActuatorState", time 

201 ) 

202 self.assertIsInstance(rowData, pd.Series) 

203 

204 stateTime = efdTimestampToAstropy(rowData["private_efdStamp"]) 

205 self.assertLess(stateTime, time) 

206 

207 def test_efdTimestampToAstropy(self) -> None: 

208 time = efdTimestampToAstropy(1687845854.736784) 

209 self.assertIsInstance(time, astropy.time.Time) 

210 return 

211 

212 def test_astropyToEfdTimestamp(self) -> None: 

213 time = Time(1687845854.736784, scale="utc", format="unix") 

214 efdTimestamp = astropyToEfdTimestamp(time) 

215 self.assertIsInstance(efdTimestamp, float) 

216 return 

217 

218 @vcr.use_cassette() 

219 def test_clipDataToEvent(self) -> None: 

220 # get 10 mins of data either side of the event we'll clip to 

221 duration = datetime.timedelta(seconds=10 * 60) 

222 queryBegin = self.event.begin - duration 

223 queryEnd = self.event.end + duration 

224 dayObsData = getEfdData(self.client, "lsst.sal.MTMount.azimuth", begin=queryBegin, end=queryEnd) 

225 

226 # clip the data, and check it's shorter, non-zero, and falls in the 

227 # right time range 

228 clippedData = clipDataToEvent(dayObsData, self.event) 

229 

230 self.assertIsInstance(clippedData, pd.DataFrame) 

231 self.assertGreater(len(clippedData), 0) 

232 self.assertLess(len(clippedData), len(dayObsData)) 

233 

234 dataStart = efdTimestampToAstropy(clippedData.iloc[0]["private_efdStamp"]) 

235 dataEnd = efdTimestampToAstropy(clippedData.iloc[-1]["private_efdStamp"]) 

236 

237 self.assertGreaterEqual(dataStart, self.event.begin) 

238 self.assertLessEqual(dataEnd, self.event.end) 

239 

240 # test the pre/post padding options 

241 clippedPaddedData = clipDataToEvent(dayObsData, self.event, prePadding=1, postPadding=2) 

242 self.assertIsInstance(clippedPaddedData, pd.DataFrame) 

243 self.assertGreater(len(clippedPaddedData), 0) 

244 self.assertLess(len(clippedPaddedData), len(dayObsData)) 

245 self.assertGreater(len(clippedPaddedData), len(clippedData)) 

246 

247 paddedDataStart = efdTimestampToAstropy(clippedPaddedData.iloc[0]["private_efdStamp"]) 

248 paddedDataEnd = efdTimestampToAstropy(clippedPaddedData.iloc[-1]["private_efdStamp"]) 

249 self.assertLessEqual(paddedDataStart, dataStart) 

250 self.assertGreaterEqual(paddedDataEnd, dataEnd) 

251 

252 # Get the minimum and maximum timestamps before padding 

253 startTimeUnpadded = clippedData["private_efdStamp"].min() 

254 endTimeUnpadded = clippedData["private_efdStamp"].max() 

255 

256 # Get the minimum and maximum timestamps after padding 

257 startTimePadded = clippedPaddedData["private_efdStamp"].min() 

258 endTimePadded = clippedPaddedData["private_efdStamp"].max() 

259 

260 # Check that the difference between the min times and max times is 

261 # approximately equal to the padding. Not exact as data sampling is 

262 # not infinite. 

263 self.assertAlmostEqual(startTimeUnpadded - startTimePadded, 1, delta=0.1) 

264 self.assertAlmostEqual(endTimePadded - endTimeUnpadded, 2, delta=0.1) 

265 return 

266 

267 

268class TestMemory(lsst.utils.tests.MemoryTestCase): 

269 pass 

270 

271 

272def setup_module(module: object) -> None: 

273 lsst.utils.tests.init() 

274 

275 

276if __name__ == "__main__": 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 lsst.utils.tests.init() 

278 unittest.main()