Coverage for tests/test_logFormatter.py: 100%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Tests for ButlerLogRecordsFormatter.""" 

29 

30import json 

31import logging 

32import os 

33import sys 

34import tempfile 

35import unittest 

36from logging import FileHandler 

37 

38from lsst.daf.butler import Butler, DatasetRef, DatasetType, FileDataset 

39from lsst.daf.butler.logging import ( 

40 ButlerLogRecordHandler, 

41 ButlerLogRecords, 

42 JsonLogFormatter, 

43 _ButlerLogRecordsModelV1, 

44) 

45from lsst.daf.butler.tests._repo_template_cache import make_repo_for_test 

46from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

47 

48TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51class ButlerLogRecordsFormatterTestCase(unittest.TestCase): 

52 """Test for ButlerLogRecords put/get.""" 

53 

54 def setUp(self): 

55 self.root = makeTestTempDir(TESTDIR) 

56 make_repo_for_test(self.root) 

57 

58 self.run = "testrun" 

59 self.butler = Butler.from_config(self.root, run=self.run) 

60 self.enterContext(self.butler) 

61 self.datasetType = DatasetType("test_logs", [], "ButlerLogRecords", universe=self.butler.dimensions) 

62 

63 self.butler.registry.registerDatasetType(self.datasetType) 

64 

65 def tearDown(self): 

66 removeTestTempDir(self.root) 

67 

68 def testButlerLogRecordsFormatter(self): 

69 """Test that we can get and put records handled by 

70 ButlerLogRecordHandler. 

71 """ 

72 handler = ButlerLogRecordHandler() 

73 

74 log = logging.getLogger(self.id()) 

75 log.setLevel(logging.INFO) 

76 log.addHandler(handler) 

77 

78 log.info("An INFO message") 

79 log.debug("A DEBUG message") 

80 log.warning("A WARNING message") 

81 

82 ref = self.butler.put(handler.records, self.datasetType) 

83 records = self.butler.get(ref) 

84 

85 self.assertEqual(records, handler.records) 

86 self.assertEqual(len(records), 2) 

87 

88 def testButlerLogRecordsFormatterExtra(self): 

89 """Test that we can get and put records handled by 

90 ButlerLogRecordHandler with extra JSON data attached. 

91 """ 

92 handler = ButlerLogRecordHandler() 

93 

94 log = logging.getLogger(self.id()) 

95 log.setLevel(logging.INFO) 

96 log.addHandler(handler) 

97 

98 log.info("An INFO message") 

99 log.debug("A DEBUG message") 

100 log.warning("A WARNING message") 

101 

102 extra_data = {"extra1": 1, "extra2": 2} 

103 handler.records.extra = extra_data 

104 ref = self.butler.put(handler.records, self.datasetType) 

105 records = self.butler.get(ref) 

106 

107 self.assertEqual(records, handler.records) 

108 self.assertEqual(len(records), 2) 

109 

110 def testButlerLogRecordsFormatterEmptyExtra(self): 

111 """Test that we can store extra information even with no log 

112 records written. 

113 """ 

114 handler = ButlerLogRecordHandler() 

115 

116 log = logging.getLogger(self.id()) 

117 log.setLevel(logging.INFO) 

118 log.addHandler(handler) 

119 

120 extra_data = {"extra1": 1, "extra2": 2} 

121 handler.records.extra = extra_data 

122 ref = self.butler.put(handler.records, self.datasetType) 

123 records = self.butler.get(ref) 

124 

125 self.assertEqual(records, handler.records) 

126 self.assertEqual(len(records), 0) 

127 

128 @unittest.skipIf( 

129 sys.version_info < (3, 12, 0), 

130 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

131 ) 

132 def testButlerLogRecordsFormatterV1(self): 

133 """Test that we can read log records stored via the old V1 format 

134 (just a JSON list). 

135 """ 

136 handler = ButlerLogRecordHandler() 

137 

138 log = logging.getLogger(self.id()) 

139 log.setLevel(logging.INFO) 

140 log.addHandler(handler) 

141 

142 log.info("An INFO message") 

143 log.debug("A DEBUG message") 

144 log.warning("A WARNING message") 

145 

146 with tempfile.NamedTemporaryFile( 

147 mode="w", suffix=".json", prefix="butler-log-", delete_on_close=False 

148 ) as tmp: 

149 # We can't use butler.put since that always writes the new format, 

150 # so we write manually and ingest. 

151 tmp.write( 

152 _ButlerLogRecordsModelV1(root=handler.records._records).model_dump_json( 

153 exclude_unset=True, exclude_defaults=True 

154 ) 

155 ) 

156 tmp.close() 

157 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

158 dataset = FileDataset(path=os.path.abspath(tmp.name), refs=ref) 

159 self.butler.ingest(dataset, transfer="move") 

160 

161 records = self.butler.get(ref) 

162 self.assertEqual(records, handler.records) 

163 self.assertEqual(len(records), 2) 

164 

165 @unittest.skipIf( 

166 sys.version_info < (3, 12, 0), 

167 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

168 ) 

169 def testJsonLogRecordsFormatter(self): 

170 """Test that externally created JSON format stream files work.""" 

171 log = logging.getLogger(self.id()) 

172 log.setLevel(logging.INFO) 

173 

174 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

175 handler = FileHandler(tmp.name) 

176 handler.setFormatter(JsonLogFormatter()) 

177 log.addHandler(handler) 

178 

179 log.info("An INFO message") 

180 log.debug("A DEBUG message") 

181 log.warning("A WARNING message") 

182 

183 handler.close() 

184 

185 # Now ingest the file. 

186 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

187 dataset = FileDataset(path=tmp.name, refs=ref) 

188 self.butler.ingest(dataset, transfer="move") 

189 

190 records = self.butler.get(ref) 

191 self.assertEqual(len(records), 2) 

192 

193 @unittest.skipIf( 

194 sys.version_info < (3, 12, 0), 

195 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

196 ) 

197 def testJsonLogRecordsFormatterExtra(self): 

198 """Test that externally created JSON format stream files work with 

199 extra JSON data appended. 

200 """ 

201 log = logging.getLogger(self.id()) 

202 log.setLevel(logging.INFO) 

203 

204 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

205 handler = FileHandler(tmp.name) 

206 handler.setFormatter(JsonLogFormatter()) 

207 log.addHandler(handler) 

208 

209 log.info("An INFO message") 

210 log.debug("A DEBUG message") 

211 log.warning("A WARNING message") 

212 

213 handler.close() 

214 extra_data = {"extra1": 1, "extra2": 2} 

215 with open(tmp.name, "a") as stream: 

216 ButlerLogRecords.write_streaming_extra(stream, json.dumps(extra_data)) 

217 

218 # Now ingest the file. 

219 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

220 dataset = FileDataset(path=tmp.name, refs=ref) 

221 self.butler.ingest(dataset, transfer="move") 

222 

223 records: ButlerLogRecords = self.butler.get(ref) 

224 self.assertEqual(len(records), 2) 

225 self.assertEqual(records.extra, extra_data) 

226 

227 @unittest.skipIf( 

228 sys.version_info < (3, 12, 0), 

229 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

230 ) 

231 def testJsonLogRecordsFormatterExtraEmpty(self): 

232 """Test that externally created JSON format stream files work with 

233 extra JSON data appended even if no log messages written. 

234 """ 

235 log = logging.getLogger(self.id()) 

236 log.setLevel(logging.INFO) 

237 

238 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

239 handler = FileHandler(tmp.name) 

240 handler.setFormatter(JsonLogFormatter()) 

241 log.addHandler(handler) 

242 

243 handler.close() 

244 extra_data = {"extra1": 1, "extra2": 2} 

245 with open(tmp.name, "a") as stream: 

246 ButlerLogRecords.write_streaming_extra(stream, json.dumps(extra_data)) 

247 

248 # Now ingest the file. 

249 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

250 dataset = FileDataset(path=tmp.name, refs=ref) 

251 self.butler.ingest(dataset, transfer="move") 

252 

253 records: ButlerLogRecords = self.butler.get(ref) 

254 self.assertEqual(len(records), 0) 

255 self.assertEqual(records.extra, extra_data) 

256 

257 

258if __name__ == "__main__": 

259 unittest.main()