Coverage for tests/test_repo_template_cache.py: 100%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30import os 

31import unittest 

32 

33from lsst.daf.butler import Butler, Config 

34from lsst.daf.butler.tests._repo_template_cache import ( 

35 clear_repo_template_cache, 

36 make_repo_for_test, 

37 template_cache_stats, 

38) 

39from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

40 

41TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

42 

43 

44class RepoTemplateCacheTestCase(unittest.TestCase): 

45 """Tests for the test repository template cache.""" 

46 

47 def setUp(self) -> None: 

48 self.root = makeTestTempDir(TESTDIR) 

49 clear_repo_template_cache() 

50 

51 def tearDown(self) -> None: 

52 removeTestTempDir(self.root) 

53 clear_repo_template_cache() 

54 

55 def _config(self) -> Config: 

56 config = Config() 

57 config["datastore", "cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore" 

58 config["registry", "db"] = "sqlite:///<butlerRoot>/gen3.sqlite3" 

59 return config 

60 

61 def test_second_call_is_served_from_cache(self) -> None: 

62 """An identical configuration builds one template and copies it.""" 

63 first = os.path.join(self.root, "one") 

64 second = os.path.join(self.root, "two") 

65 make_repo_for_test(first, config=self._config(), forceConfigRoot=False) 

66 make_repo_for_test(second, config=self._config(), forceConfigRoot=False) 

67 stats = template_cache_stats() 

68 self.assertEqual(stats.templates, 1) 

69 self.assertEqual(stats.served, 2) 

70 

71 def test_returned_config_points_at_the_caller_root(self) -> None: 

72 """The returned Config must describe the copy, not the template.""" 

73 first = os.path.join(self.root, "one") 

74 second = os.path.join(self.root, "two") 

75 make_repo_for_test(first, config=self._config(), forceConfigRoot=False) 

76 returned = make_repo_for_test(second, config=self._config(), forceConfigRoot=False) 

77 butler = Butler.from_config(returned, writeable=True) 

78 self.assertIn(second, str(butler._registry._db.filename)) 

79 

80 def test_copied_repo_is_usable_and_independent(self) -> None: 

81 """Two repos from one template do not share state.""" 

82 first = os.path.join(self.root, "one") 

83 second = os.path.join(self.root, "two") 

84 make_repo_for_test(first, config=self._config(), forceConfigRoot=False) 

85 make_repo_for_test(second, config=self._config(), forceConfigRoot=False) 

86 b1 = Butler.from_config(first, writeable=True, run="r1") 

87 b2 = Butler.from_config(second, writeable=True, run="r2") 

88 b1.registry.registerRun("only_in_first") 

89 self.assertNotIn("only_in_first", set(b2.registry.queryCollections())) 

90 

91 def test_datastore_config_shares_one_database(self) -> None: 

92 """Datastore settings do not reach the database, so it is reused.""" 

93 other = self._config() 

94 other["datastore", "cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore" 

95 other["datastore", "checksum"] = False 

96 make_repo_for_test(os.path.join(self.root, "one"), config=self._config(), forceConfigRoot=False) 

97 make_repo_for_test(os.path.join(self.root, "two"), config=other, forceConfigRoot=False) 

98 stats = template_cache_stats() 

99 self.assertEqual(stats.templates, 1) 

100 self.assertEqual(stats.reused_database, 1) 

101 

102 def test_each_repo_gets_its_own_butler_yaml(self) -> None: 

103 """Sharing a database must not share the rest of the configuration.""" 

104 other = self._config() 

105 other["datastore", "cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore" 

106 first = os.path.join(self.root, "one") 

107 second = os.path.join(self.root, "two") 

108 make_repo_for_test(first, config=self._config(), forceConfigRoot=False) 

109 make_repo_for_test(second, config=other, forceConfigRoot=False) 

110 self.assertIn("inMemoryDatastore", Config(os.path.join(first, "butler.yaml"))["datastore", "cls"]) 

111 self.assertIn("fileDatastore", Config(os.path.join(second, "butler.yaml"))["datastore", "cls"]) 

112 

113 def _files_under(self, directory: str) -> set[str]: 

114 """Return every file below a directory, relative to it.""" 

115 found = set() 

116 for dirpath, _, filenames in os.walk(directory): 

117 for filename in filenames: 

118 found.add(os.path.relpath(os.path.join(dirpath, filename), directory)) 

119 return found 

120 

121 def test_root_with_uri_metacharacters(self) -> None: 

122 """A root holding `?` or `#` must land where makeRepo puts it. 

123 

124 Such a root is mangled during creation, because the tag replacement 

125 drops a URI fragment and the database connection string is then parsed 

126 as a URI. The helper has to follow that resolution rather than 

127 predict a path of its own. 

128 """ 

129 helper_parent = os.path.join(self.root, "helper") 

130 direct_parent = os.path.join(self.root, "direct") 

131 subdir = "sub?#dir" 

132 make_repo_for_test(os.path.join(helper_parent, subdir), config=self._config(), forceConfigRoot=False) 

133 Butler.makeRepo(os.path.join(direct_parent, subdir), config=self._config(), forceConfigRoot=False) 

134 self.assertEqual(self._files_under(helper_parent), self._files_under(direct_parent)) 

135 

136 # Producing the right files is not enough on its own, because failing 

137 # to locate the database would also leave the repository correct but 

138 # uncached. A second such root has to reuse the first one's database, 

139 # which it can only do if the location was resolved rather than 

140 # guessed. 

141 second_parent = os.path.join(self.root, "second") 

142 make_repo_for_test(os.path.join(second_parent, subdir), config=self._config(), forceConfigRoot=False) 

143 stats = template_cache_stats() 

144 self.assertEqual(stats.templates, 1) 

145 self.assertEqual(stats.reused_database, 1) 

146 self.assertEqual(self._files_under(second_parent), self._files_under(direct_parent)) 

147 

148 def test_obscore_config_survives_a_cache_hit(self) -> None: 

149 """Obscore settings are stripped from ``butler.yaml`` and kept in the 

150 registry, so a copy cannot recover them from the file. 

151 """ 

152 config = self._config() 

153 config["registry", "managers", "obscore"] = { 

154 "cls": "lsst.daf.butler.registry.obscore.ObsCoreLiveTableManager", 

155 "config": Config(os.path.join(TESTDIR, "config", "basic", "obscore.yaml")).toDict(), 

156 } 

157 second = os.path.join(self.root, "two") 

158 make_repo_for_test(os.path.join(self.root, "one"), config=config, forceConfigRoot=False) 

159 returned = make_repo_for_test(second, config=config, forceConfigRoot=False) 

160 self.assertEqual(template_cache_stats().reused_config, 1) 

161 obscore_config = ("registry", "managers", "obscore", "config") 

162 self.assertNotIn(obscore_config, Config(os.path.join(second, "butler.yaml"))) 

163 self.assertIn(obscore_config, returned) 

164 with Butler.from_config(second, writeable=True) as butler: 

165 self.assertIsNotNone(butler._registry.obsCoreTableManager) 

166 

167 def test_existing_config_is_not_overwritten(self) -> None: 

168 """A copy must refuse an occupied root, as makeRepo does.""" 

169 second = os.path.join(self.root, "two") 

170 make_repo_for_test(os.path.join(self.root, "one"), config=self._config(), forceConfigRoot=False) 

171 make_repo_for_test(second, config=self._config(), forceConfigRoot=False) 

172 self.assertEqual(template_cache_stats().reused_config, 1) 

173 with self.assertRaises(FileExistsError): 

174 make_repo_for_test(second, config=self._config(), forceConfigRoot=False) 

175 

176 def test_rewritten_config_file_is_not_reused(self) -> None: 

177 """A configuration named by path is identified by its contents.""" 

178 path = os.path.join(self.root, "repo.yaml") 

179 self._config().dumpToUri(path) 

180 make_repo_for_test(os.path.join(self.root, "one"), config=path, forceConfigRoot=False) 

181 changed = self._config() 

182 changed["datastore", "cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore" 

183 changed.dumpToUri(path, overwrite=True) 

184 second = os.path.join(self.root, "two") 

185 make_repo_for_test(second, config=path, forceConfigRoot=False) 

186 self.assertIn("fileDatastore", Config(os.path.join(second, "butler.yaml"))["datastore", "cls"]) 

187 self.assertEqual(template_cache_stats().reused_config, 0) 

188 

189 def test_different_dimensions_build_a_second_database(self) -> None: 

190 """The dimension universe does reach the database, so it keys.""" 

191 path = os.path.join(TESTDIR, "config", "dimensions", "dimensions1.yaml") 

192 make_repo_for_test(os.path.join(self.root, "one"), config=self._config(), forceConfigRoot=False) 

193 make_repo_for_test( 

194 os.path.join(self.root, "two"), 

195 config=self._config(), 

196 dimensionConfig=path, 

197 forceConfigRoot=False, 

198 ) 

199 self.assertEqual(template_cache_stats().templates, 2) 

200 

201 def test_outfile_bypasses_the_cache(self) -> None: 

202 """Outfile writes the config elsewhere, so it cannot be cached.""" 

203 make_repo_for_test(os.path.join(self.root, "one"), config=self._config(), forceConfigRoot=False) 

204 make_repo_for_test( 

205 os.path.join(self.root, "two"), 

206 config=self._config(), 

207 forceConfigRoot=False, 

208 outfile=os.path.join(self.root, "out.yaml"), 

209 ) 

210 self.assertEqual(template_cache_stats().bypassed, 1) 

211 

212 def test_non_sqlite_registry_bypasses_the_cache(self) -> None: 

213 """A client/server registry lives outside the copied directory.""" 

214 config = self._config() 

215 config["registry", "db"] = "postgresql://example.invalid/butler" 

216 config["registry", "namespace"] = "namespace_deadbeef" 

217 with self.assertRaises(Exception): 

218 # The database does not exist, so creation fails. What matters is 

219 # that the attempt bypassed the cache rather than building and 

220 # retaining a single-use template. 

221 make_repo_for_test(os.path.join(self.root, "pg"), config=config, forceConfigRoot=False) 

222 stats = template_cache_stats() 

223 self.assertEqual(stats.bypassed, 1) 

224 self.assertEqual(stats.templates, 0) 

225 

226 def test_standalone_and_overwrite_bypass_the_cache(self) -> None: 

227 """Standalone and overwrite change what makeRepo writes.""" 

228 make_repo_for_test(os.path.join(self.root, "a"), config=self._config(), standalone=True) 

229 make_repo_for_test(os.path.join(self.root, "b"), config=self._config(), overwrite=True) 

230 self.assertEqual(template_cache_stats().bypassed, 2) 

231 

232 

233if __name__ == "__main__": 

234 unittest.main()