From 08ccec1be9c373ebcd2dcf16cf4ff56813ef65cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela?= Date: Mon, 24 Oct 2016 17:11:36 +0200 Subject: [PATCH 1/3] Add unit tests for reproducible createManifestEntry --- unittests.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/unittests.py b/unittests.py index b704ea10..03058c3d 100644 --- a/unittests.py +++ b/unittests.py @@ -13,6 +13,7 @@ import multiprocessing import os import unittest +import tempfile import clcache from clcache import ( @@ -939,6 +940,45 @@ def testTouchEntry(self): self.assertEqual(TestManifest.entry2, manifest.entries()[0]) +class TestCreateManifestEntry(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tempDir = tempfile.TemporaryDirectory() + for i in range(10): + sampleName = 'sample{}.h'.format(i) + filePath = os.path.join(cls.tempDir.name, '{}.h'.format(sampleName)) + with open(filePath, 'w') as f: + f.write('#define {}'.format(sampleName)) + + cls.includePaths = list(sorted(clcache.filesBeneath(cls.tempDir.name))) + cls.manifestHash = 'ffffffffffffffffffffffffffffffff' + cls.expectedManifestEntry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, + TestCreateManifestEntry.includePaths) + + @classmethod + def tearDownClass(cls): + cls.tempDir.cleanup() + + def assertManifestEntryIsCorrect(self, entry): + self.assertEqual(entry.includesContentHash, TestCreateManifestEntry.expectedManifestEntry.includesContentHash) + self.assertEqual(entry.objectHash, TestCreateManifestEntry.expectedManifestEntry.objectHash) + self.assertEqual(entry.includeFiles, TestCreateManifestEntry.expectedManifestEntry.includeFiles) + + def testIsConsistentWithSameInput(self): + entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, TestCreateManifestEntry.includePaths) + self.assertManifestEntryIsCorrect(entry) + + def testIsConsistentWithReverseList(self): + reversedIncludePaths = list(reversed(TestCreateManifestEntry.includePaths)) + entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, reversedIncludePaths) + self.assertManifestEntryIsCorrect(entry) + + def testIsConsistentWithDuplicateEntries(self): + includePathsWithDuplicates = TestCreateManifestEntry.includePaths + TestCreateManifestEntry.includePaths + entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, includePathsWithDuplicates) + self.assertManifestEntryIsCorrect(entry) + + if __name__ == '__main__': unittest.TestCase.longMessage = True unittest.main() From ab441ad6fce2121d6ec4b1b7d9f2020e64dee459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela?= Date: Mon, 24 Oct 2016 18:07:42 +0200 Subject: [PATCH 2/3] Make createManifestEntry deterministic --- clcache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/clcache.py b/clcache.py index cbbe4d54..d47c60e0 100644 --- a/clcache.py +++ b/clcache.py @@ -1398,11 +1398,13 @@ def processCacheHit(cache, objectFile, cachekey): def createManifestEntry(manifestHash, includePaths): - includesWithHash = {path:getFileHash(path) for path in includePaths} - includesContentHash = ManifestRepository.getIncludesContentHashForHashes(includesWithHash.values()) + sortedIncludePaths = sorted(set(includePaths)) + includeHashes = [getFileHash(path) for path in sortedIncludePaths] + + safeIncludes = [collapseBasedirToPlaceholder(path) for path in sortedIncludePaths] + includesContentHash = ManifestRepository.getIncludesContentHashForHashes(includeHashes) cachekey = CompilerArtifactsRepository.computeKeyDirect(manifestHash, includesContentHash) - safeIncludes = [collapseBasedirToPlaceholder(path) for path in includesWithHash.keys()] return ManifestEntry(safeIncludes, includesContentHash, cachekey) From fc41196222481bc86be8e32699dab7899058bdb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela?= Date: Mon, 24 Oct 2016 18:14:09 +0200 Subject: [PATCH 3/3] Bump manifest format version --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index d47c60e0..4bf37c27 100644 --- a/clcache.py +++ b/clcache.py @@ -192,7 +192,7 @@ class ManifestRepository(object): # invalidation, such that a manifest that was stored using the old format is not # interpreted using the new format. Instead the old file will not be touched # again due to a new manifest hash and is cleaned away after some time. - MANIFEST_FILE_FORMAT_VERSION = 5 + MANIFEST_FILE_FORMAT_VERSION = 6 def __init__(self, manifestsRootDir): self._manifestsRootDir = manifestsRootDir