From f0227bb98a16a9804e2b71ef2a6d100c1c7b2719 Mon Sep 17 00:00:00 2001 From: xoviat Date: Fri, 6 Oct 2017 20:05:32 -0500 Subject: [PATCH 01/27] BUG: fix source file duplication in command liine Previously, if the /Tp or /Tc option were passed, clcache would duplicate the source file in the command line, making MSVC think that it was compiling two separate files (/Tcexample.c and example.c), when it was really compiling one. The problem with this was that MSVC only allows some options if it is compiling one file, which caused some invocations of clcache to fail. Closes #289. --- clcache.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clcache.py b/clcache.py index 530fdb04..94f862ab 100644 --- a/clcache.py +++ b/clcache.py @@ -1293,6 +1293,12 @@ def analyze(cmdline): def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsString=True, environment=None): realCmdline = [compilerBinary] + cmdLine + + sourceFile = realCmdline[-1] + if '/Tc' + sourceFile in realCmdline or '/Tp' + sourceFile in realCmdline: + printTraceStatement("Removing last argument becuase of /Tc (Issue #289)") + realCmdline = realCmdline[:-1] + printTraceStatement("Invoking real compiler as {}".format(realCmdline)) environment = environment or os.environ From 40cd447f9c16dc9c1c42022e6bbd273b3f071ff4 Mon Sep 17 00:00:00 2001 From: xoviat Date: Sat, 7 Oct 2017 16:56:09 -0500 Subject: [PATCH 02/27] BUG: improve handling of specified sources --- clcache.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/clcache.py b/clcache.py index 94f862ab..4a53214a 100644 --- a/clcache.py +++ b/clcache.py @@ -1243,13 +1243,17 @@ def parseArgumentsAndInputFiles(cmdline): @staticmethod def analyze(cmdline): + # type: List[str] -> Tuple[List[str], List[str], Set[str]] options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) + skippedSources = [] compl = False if 'Tp' in options: inputFiles += options['Tp'] + skippedSources += options['Tp'] compl = True if 'Tc' in options: inputFiles += options['Tc'] + skippedSources += options['Tc'] compl = True if not inputFiles: @@ -1288,7 +1292,7 @@ def analyze(cmdline): printTraceStatement("Compiler source files: {}".format(inputFiles)) printTraceStatement("Compiler object file: {}".format(objectFiles)) - return inputFiles, objectFiles + return inputFiles, objectFiles, set(skippedSources) def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsString=True, environment=None): @@ -1588,8 +1592,8 @@ def processCompileRequest(cache, compiler, args): printTraceStatement("Expanded commandline '{0!s}'".format(cmdLine)) try: - sourceFiles, objectFiles = CommandLineAnalyzer.analyze(cmdLine) - return scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles) + sourceFiles, objectFiles, skippedSources = CommandLineAnalyzer.analyze(cmdLine) + return scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles, skippedSources) except InvalidArgumentError: printTraceStatement("Cannot cache invocation as {}: invalid argument".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallWithInvalidArgument) @@ -1618,7 +1622,8 @@ def processCompileRequest(cache, compiler, args): printOutAndErr(out, err) return exitCode -def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): +def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles, skippedSources): + # type: (???, str, List[str], ???, List[str], List[str], Set[str]) -> int baseCmdLine = [] setOfSources = set(sourceFiles) for arg in cmdLine: @@ -1630,7 +1635,9 @@ def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] for srcFile, objFile in zip(sourceFiles, objectFiles): - jobCmdLine = baseCmdLine + [srcFile] + jobCmdLine = baseCmdLine + if srcFile not in skippedSources: + jobCmdLine += [srcFile] jobs.append(executor.submit( processSingleSource, compiler, jobCmdLine, srcFile, objFile, environment)) From cc8a52b37a301992f0e1a1aca8ee3645e1182dc6 Mon Sep 17 00:00:00 2001 From: xoviat Date: Sat, 7 Oct 2017 16:57:38 -0500 Subject: [PATCH 03/27] BUG: revert f0227bb --- clcache.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/clcache.py b/clcache.py index 4a53214a..2b8f0cab 100644 --- a/clcache.py +++ b/clcache.py @@ -1297,12 +1297,6 @@ def analyze(cmdline): def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsString=True, environment=None): realCmdline = [compilerBinary] + cmdLine - - sourceFile = realCmdline[-1] - if '/Tc' + sourceFile in realCmdline or '/Tp' + sourceFile in realCmdline: - printTraceStatement("Removing last argument becuase of /Tc (Issue #289)") - realCmdline = realCmdline[:-1] - printTraceStatement("Invoking real compiler as {}".format(realCmdline)) environment = environment or os.environ From ba1f7d11f650c5d84aae24c5eea433209a8f4e95 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 11:53:02 -0500 Subject: [PATCH 04/27] BUG: revise analyze method to detect language --- clcache.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/clcache.py b/clcache.py index 2b8f0cab..021f1a54 100644 --- a/clcache.py +++ b/clcache.py @@ -1243,19 +1243,21 @@ def parseArgumentsAndInputFiles(cmdline): @staticmethod def analyze(cmdline): - # type: List[str] -> Tuple[List[str], List[str], Set[str]] + # type: List[str] -> Tuple[List[Tuple[str, str]], List[str]] options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) - skippedSources = [] + # Use an override pattern to shadow input files that have + # already been specified in the function above + inputFiles = {inputFile : '' for inputFile in inputFiles} compl = False if 'Tp' in options: - inputFiles += options['Tp'] - skippedSources += options['Tp'] + inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']} compl = True if 'Tc' in options: - inputFiles += options['Tc'] - skippedSources += options['Tc'] + inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']} compl = True + # Now collect the inputFiles into the return format + inputFiles = list(inputFiles.items()) if not inputFiles: raise NoSourceFileError() @@ -1288,11 +1290,11 @@ def analyze(cmdline): objectFiles = [tmp] if objectFiles is None: # Generate from .c/.cpp filenames - objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f in inputFiles] + objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f, _ in inputFiles] printTraceStatement("Compiler source files: {}".format(inputFiles)) printTraceStatement("Compiler object file: {}".format(objectFiles)) - return inputFiles, objectFiles, set(skippedSources) + return inputFiles, objectFiles def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsString=True, environment=None): @@ -1586,8 +1588,8 @@ def processCompileRequest(cache, compiler, args): printTraceStatement("Expanded commandline '{0!s}'".format(cmdLine)) try: - sourceFiles, objectFiles, skippedSources = CommandLineAnalyzer.analyze(cmdLine) - return scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles, skippedSources) + sourceFiles, objectFiles = CommandLineAnalyzer.analyze(cmdLine) + return scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles) except InvalidArgumentError: printTraceStatement("Cannot cache invocation as {}: invalid argument".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallWithInvalidArgument) @@ -1616,22 +1618,24 @@ def processCompileRequest(cache, compiler, args): printOutAndErr(out, err) return exitCode -def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles, skippedSources): - # type: (???, str, List[str], ???, List[str], List[str], Set[str]) -> int +def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): + # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int + # Filter out all source files from the command line to form baseCmdLine baseCmdLine = [] - setOfSources = set(sourceFiles) + setOfSources = set([sourceFile for sourceFile, _ in sourceFiles]) + skippedArgs = ['/MP', '/Tc', '/Tp'] for arg in cmdLine: - if not (arg in setOfSources or arg.startswith("/MP")): + if not (arg in setOfSources or any([ + arg.startswith(skippedArg) for skippedArg in skippedArgs])): baseCmdLine.append(arg) exitCode = 0 cleanupRequired = False with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] - for srcFile, objFile in zip(sourceFiles, objectFiles): - jobCmdLine = baseCmdLine - if srcFile not in skippedSources: - jobCmdLine += [srcFile] + for source, objFile in zip(sourceFiles, objectFiles): + srcFile, srcLanguage = source + jobCmdLine = baseCmdLine + list(filter(None, [srcLanguage, srcFile])) jobs.append(executor.submit( processSingleSource, compiler, jobCmdLine, srcFile, objFile, environment)) From 3df60a6eaf3803f4137b4671862caeaed4b96029 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 11:58:19 -0500 Subject: [PATCH 05/27] MAINT: cleanup zip iterator --- clcache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clcache.py b/clcache.py index 021f1a54..c55837c8 100644 --- a/clcache.py +++ b/clcache.py @@ -1633,8 +1633,7 @@ def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles cleanupRequired = False with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] - for source, objFile in zip(sourceFiles, objectFiles): - srcFile, srcLanguage = source + for srcFile, srcLanguage, objFile in zip(zip(*sourceFiles), objectFiles): jobCmdLine = baseCmdLine + list(filter(None, [srcLanguage, srcFile])) jobs.append(executor.submit( processSingleSource, From 662f7ed140ebe2dc6426e98b36c0299aaed9c106 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 11:59:22 -0500 Subject: [PATCH 06/27] BUG: fix syntax error --- clcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clcache.py b/clcache.py index c55837c8..5d43a59c 100644 --- a/clcache.py +++ b/clcache.py @@ -1250,10 +1250,10 @@ def analyze(cmdline): inputFiles = {inputFile : '' for inputFile in inputFiles} compl = False if 'Tp' in options: - inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']} + inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']}) compl = True if 'Tc' in options: - inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']} + inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']}) compl = True # Now collect the inputFiles into the return format From bbdd2571a4e1959754e56e9fe8dd657d6741061a Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 12:11:59 -0500 Subject: [PATCH 07/27] MAINT: cleanup formBaseCommandLine --- clcache.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/clcache.py b/clcache.py index 5d43a59c..fc633787 100644 --- a/clcache.py +++ b/clcache.py @@ -1618,16 +1618,21 @@ def processCompileRequest(cache, compiler, args): printOutAndErr(out, err) return exitCode +def formBaseCommandLine(cmdLine, sourceFiles): + # type: (List[str], List[Tuple[str, str]]) -> List[str] + setOfSources = set([sourceFile for sourceFile, _ in sourceFiles]) + skippedArgs = ('/MP', '/Tc', '/Tp') + baseCmdLine = [ + arg for arg in cmdLine + if not (arg in setOfSources or arg.startswith(skippedArgs)) + ] + + return baseCmdLine + def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int # Filter out all source files from the command line to form baseCmdLine - baseCmdLine = [] - setOfSources = set([sourceFile for sourceFile, _ in sourceFiles]) - skippedArgs = ['/MP', '/Tc', '/Tp'] - for arg in cmdLine: - if not (arg in setOfSources or any([ - arg.startswith(skippedArg) for skippedArg in skippedArgs])): - baseCmdLine.append(arg) + baseCmdLine = formBaseCommandLine(cmdLine, sourceFiles) exitCode = 0 cleanupRequired = False From 94218fd8df7194c86bc35bd01c40dd7d04be9c0b Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 12:13:48 -0500 Subject: [PATCH 08/27] :art: --- clcache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clcache.py b/clcache.py index fc633787..2dc246dd 100644 --- a/clcache.py +++ b/clcache.py @@ -1622,8 +1622,8 @@ def formBaseCommandLine(cmdLine, sourceFiles): # type: (List[str], List[Tuple[str, str]]) -> List[str] setOfSources = set([sourceFile for sourceFile, _ in sourceFiles]) skippedArgs = ('/MP', '/Tc', '/Tp') - baseCmdLine = [ - arg for arg in cmdLine + baseCmdLine = [ + arg for arg in cmdLine if not (arg in setOfSources or arg.startswith(skippedArgs)) ] @@ -1631,7 +1631,7 @@ def formBaseCommandLine(cmdLine, sourceFiles): def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int - # Filter out all source files from the command line to form baseCmdLine + # Filter out all source files from the command line to form baseCmdLine baseCmdLine = formBaseCommandLine(cmdLine, sourceFiles) exitCode = 0 From f69b709b6ddd9649ed44f30194ee31f8a90542f6 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 13:51:49 -0500 Subject: [PATCH 09/27] MAINT: cleanup source file filtering --- clcache.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/clcache.py b/clcache.py index 2dc246dd..d5c02cbe 100644 --- a/clcache.py +++ b/clcache.py @@ -1618,27 +1618,25 @@ def processCompileRequest(cache, compiler, args): printOutAndErr(out, err) return exitCode -def formBaseCommandLine(cmdLine, sourceFiles): - # type: (List[str], List[Tuple[str, str]]) -> List[str] - setOfSources = set([sourceFile for sourceFile, _ in sourceFiles]) - skippedArgs = ('/MP', '/Tc', '/Tp') - baseCmdLine = [ +def filterSourceFiles(cmdLine, sourceFiles): + # type: (List[str], List[Tuple[str, str]]) -> Iterator[str] + setOfSources = set(sourceFile for sourceFile, _ in sourceFiles) + skippedArgs = ('/Tc', '/Tp', '-Tp', '-Tc') + yield from ( arg for arg in cmdLine if not (arg in setOfSources or arg.startswith(skippedArgs)) - ] - - return baseCmdLine + ) def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int # Filter out all source files from the command line to form baseCmdLine - baseCmdLine = formBaseCommandLine(cmdLine, sourceFiles) + baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if arg != '/MP'] exitCode = 0 cleanupRequired = False with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] - for srcFile, srcLanguage, objFile in zip(zip(*sourceFiles), objectFiles): + for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles): jobCmdLine = baseCmdLine + list(filter(None, [srcLanguage, srcFile])) jobs.append(executor.submit( processSingleSource, From df746604d0b0378855543778514c8c6e4fa17747 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 13:54:02 -0500 Subject: [PATCH 10/27] MAINT: improve jobCmdLine readability --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index d5c02cbe..2dde7f8a 100644 --- a/clcache.py +++ b/clcache.py @@ -1637,7 +1637,7 @@ def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles): - jobCmdLine = baseCmdLine + list(filter(None, [srcLanguage, srcFile])) + jobCmdLine = baseCmdLine + [srcLanguage + srcFile] jobs.append(executor.submit( processSingleSource, compiler, jobCmdLine, srcFile, objFile, environment)) From 0aac12bddade3d536130a84492dc6510bbc21517 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 16:10:34 -0500 Subject: [PATCH 11/27] TST: fix existing unittests --- unittests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests.py b/unittests.py index 05190df1..182fef09 100644 --- a/unittests.py +++ b/unittests.py @@ -505,7 +505,7 @@ def _testFailure(self, cmdLine, expectedExceptionClass): def _testFull(self, cmdLine, expectedSourceFiles, expectedOutputFile): sourceFiles, outputFile = CommandLineAnalyzer.analyze(cmdLine) - self.assertEqual(sourceFiles, expectedSourceFiles) + self.assertEqual([s for s, _ in sourceFiles], expectedSourceFiles) self.assertEqual(outputFile, expectedOutputFile) def _testFo(self, foArgument, expectedObjectFilepath): From ce9de5ef0f78cbc4b37325f6fe2d12eaa2f436f8 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 19:00:56 -0500 Subject: [PATCH 12/27] appveyor: define CLCACHE_LOG --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 9beb9350..a52ca512 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,6 +4,8 @@ branches: - master environment: + global: + CLCACHE_LOG: true matrix: # AppVeyor installed Python versions # http://www.appveyor.com/docs/installed-software#python From ec0fae12ebadf98a86049c8218a9b1a6a48ea268 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 19:13:35 -0500 Subject: [PATCH 13/27] TST: printTraceStatement: print caller line --- clcache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index 2dde7f8a..f5b3b713 100644 --- a/clcache.py +++ b/clcache.py @@ -24,6 +24,7 @@ import subprocess import sys import threading +import inspect from tempfile import TemporaryFile VERSION = "4.1.0-dev" @@ -996,9 +997,10 @@ def findCompilerBinary(): def printTraceStatement(msg): if "CLCACHE_LOG" in os.environ: + _, _, lineNumber, _, _, _ = inspect.getouterframes(inspect.currentframe())[1] scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: - print(os.path.join(scriptDir, "clcache.py") + " " + msg) + print(os.path.join(scriptDir, "clcache.py") + ":" + lineNumber + " " + msg) class CommandLineTokenizer(object): From 2e10786efb5140024eaba4454672783685178624 Mon Sep 17 00:00:00 2001 From: xoviat Date: Mon, 9 Oct 2017 19:17:03 -0500 Subject: [PATCH 14/27] BUG: fix printTraceStatement --- clcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index f5b3b713..44f57356 100644 --- a/clcache.py +++ b/clcache.py @@ -996,11 +996,12 @@ def findCompilerBinary(): def printTraceStatement(msg): + # type: (str) -> None if "CLCACHE_LOG" in os.environ: _, _, lineNumber, _, _, _ = inspect.getouterframes(inspect.currentframe())[1] scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: - print(os.path.join(scriptDir, "clcache.py") + ":" + lineNumber + " " + msg) + print(os.path.join(scriptDir, "clcache.py") + ":" + str(lineNumber) + " " + msg) class CommandLineTokenizer(object): From 019ae2ff4c8f8e4e23c5997123c207ced7e05e70 Mon Sep 17 00:00:00 2001 From: xoviat Date: Tue, 10 Oct 2017 12:36:32 -0500 Subject: [PATCH 15/27] FIX: switch to OrderedDict --- clcache.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/clcache.py b/clcache.py index 44f57356..cef9e083 100644 --- a/clcache.py +++ b/clcache.py @@ -9,6 +9,7 @@ from collections import defaultdict, namedtuple from ctypes import windll, wintypes from shutil import copyfile, rmtree +from collections import OrderedDict import cProfile import codecs import concurrent.futures @@ -1250,13 +1251,13 @@ def analyze(cmdline): options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) # Use an override pattern to shadow input files that have # already been specified in the function above - inputFiles = {inputFile : '' for inputFile in inputFiles} + inputFiles = OrderedDict((inputFile, '') for inputFile in inputFiles) compl = False if 'Tp' in options: - inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']}) + inputFiles.update(OrderedDict((inputFile, '/Tp') for inputFile in options['Tp'])) compl = True if 'Tc' in options: - inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']}) + inputFiles.update(OrderedDict((inputFile, '/Tc') for inputFile in options['Tc'])) compl = True # Now collect the inputFiles into the return format From 6c3a85501a298cc8dee96245682a8291313c35ad Mon Sep 17 00:00:00 2001 From: xoviat Date: Tue, 10 Oct 2017 13:14:04 -0500 Subject: [PATCH 16/27] FIX: move import --- clcache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clcache.py b/clcache.py index cef9e083..7cb54885 100644 --- a/clcache.py +++ b/clcache.py @@ -6,10 +6,9 @@ # full text of which is available in the accompanying LICENSE file at the # root directory of this project. # -from collections import defaultdict, namedtuple +from collections import defaultdict, namedtuple, OrderedDict from ctypes import windll, wintypes from shutil import copyfile, rmtree -from collections import OrderedDict import cProfile import codecs import concurrent.futures From 56be5e24c44b18851d693b18d2ae52433dd7fbe0 Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:15:07 -0500 Subject: [PATCH 17/27] FIX: check whether cmdLine arg startswith /MP --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index 7cb54885..3349b096 100644 --- a/clcache.py +++ b/clcache.py @@ -1633,7 +1633,7 @@ def filterSourceFiles(cmdLine, sourceFiles): def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int # Filter out all source files from the command line to form baseCmdLine - baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if arg != '/MP'] + baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if not arg.startswith('/MP')] exitCode = 0 cleanupRequired = False From 5de5fcc86341776580f3e3dd748ed230d237cbae Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:16:53 -0500 Subject: [PATCH 18/27] FIX: revert use of OrderedDict --- clcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clcache.py b/clcache.py index 3349b096..6b6e2b81 100644 --- a/clcache.py +++ b/clcache.py @@ -6,7 +6,7 @@ # full text of which is available in the accompanying LICENSE file at the # root directory of this project. # -from collections import defaultdict, namedtuple, OrderedDict +from collections import defaultdict, namedtuple from ctypes import windll, wintypes from shutil import copyfile, rmtree import cProfile @@ -1250,13 +1250,13 @@ def analyze(cmdline): options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) # Use an override pattern to shadow input files that have # already been specified in the function above - inputFiles = OrderedDict((inputFile, '') for inputFile in inputFiles) + inputFiles = {inputFile: '' for inputFile in inputFiles} compl = False if 'Tp' in options: - inputFiles.update(OrderedDict((inputFile, '/Tp') for inputFile in options['Tp'])) + inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']}) compl = True if 'Tc' in options: - inputFiles.update(OrderedDict((inputFile, '/Tc') for inputFile in options['Tc'])) + inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']}) compl = True # Now collect the inputFiles into the return format From 725410ebfa20d483bbe2a92ce4089f2a89e163fd Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:20:24 -0500 Subject: [PATCH 19/27] FIX: appveyor: revert CLCACHE_LOG --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a52ca512..9beb9350 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,8 +4,6 @@ branches: - master environment: - global: - CLCACHE_LOG: true matrix: # AppVeyor installed Python versions # http://www.appveyor.com/docs/installed-software#python From 19e7126e41ce6ea13331ca2efbb67ea4a99c71fd Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:22:40 -0500 Subject: [PATCH 20/27] TST: revert changes --- unittests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unittests.py b/unittests.py index 182fef09..c1e3340b 100644 --- a/unittests.py +++ b/unittests.py @@ -504,8 +504,9 @@ def _testFailure(self, cmdLine, expectedExceptionClass): self.assertRaises(expectedExceptionClass, lambda: CommandLineAnalyzer.analyze(cmdLine)) def _testFull(self, cmdLine, expectedSourceFiles, expectedOutputFile): + # type: (List[str], List[Tuple[str, str]], List[str]) -> None sourceFiles, outputFile = CommandLineAnalyzer.analyze(cmdLine) - self.assertEqual([s for s, _ in sourceFiles], expectedSourceFiles) + self.assertEqual(sourceFiles, expectedSourceFiles) self.assertEqual(outputFile, expectedOutputFile) def _testFo(self, foArgument, expectedObjectFilepath): From 9079f323f9c4bbfcbe5857035aa9a74bd5a9cc55 Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:27:06 -0500 Subject: [PATCH 21/27] TST: unit: correct comparisons The lists are modified to use the new tuple format --- unittests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unittests.py b/unittests.py index c1e3340b..a8a814b6 100644 --- a/unittests.py +++ b/unittests.py @@ -511,7 +511,7 @@ def _testFull(self, cmdLine, expectedSourceFiles, expectedOutputFile): def _testFo(self, foArgument, expectedObjectFilepath): self._testFull(['/c', foArgument, 'main.cpp'], - ["main.cpp"], [expectedObjectFilepath]) + [("main.cpp", '')], [expectedObjectFilepath]) def _testFi(self, fiArgument): self._testPreprocessingOutfile(['/c', '/P', fiArgument, 'main.cpp']) @@ -529,7 +529,7 @@ def testEmpty(self): self._testFailure([], NoSourceFileError) def testSimple(self): - self._testFull(["/c", "main.cpp"], ["main.cpp"], ["main.obj"]) + self._testFull(["/c", "main.cpp"], [("main.cpp", "")], ["main.obj"]) def testNoSource(self): # No source file has priority over other errors, for consistency @@ -548,7 +548,7 @@ def testNoSource(self): def testOutputFileFromSourcefile(self): # For object file self._testFull(['/c', 'main.cpp'], - ['main.cpp'], ['main.obj']) + [('main.cpp', '')], ['main.obj']) # For preprocessor file self._testFailure(['/c', '/P', 'main.cpp'], CalledForPreprocessingError) @@ -635,9 +635,9 @@ def testPreprocessingFi(self): def testTpTcSimple(self): # clcache can handle /Tc or /Tp as long as there is only one of them self._testFull(['/c', '/TcMyCcProgram.c'], - ['MyCcProgram.c'], ['MyCcProgram.obj']) + [('MyCcProgram.c', '/Tc')], ['MyCcProgram.obj']) self._testFull(['/c', '/TpMyCxxProgram.cpp'], - ['MyCxxProgram.cpp'], ['MyCxxProgram.obj']) + [('MyCxxProgram.cpp', '/Tp')], ['MyCxxProgram.obj']) def testLink(self): self._testFailure(["main.cpp"], CalledForLinkError) From 7fb44804a0ab792e1b3c60b7efde8d9576932612 Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 08:37:15 -0500 Subject: [PATCH 22/27] TST: integration: print output --- integrationtests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integrationtests.py b/integrationtests.py index 33400443..bcef1d45 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -806,6 +806,7 @@ def testHitsViaMpConcurrent(self): self.assertEqual(stats.numCacheEntries(), 2) def testOutput(self): + # type: () -> None with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: sources = glob.glob("*.cpp") clcache.Cache(tempDir) @@ -813,6 +814,8 @@ def testOutput(self): cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] mpFlag = "/MP" + str(len(sources)) out = subprocess.check_output(cmd + [mpFlag] + sources, env=customEnv).decode("ascii") + # print the output so that it shows up in py.test + print(out) for s in sources: self.assertEqual(out.count(s), 1) From 4a7ee82449c5415f4f03655328bd7d860469263f Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 10:15:46 -0500 Subject: [PATCH 23/27] TST: unit: add some reasonable tests for filterSourceFiles --- unittests.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/unittests.py b/unittests.py index a8a814b6..c55c23f3 100644 --- a/unittests.py +++ b/unittests.py @@ -488,6 +488,30 @@ def testNestedResponseFiles(self): ) +class TestFilterSourceFiles(unittest.TestCase): + def _assertFiltered(self, cmdLine, files, filteredCmdLine): + # type: (List[str], List[Tuple[str, str]]) -> List[str] + files = CommandLineAnalyzer.filterSourceFiles(cmdLine, files) + self.assertEqual(list(files), filteredCmdLine) + + def testAnalyzerFiltersSourceFilesFromCommandLine(self): + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '/Tp')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '/Tc')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', '/Tcmain.cpp'], [('main.cpp', '')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', '/Tcmain.cpp'], [('main.cpp', '-Tc')], + ['/c', '/EP', '/FoSome.obj']) + + class TestAnalyzeCommandLine(unittest.TestCase): def _testSourceFilesOk(self, cmdLine): try: From 34fcb5d9ec47449b2c7c5274d2f5e4b9e0b5acd1 Mon Sep 17 00:00:00 2001 From: xoviat Date: Wed, 11 Oct 2017 13:12:18 -0500 Subject: [PATCH 24/27] TST: unit: fix --- unittests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests.py b/unittests.py index c55c23f3..131ee60f 100644 --- a/unittests.py +++ b/unittests.py @@ -491,10 +491,10 @@ def testNestedResponseFiles(self): class TestFilterSourceFiles(unittest.TestCase): def _assertFiltered(self, cmdLine, files, filteredCmdLine): # type: (List[str], List[Tuple[str, str]]) -> List[str] - files = CommandLineAnalyzer.filterSourceFiles(cmdLine, files) + files = clcache.filterSourceFiles(cmdLine, files) self.assertEqual(list(files), filteredCmdLine) - def testAnalyzerFiltersSourceFilesFromCommandLine(self): + def testFilterSourceFiles(self): self._assertFiltered( ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '')], ['/c', '/EP', '/FoSome.obj']) From 9ebb89bc82febb6ecca196015080d536f02b8cb1 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 12 Oct 2017 08:52:03 -0500 Subject: [PATCH 25/27] FIX: revert changes to printTraceStatement --- clcache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clcache.py b/clcache.py index 6b6e2b81..99d0e768 100644 --- a/clcache.py +++ b/clcache.py @@ -998,10 +998,9 @@ def findCompilerBinary(): def printTraceStatement(msg): # type: (str) -> None if "CLCACHE_LOG" in os.environ: - _, _, lineNumber, _, _, _ = inspect.getouterframes(inspect.currentframe())[1] scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: - print(os.path.join(scriptDir, "clcache.py") + ":" + str(lineNumber) + " " + msg) + print(os.path.join(scriptDir, "clcache.py") + " " + msg) class CommandLineTokenizer(object): From 084b23b0aa67709f0399a0b5be09fe638d79c49d Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 12 Oct 2017 08:56:41 -0500 Subject: [PATCH 26/27] FIX: remove unused import --- clcache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clcache.py b/clcache.py index 99d0e768..cce142c1 100644 --- a/clcache.py +++ b/clcache.py @@ -24,7 +24,6 @@ import subprocess import sys import threading -import inspect from tempfile import TemporaryFile VERSION = "4.1.0-dev" From 275ae159c81185a4ba7b155f47083af59b0dab64 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 12 Oct 2017 12:37:23 -0500 Subject: [PATCH 27/27] FIX: update ??? to type: Any --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index cce142c1..ff9ee062 100644 --- a/clcache.py +++ b/clcache.py @@ -1629,7 +1629,7 @@ def filterSourceFiles(cmdLine, sourceFiles): ) def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): - # type: (???, str, List[str], ???, List[Tuple[str, str]], List[str]) -> int + # type: (Any, str, List[str], Any, List[Tuple[str, str]], List[str]) -> int # Filter out all source files from the command line to form baseCmdLine baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if not arg.startswith('/MP')]