From 46365cb09f168c8f51cc7514a115cf1dfda94efa Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 12:56:28 +0300 Subject: [PATCH 01/12] CM-40907 supports restoring npm and refactoring process of generating dependencies file --- .../sca/base_restore_dependencies.py | 13 +++++--- .../sca/maven/restore_gradle_dependencies.py | 6 +++- .../sca/maven/restore_maven_dependencies.py | 8 +++-- .../cli/files_collector/sca/npm/__init__.py | 0 .../sca/npm/restore_npm_dependencies.py | 33 +++++++++++++++++++ .../sca/nuget/restore_nuget_dependencies.py | 3 +- .../files_collector/sca/sca_code_scanner.py | 31 +++++++++-------- cycode/cli/utils/shell_executor.py | 9 ++++- 8 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 cycode/cli/files_collector/sca/npm/__init__.py create mode 100644 cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 78ba3fd4..a65c33cd 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -13,9 +13,9 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: return join_paths(get_file_dir(path), generated_file_name) -def execute_command(command: List[str], file_name: str, command_timeout: int) -> Optional[str]: +def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> Optional[str]: try: - dependencies = shell(command, command_timeout) + dependencies = shell(command=command, timeout=command_timeout, execute_in_shell=False, output_file_path=dependencies_file_name) except Exception as e: logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) return None @@ -24,10 +24,11 @@ def execute_command(command: List[str], file_name: str, command_timeout: int) -> class BaseRestoreDependencies(ABC): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool) -> None: self.context = context self.is_git_diff = is_git_diff self.command_timeout = command_timeout + self.create_output_file_manually = create_output_file_manually def restore(self, document: Document) -> Optional[Document]: return self.try_restore_dependencies(document) @@ -46,9 +47,11 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: if self.verify_restore_file_already_exist(restore_file_path): restore_file_content = get_file_content(restore_file_path) else: - restore_file_content = execute_command( - self.get_command(manifest_file_path), manifest_file_path, self.command_timeout + output_file_path = restore_file_path if self.create_output_file_manually else None + execute_command( + self.get_command(manifest_file_path), manifest_file_path, self.command_timeout, output_file_path ) + restore_file_content = get_file_content(restore_file_path) return Document(restore_file_path, restore_file_content, self.is_git_diff) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index d4896e95..f1d1743d 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -9,11 +9,12 @@ BUILD_GRADLE_FILE_NAME = 'build.gradle' BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' +OUTPUT_FILE_MANUALLY = True class RestoreGradleDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) @@ -26,3 +27,6 @@ def get_lock_file_name(self) -> str: def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) + + def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str: + return '/' + manifest_file_path.strip('/' + BUILD_GRADLE_FILE_NAME) + '/' + BUILD_GRADLE_DEP_TREE_FILE_NAME diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index 084dec6c..900d8a0b 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -1,3 +1,4 @@ +import os from os import path from typing import List, Optional @@ -14,11 +15,12 @@ BUILD_MAVEN_FILE_NAME = 'pom.xml' MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json' MAVEN_DEP_TREE_FILE_NAME = 'bcde.mvndeps' +OUTPUT_FILE_MANUALLY = False class RestoreMavenDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + super().__init__(context, is_git_diff, command_timeout, False) def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME @@ -30,7 +32,7 @@ def get_lock_file_name(self) -> str: return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return False + return os.path.isfile(restore_file_path) def try_restore_dependencies(self, document: Document) -> Optional[Document]: restore_dependencies_document = super().try_restore_dependencies(document) @@ -47,7 +49,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: return restore_dependencies_document def restore_from_secondary_command( - self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] + self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] ) -> Optional[Document]: # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg secondary_restore_command = create_secondary_restore_command(manifest_file_path) diff --git a/cycode/cli/files_collector/sca/npm/__init__.py b/cycode/cli/files_collector/sca/npm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py new file mode 100644 index 00000000..f59d5021 --- /dev/null +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -0,0 +1,33 @@ +import os +from typing import List + +import click + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +NPM_PROJECT_FILE_EXTENSIONS = ['.json'] +NPM_LOCK_FILE_NAME = 'package-lock.json' +NPM_MANIFEST_FILE_NAME = 'package.json' +OUTPUT_FILE_MANUALLY = False + + +class RestoreNpmDependencies(BaseRestoreDependencies): + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) + + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) + + def get_command(self, manifest_file_path: str) -> List[str]: + return ['npm', 'install', '--prefix', self.prepare_manifest_file_path_for_command(manifest_file_path), + '--package-lock-only', '--ignore-scripts', '--no-audit'] + + def get_lock_file_name(self) -> str: + return NPM_LOCK_FILE_NAME + + def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: + return os.path.isfile(restore_file_path) + + def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str: + return '/' + manifest_file_path.strip('/' + NPM_MANIFEST_FILE_NAME) diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index c54c3e5e..878e072b 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -8,11 +8,12 @@ NUGET_PROJECT_FILE_EXTENSIONS = ['.csproj', '.vbproj'] NUGET_LOCK_FILE_NAME = 'packages.lock.json' +OUTPUT_FILE_MANUALLY = False class RestoreNugetDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index d4cc0293..ae59f7ce 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -7,6 +7,7 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies +from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy @@ -18,14 +19,15 @@ BUILD_GRADLE_DEP_TREE_TIMEOUT = 180 BUILD_NUGET_DEP_TREE_TIMEOUT = 180 +BUILD_NPM_DEP_TREE_TIMEOUT = 180 def perform_pre_commit_range_scan_actions( - path: str, - from_commit_documents: List[Document], - from_commit_rev: str, - to_commit_documents: List[Document], - to_commit_rev: str, + path: str, + from_commit_documents: List[Document], + from_commit_rev: str, + to_commit_documents: List[Document], + to_commit_rev: str, ) -> None: repo = git_proxy.get_repo(path) add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) @@ -33,7 +35,7 @@ def perform_pre_commit_range_scan_actions( def perform_pre_hook_range_scan_actions( - git_head_documents: List[Document], pre_committed_documents: List[Document] + git_head_documents: List[Document], pre_committed_documents: List[Document] ) -> None: repo = git_proxy.get_repo(os.getcwd()) add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) @@ -41,7 +43,7 @@ def perform_pre_hook_range_scan_actions( def add_ecosystem_related_files_if_exists( - documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None + documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None ) -> None: documents_to_add: List[Document] = [] for doc in documents: @@ -56,7 +58,7 @@ def add_ecosystem_related_files_if_exists( def get_doc_ecosystem_related_project_files( - doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] + doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] ) -> List[Document]: documents_to_add: List[Document] = [] for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): @@ -86,10 +88,10 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: def try_restore_dependencies( - context: click.Context, - documents_to_add: Dict[str, Document], - restore_dependencies: 'BaseRestoreDependencies', - document: Document, + context: click.Context, + documents_to_add: Dict[str, Document], + restore_dependencies: 'BaseRestoreDependencies', + document: Document, ) -> None: if restore_dependencies.is_project(document): restore_dependencies_document = restore_dependencies.restore(document) @@ -115,7 +117,7 @@ def try_restore_dependencies( def add_dependencies_tree_document( - context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False + context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: documents_to_add: Dict[str, Document] = {} restore_dependencies_list = restore_handlers(context, is_git_diff) @@ -132,6 +134,7 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest RestoreGradleDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), RestoreNugetDependencies(context, is_git_diff, BUILD_NUGET_DEP_TREE_TIMEOUT), + RestoreNpmDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT) ] @@ -147,7 +150,7 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O def perform_pre_scan_documents_actions( - context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False + context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: if scan_type == consts.SCA_SCAN_TYPE and not context.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): logger.debug('Perform pre-scan document add_dependencies_tree_document action') diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index fb023451..92686b52 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -9,7 +9,8 @@ def shell( - command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, execute_in_shell: bool = False + command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, execute_in_shell: bool = False, + output_file_path: Optional[str] = None ) -> Optional[str]: logger.debug('Executing shell command: %s', command) @@ -20,8 +21,14 @@ def shell( shell=execute_in_shell, check=True, capture_output=True, + text=True, ) + # Write stdout output to the file if output_file_path is provided + if output_file_path: + with open(output_file_path, 'w') as output_file: + output_file.write(result.stdout) + return result.stdout.decode('UTF-8').strip() except subprocess.CalledProcessError as e: logger.debug('Error occurred while running shell command', exc_info=e) From 0b720e407d6234d5135323f9e6558e172f2ed076 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 13:09:52 +0300 Subject: [PATCH 02/12] CM-40907 fromatting --- .../cli/files_collector/sca/base_restore_dependencies.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index a65c33cd..5a82b740 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -13,9 +13,11 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: return join_paths(get_file_dir(path), generated_file_name) -def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> Optional[str]: +def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ +Optional[str]: try: - dependencies = shell(command=command, timeout=command_timeout, execute_in_shell=False, output_file_path=dependencies_file_name) + dependencies = shell(command=command, timeout=command_timeout, execute_in_shell=False, + output_file_path=dependencies_file_name) except Exception as e: logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) return None @@ -24,7 +26,8 @@ def execute_command(command: List[str], file_name: str, command_timeout: int, de class BaseRestoreDependencies(ABC): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool) -> None: + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, + create_output_file_manually: bool) -> None: self.context = context self.is_git_diff = is_git_diff self.command_timeout = command_timeout From a347a62c8c3b3b6bc9e3f3dbb1651b106b116771 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 13:44:32 +0300 Subject: [PATCH 03/12] CM-40907 set secured shell command --- cycode/cli/files_collector/sca/base_restore_dependencies.py | 3 +-- cycode/cli/utils/shell_executor.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 5a82b740..86f36ea0 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -16,8 +16,7 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ Optional[str]: try: - dependencies = shell(command=command, timeout=command_timeout, execute_in_shell=False, - output_file_path=dependencies_file_name) + dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name) except Exception as e: logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) return None diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 92686b52..8286fca1 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -9,7 +9,7 @@ def shell( - command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, execute_in_shell: bool = False, + command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, output_file_path: Optional[str] = None ) -> Optional[str]: logger.debug('Executing shell command: %s', command) @@ -18,7 +18,7 @@ def shell( result = subprocess.run( # noqa: S603 command, timeout=timeout, - shell=execute_in_shell, + shell=False, check=True, capture_output=True, text=True, From 277d78387175e56ac398e842236b78ba20d2f082 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 18:17:11 +0300 Subject: [PATCH 04/12] CM-40907 fix --- cycode/cli/files_collector/sca/base_restore_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 86f36ea0..0ae788c0 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -14,7 +14,7 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ -Optional[str]: +str: try: dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name) except Exception as e: From 98bf5492fc2f3325f74fc4a71b673de3ddaf4fc0 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 18:18:25 +0300 Subject: [PATCH 05/12] CM-40907 fix --- cycode/cli/files_collector/sca/base_restore_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 0ae788c0..86f36ea0 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -14,7 +14,7 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ -str: +Optional[str]: try: dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name) except Exception as e: From f173e502cac1711c1c67fb4997c5dd1b95b05df7 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 14 Oct 2024 18:25:38 +0300 Subject: [PATCH 06/12] CM-40909 init --- cycode/cli/files_collector/sca/go/__init__.py | 0 .../sca/go/restore_go_dependencies.py | 33 +++++++++++++++++++ .../files_collector/sca/sca_code_scanner.py | 6 ++-- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 cycode/cli/files_collector/sca/go/__init__.py create mode 100644 cycode/cli/files_collector/sca/go/restore_go_dependencies.py diff --git a/cycode/cli/files_collector/sca/go/__init__.py b/cycode/cli/files_collector/sca/go/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py new file mode 100644 index 00000000..15770d1b --- /dev/null +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -0,0 +1,33 @@ +import os +from typing import List + +import click + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +NPM_PROJECT_FILE_EXTENSIONS = ['.json'] +NPM_LOCK_FILE_NAME = 'package-lock.json' +NPM_MANIFEST_FILE_NAME = 'package.json' +OUTPUT_FILE_MANUALLY = False + + +class RestoreGoDependencies(BaseRestoreDependencies): + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) + + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) + + def get_command(self, manifest_file_path: str) -> List[str]: + return ['npm', 'install', '--prefix', self.prepare_manifest_file_path_for_command(manifest_file_path), + '--package-lock-only', '--ignore-scripts', '--no-audit'] + + def get_lock_file_name(self) -> str: + return NPM_LOCK_FILE_NAME + + def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: + return os.path.isfile(restore_file_path) + + def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str: + return '/' + manifest_file_path.strip('/' + NPM_MANIFEST_FILE_NAME) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index ae59f7ce..560357e4 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -5,6 +5,7 @@ from cycode.cli import consts from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies @@ -20,7 +21,7 @@ BUILD_GRADLE_DEP_TREE_TIMEOUT = 180 BUILD_NUGET_DEP_TREE_TIMEOUT = 180 BUILD_NPM_DEP_TREE_TIMEOUT = 180 - +BUILD_GO_DEP_TREE_TIMEOUT = 180 def perform_pre_commit_range_scan_actions( path: str, @@ -134,7 +135,8 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest RestoreGradleDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), RestoreNugetDependencies(context, is_git_diff, BUILD_NUGET_DEP_TREE_TIMEOUT), - RestoreNpmDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT) + RestoreNpmDependencies(context, is_git_diff, BUILD_NPM_DEP_TREE_TIMEOUT), + RestoreGoDependencies(context, is_git_diff, BUILD_GO_DEP_TREE_TIMEOUT) ] From 60e57777d94c129f3c83111c1f5d98903337a920 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 15 Oct 2024 14:23:40 +0300 Subject: [PATCH 07/12] CM-40909 add go restore --- .../sca/go/restore_go_dependencies.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 15770d1b..1ecfd9e5 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -6,28 +6,21 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document -NPM_PROJECT_FILE_EXTENSIONS = ['.json'] -NPM_LOCK_FILE_NAME = 'package-lock.json' -NPM_MANIFEST_FILE_NAME = 'package.json' -OUTPUT_FILE_MANUALLY = False - +NPM_PROJECT_FILE_EXTENSIONS = ['.mod'] class RestoreGoDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) + super().__init__(context, is_git_diff, command_timeout, True) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) def get_command(self, manifest_file_path: str) -> List[str]: - return ['npm', 'install', '--prefix', self.prepare_manifest_file_path_for_command(manifest_file_path), - '--package-lock-only', '--ignore-scripts', '--no-audit'] + return ['go', 'list', '-m', '-json', 'all'] def get_lock_file_name(self) -> str: - return NPM_LOCK_FILE_NAME + return None def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) - def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str: - return '/' + manifest_file_path.strip('/' + NPM_MANIFEST_FILE_NAME) From 0f5b002254d145df78d0ade80f3b675dd2e2e68b Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 15 Oct 2024 16:24:27 +0300 Subject: [PATCH 08/12] CM-40909 implement go restore --- .../sca/base_restore_dependencies.py | 2 +- .../sca/go/restore_go_dependencies.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 86f36ea0..0d258627 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -14,7 +14,7 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ -Optional[str]: + Optional[str]: try: dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name) except Exception as e: diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 1ecfd9e5..8cdb5dce 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -6,21 +6,26 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document -NPM_PROJECT_FILE_EXTENSIONS = ['.mod'] +GO_PROJECT_FILE_EXTENSIONS = ['.mod'] +GO_RESTORE_FILE_NAME = 'go.sum' +BUILD_GO_FILE_NAME = 'go.mod' + class RestoreGoDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: super().__init__(context, is_git_diff, command_timeout, True) def is_project(self, document: Document) -> bool: - return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) + return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) def get_command(self, manifest_file_path: str) -> List[str]: - return ['go', 'list', '-m', '-json', 'all'] + return ['cd', self.prepare_tree_file_path_for_command(manifest_file_path), '&&', 'go', 'list', '-m', '-json'] def get_lock_file_name(self) -> str: - return None + return GO_RESTORE_FILE_NAME def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) + def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str: + return manifest_file_path.replace('/' + BUILD_GO_FILE_NAME, '') From dde8b9ae0404ce2869d471ad76a4fd60fd15eb45 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Mon, 21 Oct 2024 22:05:14 +0300 Subject: [PATCH 09/12] CM-40909 format --- .../sca/base_restore_dependencies.py | 10 +++--- .../sca/maven/restore_maven_dependencies.py | 2 +- .../sca/npm/restore_npm_dependencies.py | 11 +++++-- .../files_collector/sca/sca_code_scanner.py | 31 ++++++++++--------- cycode/cli/utils/shell_executor.py | 5 +-- 5 files changed, 35 insertions(+), 24 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 0d258627..5b43fa69 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -13,8 +13,9 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: return join_paths(get_file_dir(path), generated_file_name) -def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \ - Optional[str]: +def execute_command( + command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None +) -> Optional[str]: try: dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name) except Exception as e: @@ -25,8 +26,9 @@ def execute_command(command: List[str], file_name: str, command_timeout: int, de class BaseRestoreDependencies(ABC): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, - create_output_file_manually: bool) -> None: + def __init__( + self, context: click.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool + ) -> None: self.context = context self.is_git_diff = is_git_diff self.command_timeout = command_timeout diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index 900d8a0b..9f62b98d 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -49,7 +49,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: return restore_dependencies_document def restore_from_secondary_command( - self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] + self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] ) -> Optional[Document]: # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg secondary_restore_command = create_secondary_restore_command(manifest_file_path) diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index f59d5021..089f768e 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -20,8 +20,15 @@ def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) def get_command(self, manifest_file_path: str) -> List[str]: - return ['npm', 'install', '--prefix', self.prepare_manifest_file_path_for_command(manifest_file_path), - '--package-lock-only', '--ignore-scripts', '--no-audit'] + return [ + 'npm', + 'install', + '--prefix', + self.prepare_manifest_file_path_for_command(manifest_file_path), + '--package-lock-only', + '--ignore-scripts', + '--no-audit', + ] def get_lock_file_name(self) -> str: return NPM_LOCK_FILE_NAME diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index 560357e4..7554b5df 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -23,12 +23,13 @@ BUILD_NPM_DEP_TREE_TIMEOUT = 180 BUILD_GO_DEP_TREE_TIMEOUT = 180 + def perform_pre_commit_range_scan_actions( - path: str, - from_commit_documents: List[Document], - from_commit_rev: str, - to_commit_documents: List[Document], - to_commit_rev: str, + path: str, + from_commit_documents: List[Document], + from_commit_rev: str, + to_commit_documents: List[Document], + to_commit_rev: str, ) -> None: repo = git_proxy.get_repo(path) add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) @@ -36,7 +37,7 @@ def perform_pre_commit_range_scan_actions( def perform_pre_hook_range_scan_actions( - git_head_documents: List[Document], pre_committed_documents: List[Document] + git_head_documents: List[Document], pre_committed_documents: List[Document] ) -> None: repo = git_proxy.get_repo(os.getcwd()) add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) @@ -44,7 +45,7 @@ def perform_pre_hook_range_scan_actions( def add_ecosystem_related_files_if_exists( - documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None + documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None ) -> None: documents_to_add: List[Document] = [] for doc in documents: @@ -59,7 +60,7 @@ def add_ecosystem_related_files_if_exists( def get_doc_ecosystem_related_project_files( - doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] + doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] ) -> List[Document]: documents_to_add: List[Document] = [] for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): @@ -89,10 +90,10 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: def try_restore_dependencies( - context: click.Context, - documents_to_add: Dict[str, Document], - restore_dependencies: 'BaseRestoreDependencies', - document: Document, + context: click.Context, + documents_to_add: Dict[str, Document], + restore_dependencies: 'BaseRestoreDependencies', + document: Document, ) -> None: if restore_dependencies.is_project(document): restore_dependencies_document = restore_dependencies.restore(document) @@ -118,7 +119,7 @@ def try_restore_dependencies( def add_dependencies_tree_document( - context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False + context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: documents_to_add: Dict[str, Document] = {} restore_dependencies_list = restore_handlers(context, is_git_diff) @@ -136,7 +137,7 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), RestoreNugetDependencies(context, is_git_diff, BUILD_NUGET_DEP_TREE_TIMEOUT), RestoreNpmDependencies(context, is_git_diff, BUILD_NPM_DEP_TREE_TIMEOUT), - RestoreGoDependencies(context, is_git_diff, BUILD_GO_DEP_TREE_TIMEOUT) + RestoreGoDependencies(context, is_git_diff, BUILD_GO_DEP_TREE_TIMEOUT), ] @@ -152,7 +153,7 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O def perform_pre_scan_documents_actions( - context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False + context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: if scan_type == consts.SCA_SCAN_TYPE and not context.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): logger.debug('Perform pre-scan document add_dependencies_tree_document action') diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 8286fca1..f527bead 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -9,8 +9,9 @@ def shell( - command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, - output_file_path: Optional[str] = None + command: Union[str, List[str]], + timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, + output_file_path: Optional[str] = None, ) -> Optional[str]: logger.debug('Executing shell command: %s', command) From 51a4101b849a97dc5abe1e88403ab9af003949a2 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 22 Oct 2024 13:16:58 +0300 Subject: [PATCH 10/12] CM-40909 fix --- .../files_collector/sca/nuget/restore_nuget_dependencies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index 878e072b..c54c3e5e 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -8,12 +8,11 @@ NUGET_PROJECT_FILE_EXTENSIONS = ['.csproj', '.vbproj'] NUGET_LOCK_FILE_NAME = 'packages.lock.json' -OUTPUT_FILE_MANUALLY = False class RestoreNugetDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY) + super().__init__(context, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) From fadf6f94a95483495b8a39525ff9843a351104d9 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 22 Oct 2024 13:17:45 +0300 Subject: [PATCH 11/12] CM-40909 fix --- .../files_collector/sca/maven/restore_maven_dependencies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index 9f62b98d..84732021 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -15,12 +15,11 @@ BUILD_MAVEN_FILE_NAME = 'pom.xml' MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json' MAVEN_DEP_TREE_FILE_NAME = 'bcde.mvndeps' -OUTPUT_FILE_MANUALLY = False class RestoreMavenDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, False) + super().__init__(context, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME From ac5c73ce8743d3dc1e4b79aa2a2f4582c13e6cfa Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 22 Oct 2024 13:25:22 +0300 Subject: [PATCH 12/12] CM-40909 fix --- cycode/cli/files_collector/sca/go/restore_go_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 8cdb5dce..512af8a5 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -13,7 +13,7 @@ class RestoreGoDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, True) + super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) @@ -28,4 +28,4 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str: - return manifest_file_path.replace('/' + BUILD_GO_FILE_NAME, '') + return manifest_file_path.replace(os.sep + BUILD_GO_FILE_NAME, '')