From a58a70af1d6c01c03e032ffb396a980c4fd3d8b1 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 17 Jul 2023 17:33:17 +0200 Subject: [PATCH 01/12] Add "merge --input-file-list NAME ..." as a way to exceed CLI limits Signed-off-by: Jim Klimov --- README.md | 6 ++++++ src/cyclonedx/Commands/MergeCommand.cs | 10 +++++++++- src/cyclonedx/Commands/MergeCommandOptions.cs | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd59de2..9e6f4c7 100755 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ Usage: cyclonedx merge [options] Options: + --input-file-list A single text file with input BOM filenames (one per line). --input-files Input BOM filenames (separate filenames with a space). --output-file Output BOM filename, will write to stdout if no value provided. --input-format Specify input file format. @@ -205,6 +206,11 @@ Options: Note: To perform a hierarchical merge all BOMs need the subject of the BOM described in the metadata component element. +The `--input-file-list` option can be useful if you have so many filenames to +merge that your shell interpreter command-line limit is exceeded if you list +them all as `--input-files`, or if your path names have spaces. If you specify +both options, the effective file lists will be concatenated before merge. + ### Examples Merge two XML formatted BOMs: diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 5c150d5..a88b011 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -22,6 +22,7 @@ using System.Threading.Tasks; using CycloneDX.Models; using CycloneDX.Utils; +using System.IO; namespace CycloneDX.Cli.Commands { @@ -31,6 +32,8 @@ public static void Configure(RootCommand rootCommand) { Contract.Requires(rootCommand != null); var subCommand = new Command("merge", "Merge two or more BOMs"); + subCommand.Add(new Option("--input-file-list", "A single text file with input BOM filenames (one per line).")); + //TBD//subCommand.Add(new Option("--input-file-list0", "A single text file with input BOM filenames (separated by 0x00 characters).")); subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option("--output-file", "Output BOM filename, will write to stdout if no value provided.")); subCommand.Add(new Option("--input-format", "Specify input file format.")); @@ -61,7 +64,12 @@ public static async Task Merge(MergeCommandOptions options) return (int)ExitCode.ParameterValidationError; } - var inputBoms = await InputBoms(options.InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false); + List InputFiles = (List)options.InputFiles; + if (options.InputFilesList != null) + { + InputFiles.AddRange(File.ReadAllLines(options.InputFilesList)); + } + var inputBoms = await InputBoms(InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false); Component bomSubject = null; if (options.Group != null || options.Name != null || options.Version != null) diff --git a/src/cyclonedx/Commands/MergeCommandOptions.cs b/src/cyclonedx/Commands/MergeCommandOptions.cs index f3078c4..71df950 100644 --- a/src/cyclonedx/Commands/MergeCommandOptions.cs +++ b/src/cyclonedx/Commands/MergeCommandOptions.cs @@ -20,6 +20,8 @@ namespace CycloneDX.Cli.Commands { public class MergeCommandOptions { + public string InputFilesList { get; set; } + //TBD//public string InputFilesList0 { get; set; } public IList InputFiles { get; set; } public string OutputFile { get; set; } public CycloneDXBomFormat InputFormat { get; set; } @@ -29,4 +31,4 @@ public class MergeCommandOptions public string Name { get; set; } public string Version { get; set; } } -} \ No newline at end of file +} From f869610b879bf12300506a406fbe51bed8e10d13 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:06:20 +0200 Subject: [PATCH 02/12] MergeCommand: Convert singular --input-file-list to potentially multiple --input-files-list Signed-off-by: Jim Klimov --- README.md | 4 ++-- src/cyclonedx/Commands/MergeCommand.cs | 12 +++++++++--- src/cyclonedx/Commands/MergeCommandOptions.cs | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9e6f4c7..432af61 100755 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ Usage: cyclonedx merge [options] Options: - --input-file-list A single text file with input BOM filenames (one per line). + --input-files-list One or more text file(s) with input BOM filenames (one per line). --input-files Input BOM filenames (separate filenames with a space). --output-file Output BOM filename, will write to stdout if no value provided. --input-format Specify input file format. @@ -206,7 +206,7 @@ Options: Note: To perform a hierarchical merge all BOMs need the subject of the BOM described in the metadata component element. -The `--input-file-list` option can be useful if you have so many filenames to +The `--input-files-list` option can be useful if you have so many filenames to merge that your shell interpreter command-line limit is exceeded if you list them all as `--input-files`, or if your path names have spaces. If you specify both options, the effective file lists will be concatenated before merge. diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index a88b011..37bf5dd 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -32,8 +32,8 @@ public static void Configure(RootCommand rootCommand) { Contract.Requires(rootCommand != null); var subCommand = new Command("merge", "Merge two or more BOMs"); - subCommand.Add(new Option("--input-file-list", "A single text file with input BOM filenames (one per line).")); - //TBD//subCommand.Add(new Option("--input-file-list0", "A single text file with input BOM filenames (separated by 0x00 characters).")); + subCommand.Add(new Option>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line).")); + //TBD//subCommand.Add(new Option>("--input-files-list0", "One or more text file(s) with input BOM filenames (separated by 0x00 characters).")); subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option("--output-file", "Output BOM filename, will write to stdout if no value provided.")); subCommand.Add(new Option("--input-format", "Specify input file format.")); @@ -67,8 +67,14 @@ public static async Task Merge(MergeCommandOptions options) List InputFiles = (List)options.InputFiles; if (options.InputFilesList != null) { - InputFiles.AddRange(File.ReadAllLines(options.InputFilesList)); + ((List)options.InputFilesList).ForEach(OneInputFileList => { + Console.WriteLine($"Adding to input file list from " + OneInputFileList); + InputFiles.AddRange(File.ReadAllLines(OneInputFileList)); + }); } + // TODO: Consider InputFiles.Distinct().ToList() - + // but that requires C# 3.0 for extension method support, + // and .NET 3.5 to get the LINQ Enumerable class. var inputBoms = await InputBoms(InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false); Component bomSubject = null; diff --git a/src/cyclonedx/Commands/MergeCommandOptions.cs b/src/cyclonedx/Commands/MergeCommandOptions.cs index 71df950..0b5fa33 100644 --- a/src/cyclonedx/Commands/MergeCommandOptions.cs +++ b/src/cyclonedx/Commands/MergeCommandOptions.cs @@ -20,8 +20,8 @@ namespace CycloneDX.Cli.Commands { public class MergeCommandOptions { - public string InputFilesList { get; set; } - //TBD//public string InputFilesList0 { get; set; } + public IList InputFilesList { get; set; } + //TBD//public IList InputFilesList0 { get; set; } public IList InputFiles { get; set; } public string OutputFile { get; set; } public CycloneDXBomFormat InputFormat { get; set; } From d1aa106e33caf735a42de11e0faffabdfa4a7c33 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:07:22 +0200 Subject: [PATCH 03/12] MergeCommand: Convert singular --input-file-list to potentially multiple --input-files-list: use simple foreach() Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 37bf5dd..0513c3b 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -67,10 +67,10 @@ public static async Task Merge(MergeCommandOptions options) List InputFiles = (List)options.InputFiles; if (options.InputFilesList != null) { - ((List)options.InputFilesList).ForEach(OneInputFileList => { + foreach (string OneInputFileList in options.InputFilesList) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); InputFiles.AddRange(File.ReadAllLines(OneInputFileList)); - }); + }; } // TODO: Consider InputFiles.Distinct().ToList() - // but that requires C# 3.0 for extension method support, From 3953595591c947768316519184588e08d132abb9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:29:58 +0200 Subject: [PATCH 04/12] MergeCommand: input-files-list: fix iterations and sanity checks Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 0513c3b..f9127d1 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -23,6 +23,7 @@ using CycloneDX.Models; using CycloneDX.Utils; using System.IO; +using System.Collections.Immutable; namespace CycloneDX.Cli.Commands { @@ -64,12 +65,25 @@ public static async Task Merge(MergeCommandOptions options) return (int)ExitCode.ParameterValidationError; } - List InputFiles = (List)options.InputFiles; + List InputFiles; + if (options.InputFiles != null) { + InputFiles = (List)options.InputFiles; + } else { + InputFiles = new List(); + } + + Console.WriteLine($"Got " + InputFiles.Count + " individual input file name(s): ['" + string.Join("', '", InputFiles) + "']"); if (options.InputFilesList != null) { - foreach (string OneInputFileList in options.InputFilesList) { + // For some reason, without an immutable list this claims + // modifications of the iterable during iteration and fails: + ImmutableList InputFilesList = options.InputFilesList.ToImmutableList(); + Console.WriteLine($"Got " + InputFilesList.Count + " file(s) with actual input file names: ['" + string.Join("', '", InputFilesList) + "']"); + foreach (string OneInputFileList in InputFilesList) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); - InputFiles.AddRange(File.ReadAllLines(OneInputFileList)); + string[] lines = File.ReadAllLines(OneInputFileList); + InputFiles.AddRange(lines); + Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); }; } // TODO: Consider InputFiles.Distinct().ToList() - From 56c2493e6036ba53f6e84f697146466bebfcf190 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:30:39 +0200 Subject: [PATCH 05/12] MergeCommand: in options handle "--input-files" before "--input-files-list" (dotnet is too smart for its own good) Signed-off-by: Jim Klimov --- README.md | 2 +- src/cyclonedx/Commands/MergeCommand.cs | 2 +- src/cyclonedx/Commands/MergeCommandOptions.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 432af61..0650cce 100755 --- a/README.md +++ b/README.md @@ -192,8 +192,8 @@ Usage: cyclonedx merge [options] Options: - --input-files-list One or more text file(s) with input BOM filenames (one per line). --input-files Input BOM filenames (separate filenames with a space). + --input-files-list One or more text file(s) with input BOM filenames (one per line). --output-file Output BOM filename, will write to stdout if no value provided. --input-format Specify input file format. --output-format Specify output file format. diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index f9127d1..32e1e83 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -33,9 +33,9 @@ public static void Configure(RootCommand rootCommand) { Contract.Requires(rootCommand != null); var subCommand = new Command("merge", "Merge two or more BOMs"); + subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line).")); //TBD//subCommand.Add(new Option>("--input-files-list0", "One or more text file(s) with input BOM filenames (separated by 0x00 characters).")); - subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option("--output-file", "Output BOM filename, will write to stdout if no value provided.")); subCommand.Add(new Option("--input-format", "Specify input file format.")); subCommand.Add(new Option("--output-format", "Specify output file format.")); diff --git a/src/cyclonedx/Commands/MergeCommandOptions.cs b/src/cyclonedx/Commands/MergeCommandOptions.cs index 0b5fa33..22e2fa4 100644 --- a/src/cyclonedx/Commands/MergeCommandOptions.cs +++ b/src/cyclonedx/Commands/MergeCommandOptions.cs @@ -20,9 +20,9 @@ namespace CycloneDX.Cli.Commands { public class MergeCommandOptions { + public IList InputFiles { get; set; } public IList InputFilesList { get; set; } //TBD//public IList InputFilesList0 { get; set; } - public IList InputFiles { get; set; } public string OutputFile { get; set; } public CycloneDXBomFormat InputFormat { get; set; } public CycloneDXBomFormat OutputFormat { get; set; } From fdac250f75b012068dbba56daad5cbe8a0cad1dd Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:36:38 +0200 Subject: [PATCH 06/12] MergeCommand: behave like before if no input-files* args were specified Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 32e1e83..5731909 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -84,11 +84,16 @@ public static async Task Merge(MergeCommandOptions options) string[] lines = File.ReadAllLines(OneInputFileList); InputFiles.AddRange(lines); Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); - }; + } } // TODO: Consider InputFiles.Distinct().ToList() - // but that requires C# 3.0 for extension method support, // and .NET 3.5 to get the LINQ Enumerable class. + if (InputFiles.Count == 0) { + // Revert to legacy (error-handling) behavior below + // in case the parameter was not passed + InputFiles = null; + } var inputBoms = await InputBoms(InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false); Component bomSubject = null; From 1cc6622e2a11464f3e4ce61e44ccde2dfaeae80b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 12:51:21 +0200 Subject: [PATCH 07/12] MergeCommand: implement support for --input-files-list0 Signed-off-by: Jim Klimov --- README.md | 17 +++++++++++++++-- src/cyclonedx/Commands/MergeCommand.cs | 15 ++++++++++++++- src/cyclonedx/Commands/MergeCommandOptions.cs | 2 +- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0650cce..4b5dac7 100755 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ Usage: Options: --input-files Input BOM filenames (separate filenames with a space). --input-files-list One or more text file(s) with input BOM filenames (one per line). + --input-files-list0 One or more text-like file(s) with input BOM filenames (separated by 0x00 characters). --output-file Output BOM filename, will write to stdout if no value provided. --input-format Specify input file format. --output-format Specify output file format. @@ -208,8 +209,20 @@ described in the metadata component element. The `--input-files-list` option can be useful if you have so many filenames to merge that your shell interpreter command-line limit is exceeded if you list -them all as `--input-files`, or if your path names have spaces. If you specify -both options, the effective file lists will be concatenated before merge. +them all as `--input-files`, or if your path names have spaces. + +The related `--input-files-list0` is intended for lists prepared by commands +like `find ... -print0` and makes sense on filesystems where carriage-return +and/or line-feed characters may validly be present in a path name component. +Note: behavior with multi-byte encodings (Unicode family) where a 0x00 byte +can be part of a character may be undefined. + +If you specify several of these options, the effective file lists will be +concatenated before the actual merge (first the individual `--input-files`, +then the contents of `--input-files-list`, and finally the contents of +`--input-files-list0`). If you have a document crafted to describe the root +of your product hierarchy tree, it is recommended to list it as the first +of individual `--input-files` (or otherwise on first line among used lists). ### Examples diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 5731909..af058bc 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -35,7 +35,7 @@ public static void Configure(RootCommand rootCommand) var subCommand = new Command("merge", "Merge two or more BOMs"); subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line).")); - //TBD//subCommand.Add(new Option>("--input-files-list0", "One or more text file(s) with input BOM filenames (separated by 0x00 characters).")); + subCommand.Add(new Option>("--input-files-list0", "One or more text-like file(s) with input BOM filenames (separated by 0x00 characters).")); subCommand.Add(new Option("--output-file", "Output BOM filename, will write to stdout if no value provided.")); subCommand.Add(new Option("--input-format", "Specify input file format.")); subCommand.Add(new Option("--output-format", "Specify output file format.")); @@ -86,6 +86,19 @@ public static async Task Merge(MergeCommandOptions options) Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); } } + + if (options.InputFilesList0 != null) + { + ImmutableList InputFilesList0 = options.InputFilesList0.ToImmutableList(); + Console.WriteLine($"Got " + InputFilesList0.Count + " file(s) with NUL-separated actual input file names: ['" + string.Join("', '", InputFilesList0) + "']"); + foreach (string OneInputFileList in InputFilesList0) { + Console.WriteLine($"Adding to input file list from " + OneInputFileList); + string[] lines = File.ReadAllText(OneInputFileList).Split('\0'); + InputFiles.AddRange(lines); + Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); + } + } + // TODO: Consider InputFiles.Distinct().ToList() - // but that requires C# 3.0 for extension method support, // and .NET 3.5 to get the LINQ Enumerable class. diff --git a/src/cyclonedx/Commands/MergeCommandOptions.cs b/src/cyclonedx/Commands/MergeCommandOptions.cs index 22e2fa4..b224cb9 100644 --- a/src/cyclonedx/Commands/MergeCommandOptions.cs +++ b/src/cyclonedx/Commands/MergeCommandOptions.cs @@ -22,7 +22,7 @@ public class MergeCommandOptions { public IList InputFiles { get; set; } public IList InputFilesList { get; set; } - //TBD//public IList InputFilesList0 { get; set; } + public IList InputFilesList0 { get; set; } public string OutputFile { get; set; } public CycloneDXBomFormat InputFormat { get; set; } public CycloneDXBomFormat OutputFormat { get; set; } From 064715e0f122dfd1da97cca3d3a9764a1c31adc7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 13:36:28 +0200 Subject: [PATCH 08/12] MergeCommand: align coding style of changes for InputFilesList with existing precedent Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index af058bc..3173a06 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -66,9 +66,12 @@ public static async Task Merge(MergeCommandOptions options) } List InputFiles; - if (options.InputFiles != null) { + if (options.InputFiles != null) + { InputFiles = (List)options.InputFiles; - } else { + } + else + { InputFiles = new List(); } @@ -79,7 +82,8 @@ public static async Task Merge(MergeCommandOptions options) // modifications of the iterable during iteration and fails: ImmutableList InputFilesList = options.InputFilesList.ToImmutableList(); Console.WriteLine($"Got " + InputFilesList.Count + " file(s) with actual input file names: ['" + string.Join("', '", InputFilesList) + "']"); - foreach (string OneInputFileList in InputFilesList) { + foreach (string OneInputFileList in InputFilesList) + { Console.WriteLine($"Adding to input file list from " + OneInputFileList); string[] lines = File.ReadAllLines(OneInputFileList); InputFiles.AddRange(lines); @@ -91,7 +95,8 @@ public static async Task Merge(MergeCommandOptions options) { ImmutableList InputFilesList0 = options.InputFilesList0.ToImmutableList(); Console.WriteLine($"Got " + InputFilesList0.Count + " file(s) with NUL-separated actual input file names: ['" + string.Join("', '", InputFilesList0) + "']"); - foreach (string OneInputFileList in InputFilesList0) { + foreach (string OneInputFileList in InputFilesList0) + { Console.WriteLine($"Adding to input file list from " + OneInputFileList); string[] lines = File.ReadAllText(OneInputFileList).Split('\0'); InputFiles.AddRange(lines); @@ -102,7 +107,8 @@ public static async Task Merge(MergeCommandOptions options) // TODO: Consider InputFiles.Distinct().ToList() - // but that requires C# 3.0 for extension method support, // and .NET 3.5 to get the LINQ Enumerable class. - if (InputFiles.Count == 0) { + if (InputFiles.Count == 0) + { // Revert to legacy (error-handling) behavior below // in case the parameter was not passed InputFiles = null; From 89499f51812aa75001de85f9de45f8e4d4d4e022 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 14:08:56 +0200 Subject: [PATCH 09/12] MergeCommand: rename --input-files-list0 to --input-files-nul-list (work around stupid Options parser) Signed-off-by: Jim Klimov --- README.md | 11 ++++++----- src/cyclonedx/Commands/MergeCommand.cs | 10 +++++----- src/cyclonedx/Commands/MergeCommandOptions.cs | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4b5dac7..d1b571c 100755 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ Usage: Options: --input-files Input BOM filenames (separate filenames with a space). --input-files-list One or more text file(s) with input BOM filenames (one per line). - --input-files-list0 One or more text-like file(s) with input BOM filenames (separated by 0x00 characters). + --input-files-nul-list One or more text-like file(s) with input BOM filenames (separated by 0x00 characters). --output-file Output BOM filename, will write to stdout if no value provided. --input-format Specify input file format. --output-format Specify output file format. @@ -211,7 +211,7 @@ The `--input-files-list` option can be useful if you have so many filenames to merge that your shell interpreter command-line limit is exceeded if you list them all as `--input-files`, or if your path names have spaces. -The related `--input-files-list0` is intended for lists prepared by commands +The related `--input-files-nul-list` is intended for lists prepared by commands like `find ... -print0` and makes sense on filesystems where carriage-return and/or line-feed characters may validly be present in a path name component. Note: behavior with multi-byte encodings (Unicode family) where a 0x00 byte @@ -220,9 +220,10 @@ can be part of a character may be undefined. If you specify several of these options, the effective file lists will be concatenated before the actual merge (first the individual `--input-files`, then the contents of `--input-files-list`, and finally the contents of -`--input-files-list0`). If you have a document crafted to describe the root -of your product hierarchy tree, it is recommended to list it as the first -of individual `--input-files` (or otherwise on first line among used lists). +`--input-files-nul-list`). If you have a document crafted to describe the +root of your product hierarchy tree, it is recommended to list it as the +first of individual `--input-files` (or otherwise on first line among used +lists). ### Examples diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 3173a06..ba96777 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -35,7 +35,7 @@ public static void Configure(RootCommand rootCommand) var subCommand = new Command("merge", "Merge two or more BOMs"); subCommand.Add(new Option>("--input-files", "Input BOM filenames (separate filenames with a space).")); subCommand.Add(new Option>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line).")); - subCommand.Add(new Option>("--input-files-list0", "One or more text-like file(s) with input BOM filenames (separated by 0x00 characters).")); + subCommand.Add(new Option>("--input-files-nul-list", "One or more text-like file(s) with input BOM filenames (separated by 0x00 characters).")); subCommand.Add(new Option("--output-file", "Output BOM filename, will write to stdout if no value provided.")); subCommand.Add(new Option("--input-format", "Specify input file format.")); subCommand.Add(new Option("--output-format", "Specify output file format.")); @@ -91,11 +91,11 @@ public static async Task Merge(MergeCommandOptions options) } } - if (options.InputFilesList0 != null) + if (options.InputFilesNulList != null) { - ImmutableList InputFilesList0 = options.InputFilesList0.ToImmutableList(); - Console.WriteLine($"Got " + InputFilesList0.Count + " file(s) with NUL-separated actual input file names: ['" + string.Join("', '", InputFilesList0) + "']"); - foreach (string OneInputFileList in InputFilesList0) + ImmutableList InputFilesNulList = options.InputFilesNulList.ToImmutableList(); + Console.WriteLine($"Got " + InputFilesNulList.Count + " file(s) with NUL-separated actual input file names: ['" + string.Join("', '", InputFilesNulList) + "']"); + foreach (string OneInputFileList in InputFilesNulList) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); string[] lines = File.ReadAllText(OneInputFileList).Split('\0'); diff --git a/src/cyclonedx/Commands/MergeCommandOptions.cs b/src/cyclonedx/Commands/MergeCommandOptions.cs index b224cb9..b2c847c 100644 --- a/src/cyclonedx/Commands/MergeCommandOptions.cs +++ b/src/cyclonedx/Commands/MergeCommandOptions.cs @@ -22,7 +22,7 @@ public class MergeCommandOptions { public IList InputFiles { get; set; } public IList InputFilesList { get; set; } - public IList InputFilesList0 { get; set; } + public IList InputFilesNulList { get; set; } public string OutputFile { get; set; } public CycloneDXBomFormat InputFormat { get; set; } public CycloneDXBomFormat OutputFormat { get; set; } From bbd41e49a35cabcc20eb0cdb940bc432d5e3c8a6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 20:34:45 +0200 Subject: [PATCH 10/12] MergeCommand: when handling --input-file-(nul-)list, avoid adding duplicates and empty lines, if any Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index ba96777..0601541 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -86,8 +86,15 @@ public static async Task Merge(MergeCommandOptions options) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); string[] lines = File.ReadAllLines(OneInputFileList); - InputFiles.AddRange(lines); - Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); + int count = 0; + foreach (string line in lines) + { + if (string.IsNullOrEmpty(line)) continue; + if (InputFiles.Contains(line)) continue; + InputFiles.Add(line); + count++; + } + Console.WriteLine($"Got " + count + " new entries from " + OneInputFileList); } } @@ -99,14 +106,18 @@ public static async Task Merge(MergeCommandOptions options) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); string[] lines = File.ReadAllText(OneInputFileList).Split('\0'); - InputFiles.AddRange(lines); - Console.WriteLine($"Got " + lines.Length + " entries from " + OneInputFileList); + int count = 0; + foreach (string line in lines) + { + if (string.IsNullOrEmpty(line)) continue; + if (InputFiles.Contains(line)) continue; + InputFiles.Add(line); + count++; + } + Console.WriteLine($"Got " + count + " new entries from " + OneInputFileList); } } - // TODO: Consider InputFiles.Distinct().ToList() - - // but that requires C# 3.0 for extension method support, - // and .NET 3.5 to get the LINQ Enumerable class. if (InputFiles.Count == 0) { // Revert to legacy (error-handling) behavior below From 4017eb9c1ca0aae7b113b886718aae98b46b1ce8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 20:43:03 +0200 Subject: [PATCH 11/12] MergeCommand: refactor DetermineInputFiles() into a separate method Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 104 +++++++++++++------------ 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 0601541..2231088 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -65,6 +65,60 @@ public static async Task Merge(MergeCommandOptions options) return (int)ExitCode.ParameterValidationError; } + var inputBoms = await InputBoms(DetermineInputFiles(options), options.InputFormat, outputToConsole).ConfigureAwait(false); + + Component bomSubject = null; + if (options.Group != null || options.Name != null || options.Version != null) + bomSubject = new Component + { + Type = Component.Classification.Application, + Group = options.Group, + Name = options.Name, + Version = options.Version, + }; + + Bom outputBom; + if (options.Hierarchical) + { + outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject); + } + else + { + outputBom = CycloneDXUtils.FlatMerge(inputBoms); + if (outputBom.Metadata is null) outputBom.Metadata = new Metadata(); + if (bomSubject != null) + { + // use the params provided if possible + outputBom.Metadata.Component = bomSubject; + } + else + { + // otherwise use the first non-null component from the input BOMs as the default + foreach (var bom in inputBoms) + { + if (bom.Metadata != null && bom.Metadata.Component != null) + { + outputBom.Metadata.Component = bom.Metadata.Component; + break; + } + } + } + } + + outputBom.Version = 1; + outputBom.SerialNumber = "urn:uuid:" + System.Guid.NewGuid().ToString(); + + if (!outputToConsole) + { + Console.WriteLine("Writing output file..."); + Console.WriteLine($" Total {outputBom.Components?.Count ?? 0} components"); + } + + return await CliUtils.OutputBomHelper(outputBom, options.OutputFormat, options.OutputFile).ConfigureAwait(false); + } + + private static List DetermineInputFiles(MergeCommandOptions options) + { List InputFiles; if (options.InputFiles != null) { @@ -124,56 +178,8 @@ public static async Task Merge(MergeCommandOptions options) // in case the parameter was not passed InputFiles = null; } - var inputBoms = await InputBoms(InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false); - Component bomSubject = null; - if (options.Group != null || options.Name != null || options.Version != null) - bomSubject = new Component - { - Type = Component.Classification.Application, - Group = options.Group, - Name = options.Name, - Version = options.Version, - }; - - Bom outputBom; - if (options.Hierarchical) - { - outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject); - } - else - { - outputBom = CycloneDXUtils.FlatMerge(inputBoms); - if (outputBom.Metadata is null) outputBom.Metadata = new Metadata(); - if (bomSubject != null) - { - // use the params provided if possible - outputBom.Metadata.Component = bomSubject; - } - else - { - // otherwise use the first non-null component from the input BOMs as the default - foreach (var bom in inputBoms) - { - if(bom.Metadata != null && bom.Metadata.Component != null) - { - outputBom.Metadata.Component = bom.Metadata.Component; - break; - } - } - } - } - - outputBom.Version = 1; - outputBom.SerialNumber = "urn:uuid:" + System.Guid.NewGuid().ToString(); - - if (!outputToConsole) - { - Console.WriteLine("Writing output file..."); - Console.WriteLine($" Total {outputBom.Components?.Count ?? 0} components"); - } - - return await CliUtils.OutputBomHelper(outputBom, options.OutputFormat, options.OutputFile).ConfigureAwait(false); + return InputFiles; } private static async Task> InputBoms(IEnumerable inputFilenames, CycloneDXBomFormat inputFormat, bool outputToConsole) From d1237034043c05ed28692210fee546696956f972 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 19 Jul 2023 20:49:34 +0200 Subject: [PATCH 12/12] MergeCommand: DetermineInputFiles() refine progress and result reporting In particular, avoid starting lines with "Got" for different concepts - improve readability. Signed-off-by: Jim Klimov --- src/cyclonedx/Commands/MergeCommand.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cyclonedx/Commands/MergeCommand.cs b/src/cyclonedx/Commands/MergeCommand.cs index 2231088..37705f1 100644 --- a/src/cyclonedx/Commands/MergeCommand.cs +++ b/src/cyclonedx/Commands/MergeCommand.cs @@ -135,7 +135,7 @@ private static List DetermineInputFiles(MergeCommandOptions options) // For some reason, without an immutable list this claims // modifications of the iterable during iteration and fails: ImmutableList InputFilesList = options.InputFilesList.ToImmutableList(); - Console.WriteLine($"Got " + InputFilesList.Count + " file(s) with actual input file names: ['" + string.Join("', '", InputFilesList) + "']"); + Console.WriteLine($"Processing " + InputFilesList.Count + " file(s) with list of actual input file names: ['" + string.Join("', '", InputFilesList) + "']"); foreach (string OneInputFileList in InputFilesList) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); @@ -155,7 +155,7 @@ private static List DetermineInputFiles(MergeCommandOptions options) if (options.InputFilesNulList != null) { ImmutableList InputFilesNulList = options.InputFilesNulList.ToImmutableList(); - Console.WriteLine($"Got " + InputFilesNulList.Count + " file(s) with NUL-separated actual input file names: ['" + string.Join("', '", InputFilesNulList) + "']"); + Console.WriteLine($"Processing " + InputFilesNulList.Count + " file(s) with NUL-separated list of actual input file names: ['" + string.Join("', '", InputFilesNulList) + "']"); foreach (string OneInputFileList in InputFilesNulList) { Console.WriteLine($"Adding to input file list from " + OneInputFileList); @@ -177,6 +177,8 @@ private static List DetermineInputFiles(MergeCommandOptions options) // Revert to legacy (error-handling) behavior below // in case the parameter was not passed InputFiles = null; + } else { + Console.WriteLine($"Determined " + InputFiles.Count + " input files to merge"); } return InputFiles;