Merge Develop into Main

.gitignore

@@ -19,7 +19,6 @@
 /installer/stage
 /installer/rita-*.tar.gz
 /installer/rita-*
-/installer/install-rita-zeek-here.sh
 !/installer/*.md
 
 # only commit .env, .env.production, and test.env files

.vscode/rita.code-workspace

@@ -6,15 +6,18 @@
     ],
     "extensions": {
         "recommendations": [
+            "golang.go",
             "qufiwefefwoyn.inline-sql-syntax",
             "wayou.vscode-todo-highlight",
-            "golang.go",
-            "hjson.hjson",
-            "Tanh.hjson-formatter",
+            "github.vscode-github-actions",
             "redhat.vscode-yaml",
+            "shd101wyy.markdown-preview-enhanced",
+            "laktak.hjson",
+            "Tanh.hjson-formatter",
         ]
     },
     "settings": {
+        // === Go  ===
         "go.languageServerExperimentalFeatures": {
             "diagnostics": false
         },
@@ -31,6 +34,8 @@
             "editor.defaultFormatter": "golang.go",
             "editor.formatOnSave": true
         },
+
+        // === JSON ===
         "[json]": {
             "editor.defaultFormatter": "vscode.json-language-features",
             "editor.formatOnSave": true
@@ -39,22 +44,39 @@
             "editor.defaultFormatter": "vscode.json-language-features",
             "editor.formatOnSave": true
         },
+
         "[hjson]": {
             "editor.defaultFormatter": "Tanh.hjson-formatter",
             "editor.formatOnSave": true
         },
+        // settings for Tanh.hjson-formatter extension
         "hjson-formatter.options": {
+            "condense": 0,
             "separator": true,
-            "condense": 100,
             "bracesSameLine": true,
-            "emitRootBraces": true,
-            "quotes": "strings",
+            "emitRootBraces": false,
+            "multiline": "no-tabs",
             "space": 4,
-            "eol": "auto",
+            "eol": "auto"
         },
+
+        // === YAML ===
         "[yaml]": {
             "editor.defaultFormatter": "redhat.vscode-yaml",
             "editor.formatOnSave": true,
+            "editor.tabSize": 2
+        },
+
+        "[github-actions-workflow]": {
+            "editor.defaultFormatter": "redhat.vscode-yaml",
+            "editor.formatOnSave": true,
+            "editor.tabSize": 2,
+        },
+
+        "[dockercompose]": {
+            "editor.defaultFormatter": "redhat.vscode-yaml",
+            "editor.formatOnSave": true,
+            "editor.tabSize": 2,
         }
     }
 }

README.md

@@ -4,7 +4,9 @@
 
 If you get value out of RITA and would like to go a step further with hunting automation, futuristic visualizations, and data enrichment, then take a look at [AC-Hunter](https://www.activecountermeasures.com/).
 
-Sponsored by [Active Countermeasures](https://activecountermeasures.com/).
+Brought to you by [Active Countermeasures](https://activecountermeasures.com/).
+
+Sponsored by [BHIS](https://www.blackhillsinfosec.com/).
 
 ---
 
@@ -69,7 +71,7 @@ rita import --database=mydatabase --logs=~/mylogs
 
 `logs` is the path to the Zeek logs you wish to import
 
-For datasets that should accumulate data over time, with the logs containing network info that is current (less than 24 hours old), use the `--rolling` flag during creation and each subsequent import into the dataset. The most common use case for this is importing logs from the a Zeek sensor on a cron job each hour.
+For datasets that should accumulate data over time, with the logs containing network info that is current (less than 24 hours old), use the `--rolling` flag during creation and each subsequent import into the dataset. The most common use case for this is importing logs from a Zeek sensor on a cron job each hour.
 
 Note: For datasets that contain over 24 hours of logs, but are over 24 hours old, simply import the top-level directory of the set of logs **without** the `--rolling` flag. Importing these logs with the `--rolling` flag may result in incorrect results.
 
@@ -123,4 +125,4 @@ Check the value of the `"$TERM"` variable, this should be `xterm-256color`. If i
 
 Depending on the color theme of your terminal, the TUI will adjust to either a light mode or a dark mode.
 
-If you're really fancy and like pretty colors, consider using the [Catpuccin](https://catppuccin.com/ports?q=terminal) theme!
+If you're really fancy and like pretty colors, consider using the [Catpuccin](https://catppuccin.com/ports?q=terminal) theme!

cmd/import.go

@@ -46,10 +46,6 @@ var ErrIncompatibleFileExtension = errors.New("incompatible file extension")
 var ErrSkippedDuplicateLog = errors.New("encountered file with same name but different extension, skipping file due to older last modified time")
 var ErrMissingLogDirectory = errors.New("log directory flag is required")
 
-type WalkError struct {
-	Path  string
-	Error error
-}
 type HourlyZeekLogs []map[string][]string
 
 var ImportCommand = &cli.Command{
@@ -161,15 +157,19 @@ func RunImportCmd(startTime time.Time, cfg *config.Config, afs afero.Fs, logDir
 
 	// get list of hourly log maps of all days of log files in directory
 	logMap, walkErrors, err := WalkFiles(afs, logDir, db.Rolling)
-	if err != nil {
-		return importResults, err
-	}
 
-	// log any errors that occurred during the walk
+	// log any errors that occurred during the walk, before returning
+	// this is especially useful when all files in the directory are invalid
+	// instead of only logging 'no valid files found'
 	for _, walkErr := range walkErrors {
 		logger.Debug().Str("path", walkErr.Path).Err(walkErr.Error).Msg("file was left out of import due to error or incompatibility")
 	}
 
+	// return if the walk failed completely
+	if err != nil {
+		return importResults, err
+	}
+
 	var elapsedTime int64
 
 	// loop through each day
@@ -381,7 +381,7 @@ func ParseFolderDate(folder string) (time.Time, error) {
 // WalkFiles starts a goroutine to walk the directory tree at root and send the
 // path of each regular file on the string channel.  It sends the result of the
 // walk on the error channel.  If done is closed, WalkFiles abandons its work.
-func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []WalkError, error) {
+func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []util.WalkError, error) {
 	// check if root is a valid directory or file
 	err := util.ValidateDirectory(afs, root)
 	if err != nil && !errors.Is(err, util.ErrPathIsNotDir) {
@@ -403,13 +403,13 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
 	}
 	fTracker := make(map[string]fileTrack)
 
-	var walkErrors []WalkError
+	var walkErrors []util.WalkError
 
 	err = afero.Walk(afs, root, func(path string, info os.FileInfo, afErr error) error {
 
 		// check if afero failed to access or find a file or directory
 		if afErr != nil {
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: afErr})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: afErr})
 			return nil //nolint:nilerr // log the issue and continue walking
 		}
 
@@ -420,14 +420,13 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
 
 		// skip if file is not a compatible log file
 		if !(strings.HasSuffix(path, ".log") || strings.HasSuffix(path, ".gz")) {
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrIncompatibleFileExtension})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: ErrIncompatibleFileExtension})
 			return nil // log the issue and continue walking
 		}
 
 		// check if the file is readable
-		_, err := afs.Open(path)
-		if err != nil || !(info.Mode().Perm()&0444 == 0444) {
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrInsufficientReadPermissions})
+		if _, err := afs.Open(path); err != nil {
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: ErrInsufficientReadPermissions})
 			return nil //nolint:nilerr // log the issue and continue walking
 		}
 
@@ -454,7 +453,7 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
 		case exists && fileData.lastModified.UTC().Before(info.ModTime().UTC()):
 
 			// warn the user so that this isn't a silent operation
-			walkErrors = append(walkErrors, WalkError{Path: fTracker[trimmedFileName].path, Error: ErrSkippedDuplicateLog})
+			walkErrors = append(walkErrors, util.WalkError{Path: fTracker[trimmedFileName].path, Error: ErrSkippedDuplicateLog})
 			// logger.Warn().Str("original_path", fTracker[trimmedFileName].path).Str("replacement_path", path).Msg("encountered file with same name but different extension, potential duplicate log, skipping")
 
 			fTracker[trimmedFileName] = fileTrack{
@@ -463,7 +462,7 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
 			}
 		// if the current file is older than the one we have already seen or no other conditions are met, skip it
 		default:
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrSkippedDuplicateLog})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: ErrSkippedDuplicateLog})
 
 		}
 
@@ -497,21 +496,21 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
 		case strings.HasPrefix(filepath.Base(path), c.OpenSSLPrefix):
 			prefix = c.OpenSSLPrefix
 		default: // skip file if it doesn't match any of the accepted prefixes
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrInvalidLogType})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: ErrInvalidLogType})
 			continue
 		}
 
 		// parse the hour from the filename
 		hour, err := ParseHourFromFilename(file.path)
 		if err != nil {
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: err})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: err})
 			continue
 		}
 
 		parentDir := filepath.Base(filepath.Dir(file.path))
 		folderDate, err := ParseFolderDate(parentDir)
 		if err != nil {
-			walkErrors = append(walkErrors, WalkError{Path: path, Error: err})
+			walkErrors = append(walkErrors, util.WalkError{Path: path, Error: err})
 		}
 
 		// Check if the entry for the day exists, if not, initialize it

cmd/import_test.go

@@ -622,7 +622,6 @@ func createExpectedResults(logs []cmd.HourlyZeekLogs) []cmd.HourlyZeekLogs {
 }
 
 func TestWalkFiles(t *testing.T) {
-	afs := afero.NewMemMapFs()
 
 	tests := []struct {
 		name                 string
@@ -632,7 +631,7 @@ func TestWalkFiles(t *testing.T) {
 		subdirectories       []string
 		files                []string
 		expectedFiles        []cmd.HourlyZeekLogs
-		expectedWalkErrors   []cmd.WalkError
+		expectedWalkErrors   []util.WalkError
 		rolling              bool
 		expectedError        error
 	}{
@@ -661,7 +660,7 @@ func TestWalkFiles(t *testing.T) {
 					},
 				},
 			}),
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs/.DS_STORE", Error: cmd.ErrIncompatibleFileExtension},
 				{Path: "/logs/capture_loss.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
 				{Path: "/logs/stats.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
@@ -1062,7 +1061,7 @@ func TestWalkFiles(t *testing.T) {
 					},
 				},
 			}),
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs_dupe/conn.log", Error: cmd.ErrSkippedDuplicateLog},
 			},
 			expectedError: nil,
@@ -1083,7 +1082,7 @@ func TestWalkFiles(t *testing.T) {
 					},
 				},
 			}),
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs_dupe/conn.log.gz", Error: cmd.ErrSkippedDuplicateLog},
 			},
 			expectedError: nil,
@@ -1096,7 +1095,7 @@ func TestWalkFiles(t *testing.T) {
 			files: []string{
 				".log.gz", ".log", ".foo",
 			},
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs/.log", Error: cmd.ErrInvalidLogType},
 				{Path: "/logs/.log.gz", Error: cmd.ErrSkippedDuplicateLog},
 				{Path: "/logs/.foo", Error: cmd.ErrIncompatibleFileExtension},
@@ -1113,7 +1112,7 @@ func TestWalkFiles(t *testing.T) {
 				".conn", ".conn_", ".dns", ".dns_", ".http", ".http_", ".ssl", ".ssl_", ".bing", "._bong",
 				"dns_file",
 			},
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs/conn", Error: cmd.ErrIncompatibleFileExtension},
 				{Path: "/logs/dns", Error: cmd.ErrIncompatibleFileExtension},
 				{Path: "/logs/http", Error: cmd.ErrIncompatibleFileExtension},
@@ -1145,7 +1144,7 @@ func TestWalkFiles(t *testing.T) {
 				"files.log", "ntp.log", "radius.log", "sip.log", "x509.log.gz", "dhcp.log", "weird.log",
 				"conn_summary.log", "conn-summary.log", "foo.log",
 			},
-			expectedWalkErrors: []cmd.WalkError{
+			expectedWalkErrors: []util.WalkError{
 				{Path: "/logs/files.log", Error: cmd.ErrInvalidLogType},
 				{Path: "/logs/ntp.log", Error: cmd.ErrInvalidLogType},
 				{Path: "/logs/radius.log", Error: cmd.ErrInvalidLogType},
@@ -1159,25 +1158,33 @@ func TestWalkFiles(t *testing.T) {
 			},
 			expectedError: cmd.ErrNoValidFilesFound,
 		},
-		{
-			name:                 "No Read Permissions on Files",
-			directory:            "/logs",
-			directoryPermissions: iofs.FileMode(0o775),
-			filePermissions:      iofs.FileMode(0o000),
-			files: []string{
-				"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log",
-			},
-			expectedWalkErrors: []cmd.WalkError{
-				{Path: "/logs/conn.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/dns.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/http.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/ssl.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/open_conn.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/open_http.log", Error: cmd.ErrInsufficientReadPermissions},
-				{Path: "/logs/open_ssl.log", Error: cmd.ErrInsufficientReadPermissions},
-			},
-			expectedError: cmd.ErrNoValidFilesFound,
-		},
+
+		// Previously, read permissions were checked with !(info.Mode().Perm()&0444 == 0444), but
+		// this requires all read permissions (user, group, others)/0644 to be set which is not ideal.
+		// A better check would be to see if any read permission is set, i.e., (info.Mode().Perm()&0444 != 0).
+		// However, since some ACL systems/SELinux might interfere with this, it's better to let the Open() call
+		// return an error if permission is denied.
+		// Unfortunately, afero.MemMapFs does not support file permissions when using Open, so this test is skipped.
+		// https://github.com/spf13/afero/issues/150
+		// {
+		// 	name:                 "No Read Permissions on Files",
+		// 	directory:            "/logs",
+		// 	directoryPermissions: iofs.FileMode(0o775),
+		// 	filePermissions:      iofs.FileMode(0o000),
+		// 	files: []string{
+		// 		"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log",
+		// 	},
+		// 	expectedWalkErrors: []util.WalkError{
+		// 		{Path: "/logs/conn.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/dns.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/http.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/ssl.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/open_conn.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/open_http.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 		{Path: "/logs/open_ssl.log", Error: cmd.ErrInsufficientReadPermissions},
+		// 	},
+		// 	expectedError: cmd.ErrNoValidFilesFound,
+		// },
 		{
 			name:                 "No Files, Only SubDirectories",
 			directory:            "/logs",
@@ -1217,6 +1224,8 @@ func TestWalkFiles(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			// create a new in-memory filesystem for each test
+			afs := afero.NewMemMapFs()
 
 			// Create the directory
 			if test.directory != "" {
@@ -1283,7 +1292,7 @@ func TestWalkFiles(t *testing.T) {
 
 			// walk the directory
 			var logMap []cmd.HourlyZeekLogs
-			var walkErrors []cmd.WalkError
+			var walkErrors []util.WalkError
 			var err error
 
 			// since some of the tests are for files passed in to the import command instead of the root directory, we need to
@@ -1298,6 +1307,7 @@ func TestWalkFiles(t *testing.T) {
 			if test.expectedError == nil {
 				require.NoError(t, err, "running WalkFiles should not produce an error")
 			} else {
+				require.Error(t, err, "running WalkFiles should produce an error")
 				require.ErrorIs(t, err, test.expectedError, "error should match expected value")
 
 			}