CM-53667: improve ignore logging

Author: gotbadger

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    }
+  ]
+}

cycode/cli/files_collector/walk_ignore.py

@@ -30,7 +30,7 @@ def _collect_top_level_ignore_files(path: str) -> list[str]:
         for ignore_file in _SUPPORTED_IGNORE_PATTERN_FILES:
             ignore_file_path = os.path.join(dir_path, ignore_file)
             if os.path.exists(ignore_file_path):
-                logger.debug('Apply top level ignore file: %s', ignore_file_path)
+                logger.debug('Reading top level ignore file: %s', ignore_file_path)
                 ignore_files.append(ignore_file_path)
     return ignore_files
 
@@ -41,4 +41,15 @@ def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None,
         global_ignore_file_paths=_collect_top_level_ignore_files(path),
         global_patterns=_DEFAULT_GLOBAL_IGNORE_PATTERNS,
     )
-    yield from ignore_filter_manager.walk()
+
+    for dirpath, dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored():
+        rel_dirpath = '' if dirpath == path else os.path.relpath(dirpath, path)
+        display_dir = rel_dirpath or '.'
+        for kind, names in (
+            ('directory', ignored_dirnames),
+            ('file', ignored_filenames),
+        ):
+            for name in names:
+                logger.debug('Skipping ignore matched %s %s/%s', kind, display_dir, name)
+
+        yield dirpath, dirnames, filenames

cycode/cli/utils/ignore_utils.py

@@ -45,6 +45,8 @@
     BinaryIO,
     Optional,
     Union,
+    List,
+    Dict,
 )
 
 
@@ -388,19 +390,30 @@ def is_ignored(self, path: str) -> Optional[bool]:
             return matches[-1].is_exclude
         return None
 
-    def walk(self, **kwargs) -> Generator[tuple[str, list[str], list[str]], None, None]:
-        """Wrap os.walk() without ignored files and subdirectories and kwargs are passed to walk."""
+    def walk_with_ignored(
+        self, **kwargs
+    ) -> Generator[tuple[str, list[str], list[str], list[str], list[str]], None, None]:
+        """Wrap os.walk() and also return lists of ignored directories and files.
+
+        Yields tuples: (dirpath, included_dirnames, included_filenames, ignored_dirnames, ignored_filenames)
+        """
         for dirpath, dirnames, filenames in os.walk(self.path, topdown=True, **kwargs):
             rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path)
 
+            original_dirnames = list(dirnames)
+            included_dirnames = [d for d in original_dirnames if not self.is_ignored(os.path.join(rel_dirpath, d))]
+
             # decrease recursion depth of os.walk() by ignoring subdirectories because of topdown=True
             # slicing ([:]) is mandatory to change dict in-place!
-            dirnames[:] = [d for d in dirnames if not self.is_ignored(os.path.join(rel_dirpath, d))]
+            dirnames[:] = included_dirnames
+
+            ignored_dirnames = [d for d in original_dirnames if d not in included_dirnames]
 
-            # remove ignored files
-            filenames = [f for f in filenames if not self.is_ignored(os.path.join(rel_dirpath, f))]
+            original_filenames = list(filenames)
+            included_filenames = [f for f in original_filenames if not self.is_ignored(os.path.join(rel_dirpath, f))]
+            ignored_filenames = [f for f in original_filenames if f not in included_filenames]
 
-            yield dirpath, dirnames, filenames
+            yield dirpath, dirnames, included_filenames, ignored_dirnames, ignored_filenames
 
     @classmethod
     def build(