From 05fd46e34520aa956514a6781dda8c9b793c6f22 Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:26:11 -0400 Subject: [PATCH 1/9] Debugging freezing --- src/main.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/main.rs b/src/main.rs index a926195..4258c80 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,6 @@ use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use memmap2::Mmap; use rayon::prelude::*; use serde::Serialize; -use std::time::Duration; mod patterns; mod scan; @@ -137,7 +136,9 @@ fn main() -> Result<()> { pb }); - let (tx, rx) = channel::unbounded::(); + // Use a bounded channel to apply backpressure and prevent memory issues + // The buffer size is large enough to handle bursts but small enough to apply backpressure + let (tx, rx) = channel::bounded::(5000); let output_path = cli.output.clone(); let found_count_writer = found_count.clone(); let scan_bar_writer = scan_bar.clone(); @@ -157,6 +158,8 @@ fn main() -> Result<()> { pb.set_message(format!("Found {} cryptographic items", count)); } } + // Flush any remaining buffered output + writer.flush()?; Ok(()) }); @@ -384,11 +387,12 @@ fn process_file( metadata: HashMap::new(), }; let key = format!("lib|{}", finding.identifier); - if seen.insert(key) && tx.send_timeout(finding, Duration::from_secs(5)).is_err() { - eprintln!( - "error: writer thread appears to be blocked (filesystem I/O issue?). Aborting send." - ); - return Ok(()); + if seen.insert(key) { + // Use blocking send - the bounded channel will apply backpressure naturally + if let Err(e) = tx.send(finding) { + eprintln!("error: writer thread has stopped: {}", e); + return Ok(()); + } } // 2) algorithms for this library @@ -413,11 +417,12 @@ fn process_file( "alg|{}|{}:{}", finding.identifier, finding.evidence.line, finding.evidence.column ); - if seen.insert(key) && tx.send_timeout(finding, Duration::from_secs(5)).is_err() { - eprintln!( - "error: writer thread appears to be blocked (filesystem I/O issue?). Aborting send." - ); - return Ok(()); + if seen.insert(key) { + // Use blocking send - the bounded channel will apply backpressure naturally + if let Err(e) = tx.send(finding) { + eprintln!("error: writer thread has stopped: {}", e); + return Ok(()); + } } } } From 51cac23b26f9ad72d2b8097fbe9693c836f1921b Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:33:20 -0400 Subject: [PATCH 2/9] Debugging further --- src/main.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4258c80..702e227 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use anyhow::{Context, Result}; @@ -287,6 +288,13 @@ fn main() -> Result<()> { .unwrap(); let total_files = files_to_scan.len(); + // Log the actual count to verify + eprintln!( + "Starting scan of {} files (discovery found {})", + total_files, + file_count.load(Ordering::Relaxed) + ); + if let Some(pb) = &scan_bar { pb.set_length(total_files as u64); pb.set_message("Scanning files..."); @@ -296,10 +304,36 @@ fn main() -> Result<()> { let scanned_count_scan = scanned_count.clone(); let scan_bar_scan = scan_bar.clone(); + // Use a custom thread pool with explicit panic handling files_to_scan.into_par_iter().for_each(|path| { - if let Err(err) = process_file(&path, &patterns_for_scan, &tx) { - eprintln!("error processing {}: {err:#}", path.display()); + let start = Instant::now(); + + // Catch any panics that might occur during processing + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + process_file(&path, &patterns_for_scan, &tx) + })); + + match result { + Ok(Ok(())) => { + // File processed successfully + } + Ok(Err(err)) => { + eprintln!("Error processing {}: {err:#}", path.display()); + } + Err(_) => { + eprintln!("PANIC while processing {}", path.display()); + } } + + let elapsed = start.elapsed(); + if elapsed > Duration::from_secs(5) { + eprintln!( + "Slow file ({:.1}s): {}", + elapsed.as_secs_f32(), + path.display() + ); + } + scanned_count_scan.fetch_add(1, Ordering::Relaxed); if let Some(pb) = &scan_bar_scan { pb.inc(1); From 8b1df94302504cb02b246e405454c2465f55df91 Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:38:38 -0400 Subject: [PATCH 3/9] Freezing debugging further. --- src/main.rs | 58 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index 702e227..5edb10a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,13 +60,13 @@ struct Cli { max_file_mb: Option, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] struct Evidence { line: usize, column: usize, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] struct Finding { #[serde(rename = "assetType")] asset_type: String, @@ -340,7 +340,13 @@ fn main() -> Result<()> { } }); + eprintln!("All files submitted for processing. Scanned so far: {} / {}", + scanned_count.load(Ordering::Relaxed), total_files); + + // IMPORTANT: We must drop tx AFTER the parallel iterator completes + // The parallel iterator above should have finished, but let's verify drop(tx); + eprintln!("Channel sender dropped"); if let Some(pb) = &scan_bar { pb.finish_with_message(format!( @@ -350,7 +356,9 @@ fn main() -> Result<()> { )); } + eprintln!("Waiting for writer thread to finish..."); writer_handle.join().unwrap()?; + eprintln!("Writer thread finished"); if !cli.progress && cli.output != "-" { eprintln!( @@ -422,10 +430,25 @@ fn process_file( }; let key = format!("lib|{}", finding.identifier); if seen.insert(key) { - // Use blocking send - the bounded channel will apply backpressure naturally - if let Err(e) = tx.send(finding) { - eprintln!("error: writer thread has stopped: {}", e); - return Ok(()); + // Try to send with retries to avoid deadlock + let mut retries = 0; + loop { + match tx.try_send(finding.clone()) { + Ok(()) => break, + Err(channel::TrySendError::Full(_)) => { + // Channel is full, wait a bit and retry + retries += 1; + if retries > 100 { + eprintln!("Warning: Channel full for >10s, skipping finding"); + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(channel::TrySendError::Disconnected(_)) => { + eprintln!("error: writer thread has stopped"); + return Ok(()); + } + } } } @@ -452,10 +475,25 @@ fn process_file( finding.identifier, finding.evidence.line, finding.evidence.column ); if seen.insert(key) { - // Use blocking send - the bounded channel will apply backpressure naturally - if let Err(e) = tx.send(finding) { - eprintln!("error: writer thread has stopped: {}", e); - return Ok(()); + // Try to send with retries to avoid deadlock + let mut retries = 0; + loop { + match tx.try_send(finding.clone()) { + Ok(()) => break, + Err(channel::TrySendError::Full(_)) => { + // Channel is full, wait a bit and retry + retries += 1; + if retries > 100 { + eprintln!("Warning: Channel full for >10s, skipping finding"); + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(channel::TrySendError::Disconnected(_)) => { + eprintln!("error: writer thread has stopped"); + return Ok(()); + } + } } } } From efd972c6e3c10f5f9d13566195810964c924f02e Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:43:06 -0400 Subject: [PATCH 4/9] Try explit thread manageent --- src/main.rs | 155 +++++++++++++++++++++++++++------------------------- 1 file changed, 81 insertions(+), 74 deletions(-) diff --git a/src/main.rs b/src/main.rs index 5edb10a..54e701e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -304,43 +304,80 @@ fn main() -> Result<()> { let scanned_count_scan = scanned_count.clone(); let scan_bar_scan = scan_bar.clone(); - // Use a custom thread pool with explicit panic handling - files_to_scan.into_par_iter().for_each(|path| { - let start = Instant::now(); - - // Catch any panics that might occur during processing - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - process_file(&path, &patterns_for_scan, &tx) - })); - - match result { - Ok(Ok(())) => { - // File processed successfully - } - Ok(Err(err)) => { - eprintln!("Error processing {}: {err:#}", path.display()); - } - Err(_) => { - eprintln!("PANIC while processing {}", path.display()); - } - } - - let elapsed = start.elapsed(); - if elapsed > Duration::from_secs(5) { - eprintln!( - "Slow file ({:.1}s): {}", - elapsed.as_secs_f32(), - path.display() - ); - } - - scanned_count_scan.fetch_add(1, Ordering::Relaxed); - if let Some(pb) = &scan_bar_scan { - pb.inc(1); + // Create a work queue for better control + let work_queue = Arc::new(std::sync::Mutex::new(files_to_scan)); + let num_threads = cli.threads; + + eprintln!("Starting {} worker threads...", num_threads); + + // Spawn worker threads explicitly + let handles: Vec<_> = (0..num_threads) + .map(|thread_id| { + let work_queue = work_queue.clone(); + let patterns = patterns_for_scan.clone(); + let tx = tx.clone(); + let scanned_count = scanned_count_scan.clone(); + let scan_bar = scan_bar_scan.clone(); + + std::thread::spawn(move || { + loop { + // Get next file to process + let path = { + let mut queue = work_queue.lock().unwrap(); + if queue.is_empty() { + break; + } + queue.pop() + }; + + if let Some(path) = path { + let start = Instant::now(); + + // Process the file + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + process_file(&path, &patterns, &tx) + })); + + match result { + Ok(Ok(())) => { + // File processed successfully + } + Ok(Err(err)) => { + eprintln!("Thread {}: Error processing {}: {err:#}", + thread_id, path.display()); + } + Err(_) => { + eprintln!("Thread {}: PANIC while processing {}", + thread_id, path.display()); + } + } + + let elapsed = start.elapsed(); + if elapsed > Duration::from_secs(5) { + eprintln!("Thread {}: Slow file ({:.1}s): {}", + thread_id, elapsed.as_secs_f32(), path.display()); + } + + scanned_count.fetch_add(1, Ordering::Relaxed); + if let Some(pb) = &scan_bar { + pb.inc(1); + } + } + } + eprintln!("Thread {} finished", thread_id); + }) + }) + .collect(); + + // Wait for all threads to complete + eprintln!("Waiting for all worker threads to complete..."); + for (i, handle) in handles.into_iter().enumerate() { + if let Err(e) = handle.join() { + eprintln!("Thread {} panicked: {:?}", i, e); } - }); - - eprintln!("All files submitted for processing. Scanned so far: {} / {}", + } + + eprintln!("All worker threads completed. Scanned: {} / {}", scanned_count.load(Ordering::Relaxed), total_files); // IMPORTANT: We must drop tx AFTER the parallel iterator completes @@ -430,25 +467,10 @@ fn process_file( }; let key = format!("lib|{}", finding.identifier); if seen.insert(key) { - // Try to send with retries to avoid deadlock - let mut retries = 0; - loop { - match tx.try_send(finding.clone()) { - Ok(()) => break, - Err(channel::TrySendError::Full(_)) => { - // Channel is full, wait a bit and retry - retries += 1; - if retries > 100 { - eprintln!("Warning: Channel full for >10s, skipping finding"); - break; - } - std::thread::sleep(Duration::from_millis(100)); - } - Err(channel::TrySendError::Disconnected(_)) => { - eprintln!("error: writer thread has stopped"); - return Ok(()); - } - } + // Use blocking send but log if it takes too long + if let Err(e) = tx.send(finding) { + eprintln!("error: writer thread has stopped: {}", e); + return Ok(()); } } @@ -475,25 +497,10 @@ fn process_file( finding.identifier, finding.evidence.line, finding.evidence.column ); if seen.insert(key) { - // Try to send with retries to avoid deadlock - let mut retries = 0; - loop { - match tx.try_send(finding.clone()) { - Ok(()) => break, - Err(channel::TrySendError::Full(_)) => { - // Channel is full, wait a bit and retry - retries += 1; - if retries > 100 { - eprintln!("Warning: Channel full for >10s, skipping finding"); - break; - } - std::thread::sleep(Duration::from_millis(100)); - } - Err(channel::TrySendError::Disconnected(_)) => { - eprintln!("error: writer thread has stopped"); - return Ok(()); - } - } + // Use blocking send but log if it takes too long + if let Err(e) = tx.send(finding) { + eprintln!("error: writer thread has stopped: {}", e); + return Ok(()); } } } From d27e5469a4ee1ed2ff5b04b0117bd4b258cd3b08 Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:50:19 -0400 Subject: [PATCH 5/9] timeout --- src/main.rs | 141 +++++++++++++++++----------------------------------- 1 file changed, 46 insertions(+), 95 deletions(-) diff --git a/src/main.rs b/src/main.rs index 54e701e..f70cf64 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Duration; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use anyhow::{Context, Result}; @@ -137,9 +137,9 @@ fn main() -> Result<()> { pb }); - // Use a bounded channel to apply backpressure and prevent memory issues - // The buffer size is large enough to handle bursts but small enough to apply backpressure - let (tx, rx) = channel::bounded::(5000); + // Use an unbounded channel to avoid deadlocks + // We'll rely on the file processing being the bottleneck, not the writer + let (tx, rx) = channel::unbounded::(); let output_path = cli.output.clone(); let found_count_writer = found_count.clone(); let scan_bar_writer = scan_bar.clone(); @@ -288,102 +288,55 @@ fn main() -> Result<()> { .unwrap(); let total_files = files_to_scan.len(); - // Log the actual count to verify - eprintln!( - "Starting scan of {} files (discovery found {})", - total_files, - file_count.load(Ordering::Relaxed) - ); - if let Some(pb) = &scan_bar { pb.set_length(total_files as u64); pb.set_message("Scanning files..."); } - let patterns_for_scan = patterns.clone(); - let scanned_count_scan = scanned_count.clone(); - let scan_bar_scan = scan_bar.clone(); - - // Create a work queue for better control - let work_queue = Arc::new(std::sync::Mutex::new(files_to_scan)); - let num_threads = cli.threads; - - eprintln!("Starting {} worker threads...", num_threads); - - // Spawn worker threads explicitly - let handles: Vec<_> = (0..num_threads) - .map(|thread_id| { - let work_queue = work_queue.clone(); - let patterns = patterns_for_scan.clone(); - let tx = tx.clone(); - let scanned_count = scanned_count_scan.clone(); - let scan_bar = scan_bar_scan.clone(); - - std::thread::spawn(move || { - loop { - // Get next file to process - let path = { - let mut queue = work_queue.lock().unwrap(); - if queue.is_empty() { - break; - } - queue.pop() - }; - - if let Some(path) = path { - let start = Instant::now(); - - // Process the file - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - process_file(&path, &patterns, &tx) - })); - - match result { - Ok(Ok(())) => { - // File processed successfully - } - Ok(Err(err)) => { - eprintln!("Thread {}: Error processing {}: {err:#}", - thread_id, path.display()); - } - Err(_) => { - eprintln!("Thread {}: PANIC while processing {}", - thread_id, path.display()); - } - } - - let elapsed = start.elapsed(); - if elapsed > Duration::from_secs(5) { - eprintln!("Thread {}: Slow file ({:.1}s): {}", - thread_id, elapsed.as_secs_f32(), path.display()); - } - - scanned_count.fetch_add(1, Ordering::Relaxed); - if let Some(pb) = &scan_bar { - pb.inc(1); - } - } - } - eprintln!("Thread {} finished", thread_id); - }) - }) - .collect(); - - // Wait for all threads to complete - eprintln!("Waiting for all worker threads to complete..."); - for (i, handle) in handles.into_iter().enumerate() { - if let Err(e) = handle.join() { - eprintln!("Thread {} panicked: {:?}", i, e); + // Process files in parallel with rayon + files_to_scan.into_par_iter().for_each(|path| { + // Create a timeout mechanism using a separate thread + let (timeout_tx, timeout_rx) = channel::bounded::<()>(1); + let path_clone = path.clone(); + + let timeout_thread = std::thread::spawn(move || { + // Wait for either completion signal or timeout + if timeout_rx.recv_timeout(Duration::from_secs(10)).is_err() { + eprintln!( + "TIMEOUT: File took >10s, skipping: {}", + path_clone.display() + ); + } + }); + + // Process the file + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + process_file(&path, &patterns, &tx) + })); + + // Signal completion to timeout thread + let _ = timeout_tx.send(()); + drop(timeout_thread); + + // Handle result + match result { + Ok(Ok(())) => {} + Ok(Err(err)) => { + eprintln!("Error processing {}: {err:#}", path.display()); + } + Err(_) => { + eprintln!("PANIC while processing {}", path.display()); + } } - } - - eprintln!("All worker threads completed. Scanned: {} / {}", - scanned_count.load(Ordering::Relaxed), total_files); - - // IMPORTANT: We must drop tx AFTER the parallel iterator completes - // The parallel iterator above should have finished, but let's verify + + scanned_count.fetch_add(1, Ordering::Relaxed); + if let Some(pb) = &scan_bar { + pb.inc(1); + } + }); + + // All files have been processed drop(tx); - eprintln!("Channel sender dropped"); if let Some(pb) = &scan_bar { pb.finish_with_message(format!( @@ -393,9 +346,7 @@ fn main() -> Result<()> { )); } - eprintln!("Waiting for writer thread to finish..."); writer_handle.join().unwrap()?; - eprintln!("Writer thread finished"); if !cli.progress && cli.output != "-" { eprintln!( From 3e02be854c4eeb0392d7061d66afcfd9e1f90cde Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 14:54:58 -0400 Subject: [PATCH 6/9] timeout --- src/main.rs | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/main.rs b/src/main.rs index f70cf64..dea9a46 100644 --- a/src/main.rs +++ b/src/main.rs @@ -293,30 +293,45 @@ fn main() -> Result<()> { pb.set_message("Scanning files..."); } - // Process files in parallel with rayon - files_to_scan.into_par_iter().for_each(|path| { - // Create a timeout mechanism using a separate thread - let (timeout_tx, timeout_rx) = channel::bounded::<()>(1); - let path_clone = path.clone(); - - let timeout_thread = std::thread::spawn(move || { - // Wait for either completion signal or timeout - if timeout_rx.recv_timeout(Duration::from_secs(10)).is_err() { - eprintln!( - "TIMEOUT: File took >10s, skipping: {}", - path_clone.display() - ); + // Track currently processing files to identify stuck ones + let processing_files = Arc::new(std::sync::Mutex::new(HashMap::::new())); + let processing_files_clone = processing_files.clone(); + + // Spawn a monitor thread that reports stuck files + std::thread::spawn(move || { + loop { + std::thread::sleep(Duration::from_secs(30)); + let files = processing_files_clone.lock().unwrap(); + if !files.is_empty() { + eprintln!("\n=== FILES CURRENTLY BEING PROCESSED (>30s) ==="); + for (thread_id, path) in files.iter() { + eprintln!(" Thread {:?}: {}", thread_id, path.display()); + } + eprintln!("===============================================\n"); } - }); + } + }); + // Process files in parallel with rayon + files_to_scan.into_par_iter().for_each(|path| { + let thread_id = std::thread::current().id(); + + // Register this file as being processed + { + let mut files = processing_files.lock().unwrap(); + files.insert(thread_id, path.clone()); + } + // Process the file let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { process_file(&path, &patterns, &tx) })); - - // Signal completion to timeout thread - let _ = timeout_tx.send(()); - drop(timeout_thread); + + // Unregister this file + { + let mut files = processing_files.lock().unwrap(); + files.remove(&thread_id); + } // Handle result match result { From 23e86a539ddc0446ea0ae3d58f1a1aea8d9b282b Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 15:03:45 -0400 Subject: [PATCH 7/9] parse headers as cpp, and timeout --- src/scan.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/scan.rs b/src/scan.rs index 9d38318..dc1460f 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -102,7 +102,8 @@ pub struct AlgorithmHit<'a> { pub fn language_from_path(path: &std::path::Path) -> Option { let ext = path.extension()?.to_str()?.to_ascii_lowercase(); match ext.as_str() { - "c" | "h" => Some(Language::C), + "c" => Some(Language::C), + "h" => Some(Language::Cpp), // Parse .h as C++ since it's backwards compatible with C "cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => Some(Language::Cpp), "java" => Some(Language::Java), "py" => Some(Language::Python), @@ -134,6 +135,10 @@ pub fn parse(lang: Language, content: &str) -> Result { Language::Rust => ts_lang_rust(), }; parser.set_language(&ts_lang).context("set language")?; + + // Set a timeout to prevent the parser from hanging on malformed code + parser.set_timeout_micros(5_000_000); // 5 second timeout + parser.parse(content, None).context("parse") } From 061a6bc61f69a78749e9d024df40c0d6f2d0c852 Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 15:06:40 -0400 Subject: [PATCH 8/9] File is now fixed. --- src/main.rs | 49 ++----------------------------------------------- src/scan.rs | 4 +++- 2 files changed, 5 insertions(+), 48 deletions(-) diff --git a/src/main.rs b/src/main.rs index dea9a46..15f8640 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,6 @@ use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Duration; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use anyhow::{Context, Result}; @@ -293,55 +292,11 @@ fn main() -> Result<()> { pb.set_message("Scanning files..."); } - // Track currently processing files to identify stuck ones - let processing_files = Arc::new(std::sync::Mutex::new(HashMap::::new())); - let processing_files_clone = processing_files.clone(); - - // Spawn a monitor thread that reports stuck files - std::thread::spawn(move || { - loop { - std::thread::sleep(Duration::from_secs(30)); - let files = processing_files_clone.lock().unwrap(); - if !files.is_empty() { - eprintln!("\n=== FILES CURRENTLY BEING PROCESSED (>30s) ==="); - for (thread_id, path) in files.iter() { - eprintln!(" Thread {:?}: {}", thread_id, path.display()); - } - eprintln!("===============================================\n"); - } - } - }); - // Process files in parallel with rayon files_to_scan.into_par_iter().for_each(|path| { - let thread_id = std::thread::current().id(); - - // Register this file as being processed - { - let mut files = processing_files.lock().unwrap(); - files.insert(thread_id, path.clone()); - } - // Process the file - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - process_file(&path, &patterns, &tx) - })); - - // Unregister this file - { - let mut files = processing_files.lock().unwrap(); - files.remove(&thread_id); - } - - // Handle result - match result { - Ok(Ok(())) => {} - Ok(Err(err)) => { - eprintln!("Error processing {}: {err:#}", path.display()); - } - Err(_) => { - eprintln!("PANIC while processing {}", path.display()); - } + if let Err(err) = process_file(&path, &patterns, &tx) { + eprintln!("Error processing {}: {err:#}", path.display()); } scanned_count.fetch_add(1, Ordering::Relaxed); diff --git a/src/scan.rs b/src/scan.rs index dc1460f..7dc021d 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -136,7 +136,9 @@ pub fn parse(lang: Language, content: &str) -> Result { }; parser.set_language(&ts_lang).context("set language")?; - // Set a timeout to prevent the parser from hanging on malformed code + // Allow deprecated method for now - the new API is more complex + // and the current method works fine for our needs + #[allow(deprecated)] parser.set_timeout_micros(5_000_000); // 5 second timeout parser.parse(content, None).context("parse") From 2dc334b5997f37da1d4f55779a32a02603d049f4 Mon Sep 17 00:00:00 2001 From: Isaac Elbaz Date: Sun, 21 Sep 2025 15:08:12 -0400 Subject: [PATCH 9/9] Cargo format. --- src/scan.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scan.rs b/src/scan.rs index 7dc021d..8ffbcc1 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -135,12 +135,12 @@ pub fn parse(lang: Language, content: &str) -> Result { Language::Rust => ts_lang_rust(), }; parser.set_language(&ts_lang).context("set language")?; - + // Allow deprecated method for now - the new API is more complex // and the current method works fine for our needs #[allow(deprecated)] parser.set_timeout_micros(5_000_000); // 5 second timeout - + parser.parse(content, None).context("parse") }