extract first parts into subcrates

Cargo.toml

@@ -9,21 +9,42 @@ build = "build.rs"
 edition = "2024"
 
 [workspace]
-members = ["crates/*"]
+resolver = "2" 
+members = [
+    "crates/lib/*",
+]
 exclude = [
     "ignored",
     "tests",
     ".workspace",
     ".rustwide-docker",
 ]
 
-[dependencies]
+[workspace.dependencies]
+anyhow = { version = "1.0.42", features = ["backtrace"]}
+chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
+derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
+opentelemetry = "0.31.0"
+opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
+opentelemetry-resource-detectors = "0.10.0"
+opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
+regex = "1"
 sentry = { version = "0.46.0", features = ["panic", "tracing", "tower-http", "anyhow", "backtrace"] }
-log = "0.4"
+tokio = { version = "1.0", features = ["rt-multi-thread", "signal", "macros", "process", "sync"] }
 tracing = "0.1.37"
+url = { version = "2.1.1", features = ["serde"] }
+
+[dependencies]
+docs_rs_env_vars = { path = "crates/lib/docs_rs_env_vars" }
+docs_rs_logging = { path = "crates/lib/docs_rs_logging" }
+docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry" }
+docs_rs_utils = { path = "crates/lib/docs_rs_utils" }
+sentry = { workspace = true }
+log = "0.4"
+tracing = { workspace = true }
 tracing-subscriber = { version = "0.3.20", default-features = false, features = ["ansi", "fmt", "json", "env-filter", "tracing-log"] }
 tracing-log = "0.2.0"
-regex = "1"
+regex = { workspace = true }
 clap = { version = "4.0.22", features = [ "derive" ] }
 crates-index = { version = "3.0.0", default-features = false, features = ["git", "git-https", "git-performance", "parallel"] }
 rayon = "1.6.1"
@@ -33,17 +54,15 @@ reqwest = { version = "0.12", features = ["json", "gzip"] }
 semver = { version = "1.0.4", features = ["serde"] }
 slug = "0.1.1"
 sqlx = { version = "0.8", features = [ "runtime-tokio", "postgres", "sqlite", "chrono" ] }
-url = { version = "2.1.1", features = ["serde"] }
-docsrs-metadata = { path = "crates/metadata" }
-anyhow = { version = "1.0.42", features = ["backtrace"]}
+url = { workspace = true }
+docsrs-metadata = { path = "crates/lib/metadata" }
+anyhow = { workspace = true }
 thiserror = "2.0.3"
 comrak = { version = "0.49.0", default-features = false }
 syntect = { version = "5.0.0", default-features = false, features = ["parsing", "html", "dump-load", "regex-onig"] }
 toml = "0.9.2"
-opentelemetry = "0.31.0"
-opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
-opentelemetry-resource-detectors = "0.10.0"
-opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 rustwide = { version = "0.21.0", features = ["unstable-toolchain-ci", "unstable"] }
 mime_guess = "2"
 zstd = "0.13.0"
@@ -53,20 +72,20 @@ path-slash = "0.2.0"
 base64 = "0.22"
 strum = { version = "0.27.0", features = ["derive"] }
 lol_html = "2.0.0"
-font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" }
+font-awesome-as-a-crate = { path = "crates/lib/font-awesome-as-a-crate" }
 dashmap = "6.0.0"
 zip = {version = "6.0.0", default-features = false, features = ["bzip2"]}
 bzip2 = "0.6.0"
 getrandom = "0.3.1"
 itertools = { version = "0.14.0" }
 hex = "0.4.3"
-derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
+derive_more = { workspace = true }
 sysinfo = { version = "0.37.2", default-features = false, features = ["system"] }
 derive_builder = "0.20.2"
 
 # Async
 async-compression = { version = "0.4.32", features = ["tokio", "bzip2", "zstd", "gzip"] }
-tokio = { version = "1.0", features = ["rt-multi-thread", "signal", "macros", "process", "sync"] }
+tokio = { workspace = true }
 tokio-util = { version = "0.7.15", default-features = false, features = ["io"] }
 tracing-futures= { version = "0.2.5", features = ["std-future", "futures-03"] }
 futures-util = "0.3.5"
@@ -100,15 +119,16 @@ walkdir = "2"
 phf = "0.13.1"
 
 # Date and Time utilities
-chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
+chrono = { workspace = true }
 
 # Transitive dependencies we don't use directly but need to have specific versions of
 constant_time_eq = "0.4.2"
 md5 = "0.8.0"
 
-crates_io_validation = { path = "crates/crates_io_validation" }
+crates_io_validation = { path = "crates/lib/crates_io_validation" }
 
 [dev-dependencies]
+docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry", features = ["testing"] }
 criterion = "0.8.0"
 kuchikiki = "0.8"
 http-body-util = "0.1.0"

build.rs

@@ -73,7 +73,6 @@ type ETagMap<'a> = phf_codegen::Map<'a, String>;
 fn main() -> Result<()> {
     let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;
     let out_dir = Path::new(&out_dir);
-    read_git_version()?;
 
     let mut etag_map: ETagMap = ETagMap::new();
 
@@ -94,54 +93,6 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-fn read_git_version() -> Result<()> {
-    if let Ok(v) = env::var("GIT_SHA") {
-        // first try to read an externally provided git SAH, e.g., from CI
-        println!("cargo:rustc-env=GIT_SHA={v}");
-    } else {
-        // then try to read the git repo.
-        let maybe_hash = get_git_hash()?;
-        let git_hash = maybe_hash.as_deref().unwrap_or("???????");
-        println!("cargo:rustc-env=GIT_SHA={git_hash}");
-    }
-
-    println!(
-        "cargo:rustc-env=BUILD_DATE={}",
-        time::OffsetDateTime::now_utc().date(),
-    );
-
-    Ok(())
-}
-
-fn get_git_hash() -> Result<Option<String>> {
-    use std::process::Command;
-
-    let output = Command::new("git")
-        .args(["rev-parse", "--short", "HEAD"])
-        .output();
-
-    match output {
-        Ok(output) if output.status.success() => {
-            let hash = String::from_utf8(output.stdout)?.trim().to_string();
-
-            // TODO: are these right?
-            tracked::track(".git/HEAD")?;
-            tracked::track(".git/index")?;
-
-            Ok(Some(hash))
-        }
-        Ok(output) => {
-            let err = String::from_utf8_lossy(&output.stderr);
-            eprintln!("failed to get git repo: {}", err.trim());
-            Ok(None)
-        }
-        Err(err) => {
-            eprintln!("failed to execute git: {err}");
-            Ok(None)
-        }
-    }
-}
-
 fn etag_from_path(path: impl AsRef<Path>) -> Result<String> {
     Ok(etag_from_content(std::fs::read(&path)?))
 }

crates/lib/docs_rs_env_vars/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "docs_rs_env_vars"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+tracing = { workspace = true }

crates/lib/docs_rs_env_vars/src/lib.rs

@@ -0,0 +1,37 @@
+use anyhow::{Context as _, Result, anyhow};
+use std::{env::VarError, error::Error, str::FromStr};
+use tracing::trace;
+
+pub fn env<T>(var: &str, default: T) -> Result<T>
+where
+    T: FromStr,
+    T::Err: Error + Send + Sync + 'static,
+{
+    Ok(maybe_env(var)?.unwrap_or(default))
+}
+
+pub fn require_env<T>(var: &str) -> Result<T>
+where
+    T: FromStr,
+    <T as FromStr>::Err: Error + Send + Sync + 'static,
+{
+    maybe_env(var)?.with_context(|| anyhow!("configuration variable {} is missing", var))
+}
+
+pub fn maybe_env<T>(var: &str) -> Result<Option<T>>
+where
+    T: FromStr,
+    T::Err: Error + Send + Sync + 'static,
+{
+    match std::env::var(var) {
+        Ok(content) => Ok(content
+            .parse::<T>()
+            .map(Some)
+            .with_context(|| format!("failed to parse configuration variable {var}"))?),
+        Err(VarError::NotPresent) => {
+            trace!("optional configuration variable {} is not set", var);
+            Ok(None)
+        }
+        Err(VarError::NotUnicode(_)) => Err(anyhow!("configuration variable {} is not UTF-8", var)),
+    }
+}

crates/lib/docs_rs_logging/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "docs_rs_logging"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+docs_rs_utils = { path = "../docs_rs_utils" }
+sentry = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { version = "0.3.20", default-features = false, features = ["ansi", "fmt", "json", "env-filter", "tracing-log"] }

crates/lib/docs_rs_logging/src/lib.rs

@@ -0,0 +1,79 @@
+use sentry::{
+    TransactionContext, integrations::panic as sentry_panic,
+    integrations::tracing as sentry_tracing,
+};
+use std::{env, str::FromStr as _, sync::Arc};
+use tracing_subscriber::{EnvFilter, filter::Directive, prelude::*};
+
+pub struct Guard {
+    #[allow(dead_code)]
+    sentry_guard: Option<sentry::ClientInitGuard>,
+}
+
+pub fn init() -> anyhow::Result<Guard> {
+    let log_formatter = {
+        let log_format = env::var("DOCSRS_LOG_FORMAT").unwrap_or_default();
+
+        if log_format == "json" {
+            tracing_subscriber::fmt::layer().json().boxed()
+        } else {
+            tracing_subscriber::fmt::layer().boxed()
+        }
+    };
+
+    let tracing_registry = tracing_subscriber::registry().with(log_formatter).with(
+        EnvFilter::builder()
+            .with_default_directive(Directive::from_str("docs_rs=info")?)
+            .with_env_var("DOCSRS_LOG")
+            .from_env_lossy(),
+    );
+
+    let sentry_guard = if let Ok(sentry_dsn) = env::var("SENTRY_DSN") {
+        tracing::subscriber::set_global_default(tracing_registry.with(
+            sentry_tracing::layer().event_filter(|md| {
+                if md.fields().field("reported_to_sentry").is_some() {
+                    sentry_tracing::EventFilter::Ignore
+                } else {
+                    sentry_tracing::default_event_filter(md)
+                }
+            }),
+        ))?;
+
+        let traces_sample_rate = env::var("SENTRY_TRACES_SAMPLE_RATE")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(0.0);
+
+        let traces_sampler = move |ctx: &TransactionContext| -> f32 {
+            if let Some(sampled) = ctx.sampled() {
+                // if the transaction was already marked as "to be sampled" by
+                // the JS/frontend SDK, we want to sample it in the backend too.
+                return if sampled { 1.0 } else { 0.0 };
+            }
+
+            let op = ctx.operation();
+            if op == "docbuilder.build_package" {
+                // record all transactions for builds
+                1.
+            } else {
+                traces_sample_rate
+            }
+        };
+
+        Some(sentry::init((
+            sentry_dsn,
+            sentry::ClientOptions {
+                release: Some(docs_rs_utils::BUILD_VERSION.into()),
+                attach_stacktrace: true,
+                traces_sampler: Some(Arc::new(traces_sampler)),
+                ..Default::default()
+            }
+            .add_integration(sentry_panic::PanicIntegration::default()),
+        )))
+    } else {
+        tracing::subscriber::set_global_default(tracing_registry)?;
+        None
+    };
+
+    Ok(Guard { sentry_guard })
+}