From 36916903e615a78114fc552bdfe52cb80d5aa51c Mon Sep 17 00:00:00 2001 From: James Liu Date: Fri, 25 Jul 2025 17:50:29 -0700 Subject: [PATCH 01/68] Inline async_executor --- Cargo.toml | 4 - crates/bevy_a11y/Cargo.toml | 2 +- crates/bevy_asset/Cargo.toml | 4 +- crates/bevy_ecs/Cargo.toml | 8 +- crates/bevy_input/Cargo.toml | 2 +- crates/bevy_input_focus/Cargo.toml | 2 +- crates/bevy_internal/Cargo.toml | 9 - crates/bevy_tasks/Cargo.toml | 24 +- crates/bevy_tasks/src/async_executor.rs | 1218 +++++++++++++++++++++++ crates/bevy_tasks/src/executor.rs | 6 +- crates/bevy_tasks/src/lib.rs | 4 +- crates/bevy_transform/Cargo.toml | 8 +- 12 files changed, 1244 insertions(+), 47 deletions(-) create mode 100644 crates/bevy_tasks/src/async_executor.rs diff --git a/Cargo.toml b/Cargo.toml index 4b4f00b090393..6bcb821a23f20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,7 +125,6 @@ unused_qualifications = "warn" [features] default = [ "std", - "async_executor", "android-game-activity", "android_shared_stdcxx", "animation", @@ -554,9 +553,6 @@ custom_cursor = ["bevy_internal/custom_cursor"] # Experimental support for nodes that are ignored for UI layouting ghost_nodes = ["bevy_internal/ghost_nodes"] -# Uses `async-executor` as a task execution backend. -async_executor = ["std", "bevy_internal/async_executor"] - # Allows access to the `std` crate. std = ["bevy_internal/std"] diff --git a/crates/bevy_a11y/Cargo.toml b/crates/bevy_a11y/Cargo.toml index 262b8e5b823fe..8c1ed0438e66f 100644 --- a/crates/bevy_a11y/Cargo.toml +++ b/crates/bevy_a11y/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy", "accessibility", "a11y"] [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor"] +default = ["std", "bevy_reflect"] # Functionality diff --git a/crates/bevy_asset/Cargo.toml b/crates/bevy_asset/Cargo.toml index edf8986130a00..dc66b3e08a4fc 100644 --- a/crates/bevy_asset/Cargo.toml +++ b/crates/bevy_asset/Cargo.toml @@ -27,9 +27,7 @@ bevy_ecs = { path = "../bevy_ecs", version = "0.17.0-dev", default-features = fa bevy_reflect = { path = "../bevy_reflect", version = "0.17.0-dev", default-features = false, features = [ "uuid", ] } -bevy_tasks = { path = "../bevy_tasks", version = "0.17.0-dev", default-features = false, features = [ - "async_executor", -] } +bevy_tasks = { path = "../bevy_tasks", version = "0.17.0-dev", default-features = false } bevy_utils = { path = "../bevy_utils", version = "0.17.0-dev", default-features = false } bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ "std", diff --git a/crates/bevy_ecs/Cargo.toml b/crates/bevy_ecs/Cargo.toml index f0f9b782afff2..2c291677d5daf 100644 --- a/crates/bevy_ecs/Cargo.toml +++ b/crates/bevy_ecs/Cargo.toml @@ -11,7 +11,7 @@ categories = ["game-engines", "data-structures"] rust-version = "1.86.0" [features] -default = ["std", "bevy_reflect", "async_executor", "backtrace"] +default = ["std", "bevy_reflect", "backtrace"] # Functionality @@ -48,12 +48,6 @@ bevy_debug_stepping = [] ## This will often provide more detailed error messages. track_location = [] -# Executor Backend - -## Uses `async-executor` as a task execution backend. -## This backend is incompatible with `no_std` targets. -async_executor = ["std", "bevy_tasks/async_executor"] - # Platform Compatibility ## Allows access to the `std` crate. Enabling this feature will prevent compilation diff --git a/crates/bevy_input/Cargo.toml b/crates/bevy_input/Cargo.toml index c32a87a52d9dc..2fe037a58fba4 100644 --- a/crates/bevy_input/Cargo.toml +++ b/crates/bevy_input/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy"] [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor", "smol_str"] +default = ["std", "bevy_reflect", "smol_str"] # Functionality diff --git a/crates/bevy_input_focus/Cargo.toml b/crates/bevy_input_focus/Cargo.toml index 60b824258df31..2cd9d18771b8c 100644 --- a/crates/bevy_input_focus/Cargo.toml +++ b/crates/bevy_input_focus/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["bevy"] rust-version = "1.85.0" [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor"] +default = ["std", "bevy_reflect"] # Functionality diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index 81fcb61133085..683de2fbd4137 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -356,15 +356,6 @@ libm = [ "bevy_window?/libm", ] -# Uses `async-executor` as a task execution backend. -# This backend is incompatible with `no_std` targets. -async_executor = [ - "std", - "bevy_tasks/async_executor", - "bevy_ecs/async_executor", - "bevy_transform/async_executor", -] - # Enables use of browser APIs. # Note this is currently only applicable on `wasm32` architectures. web = [ diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index e28d7fc88d319..1c43f917a6357 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy"] [features] -default = ["std", "async_executor"] +default = ["std"] # Functionality @@ -19,19 +19,23 @@ multi_threaded = [ "std", "dep:async-channel", "dep:concurrent-queue", - "async_executor", + "dep:fastrand", ] -## Uses `async-executor` as a task execution backend. -## This backend is incompatible with `no_std` targets. -async_executor = ["std", "dep:async-executor"] - # Platform Compatibility ## Allows access to the `std` crate. Enabling this feature will prevent compilation ## on `no_std` targets, but provides access to certain additional features on ## supported platforms. -std = ["futures-lite/std", "async-task/std", "bevy_platform/std"] +std = [ + "futures-lite/std", + "async-task/std", + "bevy_platform/std", + "fastrand/std", + "dep:slab", + "dep:concurrent-queue", + "dep:pin-project-lite", +] ## `critical-section` provides the building blocks for synchronization primitives ## on all platforms, including `no_std`. @@ -42,7 +46,6 @@ critical-section = ["bevy_platform/critical-section"] web = [ "bevy_platform/web", "dep:wasm-bindgen-futures", - "dep:pin-project", "dep:futures-channel", ] @@ -60,7 +63,9 @@ derive_more = { version = "2", default-features = false, features = [ "deref_mut", ] } cfg-if = "1.0.0" -async-executor = { version = "1.11", optional = true } +slab = { version = "0.4", optional = true } +pin-project-lite = { version = "0.2", optional = true } +fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } concurrent-queue = { version = "2.0.0", optional = true } @@ -71,7 +76,6 @@ crossbeam-queue = { version = "0.3", default-features = false, features = [ [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = { version = "0.4", optional = true } -pin-project = { version = "1", optional = true } futures-channel = { version = "0.3", optional = true } [target.'cfg(not(all(target_has_atomic = "8", target_has_atomic = "16", target_has_atomic = "32", target_has_atomic = "64", target_has_atomic = "ptr")))'.dependencies] diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs new file mode 100644 index 0000000000000..1aad2a6c133ad --- /dev/null +++ b/crates/bevy_tasks/src/async_executor.rs @@ -0,0 +1,1218 @@ +use std::fmt; +use std::marker::PhantomData; +use std::panic::{RefUnwindSafe, UnwindSafe}; +use std::pin::Pin; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; +use std::task::{Context, Poll, Waker}; + +use async_task::{Builder, Runnable}; +use concurrent_queue::ConcurrentQueue; +use futures_lite::{future, prelude::*}; +use pin_project_lite::pin_project; +use bevy_platform::prelude::Vec; +use slab::Slab; + +/// An async executor. +/// +/// # Examples +/// +/// A multi-threaded executor: +/// +/// ``` +/// use async_channel::unbounded; +/// use async_executor::Executor; +/// use easy_parallel::Parallel; +/// use futures_lite::future; +/// +/// let ex = Executor::new(); +/// let (signal, shutdown) = unbounded::<()>(); +/// +/// Parallel::new() +/// // Run four executor threads. +/// .each(0..4, |_| future::block_on(ex.run(shutdown.recv()))) +/// // Run the main future on the current thread. +/// .finish(|| future::block_on(async { +/// println!("Hello world!"); +/// drop(signal); +/// })); +/// ``` +pub struct Executor<'a> { + /// The executor state. + state: AtomicPtr, + + /// Makes the `'a` lifetime invariant. + _marker: PhantomData>, +} + +#[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" +)] +// SAFETY: Executor stores no thread local state that can be accessed via other thread. +unsafe impl Send for Executor<'_> {} +#[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" +)] +// SAFETY: Executor internally synchronizes all of it's operations internally. +unsafe impl Sync for Executor<'_> {} + +impl UnwindSafe for Executor<'_> {} +impl RefUnwindSafe for Executor<'_> {} + +impl fmt::Debug for Executor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_executor(self, "Executor", f) + } +} + +impl<'a> Executor<'a> { + /// Creates a new executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// ``` + pub const fn new() -> Executor<'a> { + Executor { + state: AtomicPtr::new(std::ptr::null_mut()), + _marker: PhantomData, + } + } + + /// Returns `true` if there are no unfinished tasks. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// assert!(ex.is_empty()); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(!ex.is_empty()); + /// + /// assert!(ex.try_tick()); + /// assert!(ex.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.state().active().is_empty() + } + + /// Spawns a task onto the executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// ``` + pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + let mut active = self.state().active(); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: `T` and the future are `Send`. + unsafe { self.spawn_inner(future, &mut active) } + } + + /// Spawns many tasks onto the executor. + /// + /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and + /// spawns all of the tasks in one go. With large amounts of tasks this can improve + /// contention. + /// + /// For very large numbers of tasks the lock is occasionally dropped and re-acquired to + /// prevent runner thread starvation. It is assumed that the iterator provided does not + /// block; blocking iterators can lock up the internal mutex and therefore the entire + /// executor. + /// + /// ## Example + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::{stream, prelude::*}; + /// use std::future::ready; + /// + /// # futures_lite::future::block_on(async { + /// let mut ex = Executor::new(); + /// + /// let futures = [ + /// ready(1), + /// ready(2), + /// ready(3) + /// ]; + /// + /// // Spawn all of the futures onto the executor at once. + /// let mut tasks = vec![]; + /// ex.spawn_many(futures, &mut tasks); + /// + /// // Await all of them. + /// let results = ex.run(async move { + /// stream::iter(tasks).then(|x| x).collect::>().await + /// }).await; + /// assert_eq!(results, [1, 2, 3]); + /// # }); + /// ``` + /// + /// [`spawn`]: Executor::spawn + pub fn spawn_many + Send + 'a>( + &self, + futures: impl IntoIterator, + handles: &mut impl Extend>, + ) { + let mut active = Some(self.state().active()); + + // Convert the futures into tasks. + let tasks = futures.into_iter().enumerate().map(move |(i, future)| { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: `T` and the future are `Send`. + let task = unsafe { self.spawn_inner(future, active.as_mut().unwrap()) }; + + // Yield the lock every once in a while to ease contention. + if i.wrapping_sub(1) % 500 == 0 { + drop(active.take()); + active = Some(self.state().active()); + } + + task + }); + + // Push the tasks to the user's collection. + handles.extend(tasks); + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + /// Spawn a future while holding the inner lock. + /// + /// # Safety + /// + /// If this is an `Executor`, `F` and `T` must be `Send`. + unsafe fn spawn_inner( + &self, + future: impl Future + 'a, + active: &mut Slab, + ) -> Task { + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state_as_arc(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()); + entry.insert(runnable.waker()); + + runnable.schedule(); + task + } + + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// assert!(!ex.try_tick()); // no tasks to run + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(ex.try_tick()); // a task was found + /// ``` + pub fn try_tick(&self) -> bool { + self.state().try_tick() + } + + /// Runs a single task. + /// + /// Running a task means simply polling its future once. + /// + /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::future; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// future::block_on(ex.tick()); // runs the task + /// ``` + pub async fn tick(&self) { + self.state().tick().await; + } + + /// Runs the executor until the given future completes. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::future; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { 1 + 2 }); + /// let res = future::block_on(ex.run(async { task.await * 2 })); + /// + /// assert_eq!(res, 6); + /// ``` + pub async fn run(&self, future: impl Future) -> T { + self.state().run(future).await + } + + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { + let state = self.state_as_arc(); + + // TODO: If possible, push into the current local queue and notify the ticker. + move |runnable| { + state.queue.push(runnable).unwrap(); + state.notify(); + } + } + + /// Returns a pointer to the inner state. + #[inline] + fn state_ptr(&self) -> *const State { + #[cold] + fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { + let state = Arc::new(State::new()); + // TODO: Switch this to use cast_mut once the MSRV can be bumped past 1.65 + let ptr = Arc::into_raw(state) as *mut State; + if let Err(actual) = atomic_ptr.compare_exchange( + std::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This was just created from Arc::into_raw. + drop(unsafe { Arc::from_raw(ptr) }); + actual + } else { + ptr + } + } + + let mut ptr = self.state.load(Ordering::Acquire); + if ptr.is_null() { + ptr = alloc_state(&self.state); + } + ptr + } + + /// Returns a reference to the inner state. + #[inline] + fn state(&self) -> &State { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: So long as an Executor lives, it's state pointer will always be valid + // when accessed through state_ptr. + unsafe { &*self.state_ptr() } + } + + // Clones the inner state Arc + #[inline] + fn state_as_arc(&self) -> Arc { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: So long as an Executor lives, it's state pointer will always be a valid + // Arc when accessed through state_ptr. + let arc = unsafe { Arc::from_raw(self.state_ptr()) }; + let clone = arc.clone(); + std::mem::forget(arc); + clone + } +} + +impl Drop for Executor<'_> { + fn drop(&mut self) { + let ptr = *self.state.get_mut(); + if ptr.is_null() { + return; + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: As ptr is not null, it was allocated via Arc::new and converted + // via Arc::into_raw in state_ptr. + let state = unsafe { Arc::from_raw(ptr) }; + + let mut active = state.active(); + for w in active.drain() { + w.wake(); + } + drop(active); + + while state.queue.pop().is_ok() {} + } +} + +impl<'a> Default for Executor<'a> { + fn default() -> Executor<'a> { + Executor::new() + } +} + +/// A thread-local executor. +/// +/// The executor can only be run on the thread that created it. +/// +/// # Examples +/// +/// ``` +/// use async_executor::LocalExecutor; +/// use futures_lite::future; +/// +/// let local_ex = LocalExecutor::new(); +/// +/// future::block_on(local_ex.run(async { +/// println!("Hello world!"); +/// })); +/// ``` +pub struct LocalExecutor<'a> { + /// The inner executor. + inner: Executor<'a>, + + /// Makes the type `!Send` and `!Sync`. + _marker: PhantomData>, +} + +impl UnwindSafe for LocalExecutor<'_> {} +impl RefUnwindSafe for LocalExecutor<'_> {} + +impl fmt::Debug for LocalExecutor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_executor(&self.inner, "LocalExecutor", f) + } +} + +impl<'a> LocalExecutor<'a> { + /// Creates a single-threaded executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// ``` + pub const fn new() -> LocalExecutor<'a> { + LocalExecutor { + inner: Executor::new(), + _marker: PhantomData, + } + } + + /// Returns `true` if there are no unfinished tasks. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// assert!(local_ex.is_empty()); + /// + /// let task = local_ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(!local_ex.is_empty()); + /// + /// assert!(local_ex.try_tick()); + /// assert!(local_ex.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.inner().is_empty() + } + + /// Spawns a task onto the executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// + /// let task = local_ex.spawn(async { + /// println!("Hello world"); + /// }); + /// ``` + pub fn spawn(&self, future: impl Future + 'a) -> Task { + let mut active = self.inner().state().active(); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This executor is not thread safe, so the future and its result + // cannot be sent to another thread. + unsafe { self.inner().spawn_inner(future, &mut active) } + } + + /// Spawns many tasks onto the executor. + /// + /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and + /// spawns all of the tasks in one go. With large amounts of tasks this can improve + /// contention. + /// + /// It is assumed that the iterator provided does not block; blocking iterators can lock up + /// the internal mutex and therefore the entire executor. Unlike [`Executor::spawn`], the + /// mutex is not released, as there are no other threads that can poll this executor. + /// + /// ## Example + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::{stream, prelude::*}; + /// use std::future::ready; + /// + /// # futures_lite::future::block_on(async { + /// let mut ex = LocalExecutor::new(); + /// + /// let futures = [ + /// ready(1), + /// ready(2), + /// ready(3) + /// ]; + /// + /// // Spawn all of the futures onto the executor at once. + /// let mut tasks = vec![]; + /// ex.spawn_many(futures, &mut tasks); + /// + /// // Await all of them. + /// let results = ex.run(async move { + /// stream::iter(tasks).then(|x| x).collect::>().await + /// }).await; + /// assert_eq!(results, [1, 2, 3]); + /// # }); + /// ``` + /// + /// [`spawn`]: LocalExecutor::spawn + /// [`Executor::spawn_many`]: Executor::spawn_many + pub fn spawn_many + 'a>( + &self, + futures: impl IntoIterator, + handles: &mut impl Extend>, + ) { + let mut active = self.inner().state().active(); + + // Convert all of the futures to tasks. + let tasks = futures.into_iter().map(|future| { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This executor is not thread safe, so the future and its result + // cannot be sent to another thread. + unsafe { self.inner().spawn_inner(future, &mut active) } + + // As only one thread can spawn or poll tasks at a time, there is no need + // to release lock contention here. + }); + + // Push them to the user's collection. + handles.extend(tasks); + } + + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let ex = LocalExecutor::new(); + /// assert!(!ex.try_tick()); // no tasks to run + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(ex.try_tick()); // a task was found + /// ``` + pub fn try_tick(&self) -> bool { + self.inner().try_tick() + } + + /// Runs a single task. + /// + /// Running a task means simply polling its future once. + /// + /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::future; + /// + /// let ex = LocalExecutor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// future::block_on(ex.tick()); // runs the task + /// ``` + pub async fn tick(&self) { + self.inner().tick().await + } + + /// Runs the executor until the given future completes. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::future; + /// + /// let local_ex = LocalExecutor::new(); + /// + /// let task = local_ex.spawn(async { 1 + 2 }); + /// let res = future::block_on(local_ex.run(async { task.await * 2 })); + /// + /// assert_eq!(res, 6); + /// ``` + pub async fn run(&self, future: impl Future) -> T { + self.inner().run(future).await + } + + /// Returns a reference to the inner executor. + fn inner(&self) -> &Executor<'a> { + &self.inner + } +} + +impl<'a> Default for LocalExecutor<'a> { + fn default() -> LocalExecutor<'a> { + LocalExecutor::new() + } +} + +/// The state of a executor. +struct State { + /// The global queue. + queue: ConcurrentQueue, + + /// Local queues created by runners. + local_queues: RwLock>>>, + + /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. + notified: AtomicBool, + + /// A list of sleeping tickers. + sleepers: Mutex, + + /// Currently active tasks. + active: Mutex>, +} + +impl State { + /// Creates state for a new executor. + const fn new() -> State { + State { + queue: ConcurrentQueue::unbounded(), + local_queues: RwLock::new(Vec::new()), + notified: AtomicBool::new(true), + sleepers: Mutex::new(Sleepers { + count: 0, + wakers: Vec::new(), + free_ids: Vec::new(), + }), + active: Mutex::new(Slab::new()), + } + } + + /// Returns a reference to currently active tasks. + fn active(&self) -> MutexGuard<'_, Slab> { + self.active.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Notifies a sleeping ticker. + #[inline] + fn notify(&self) { + if self + .notified + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + let waker = self.sleepers.lock().unwrap().notify(); + if let Some(w) = waker { + w.wake(); + } + } + } + + pub(crate) fn try_tick(&self) -> bool { + match self.queue.pop() { + Err(_) => false, + Ok(runnable) => { + // Notify another ticker now to pick up where this ticker left off, just in case + // running the task takes a long time. + self.notify(); + + // Run the task. + runnable.run(); + true + } + } + } + + pub(crate) async fn tick(&self) { + let runnable = Ticker::new(self).runnable().await; + runnable.run(); + } + + pub async fn run(&self, future: impl Future) -> T { + let mut runner = Runner::new(self); + let mut rng = fastrand::Rng::new(); + + // A future that runs tasks forever. + let run_forever = async { + loop { + for _ in 0..200 { + let runnable = runner.runnable(&mut rng).await; + runnable.run(); + } + future::yield_now().await; + } + }; + + // Run `future` and `run_forever` concurrently until `future` completes. + future.or(run_forever).await + } +} + +/// A list of sleeping tickers. +struct Sleepers { + /// Number of sleeping tickers (both notified and unnotified). + count: usize, + + /// IDs and wakers of sleeping unnotified tickers. + /// + /// A sleeping ticker is notified when its waker is missing from this list. + wakers: Vec<(usize, Waker)>, + + /// Reclaimed IDs. + free_ids: Vec, +} + +impl Sleepers { + /// Inserts a new sleeping ticker. + fn insert(&mut self, waker: &Waker) -> usize { + let id = match self.free_ids.pop() { + Some(id) => id, + None => self.count + 1, + }; + self.count += 1; + self.wakers.push((id, waker.clone())); + id + } + + /// Re-inserts a sleeping ticker's waker if it was notified. + /// + /// Returns `true` if the ticker was notified. + fn update(&mut self, id: usize, waker: &Waker) -> bool { + for item in &mut self.wakers { + if item.0 == id { + item.1.clone_from(waker); + return false; + } + } + + self.wakers.push((id, waker.clone())); + true + } + + /// Removes a previously inserted sleeping ticker. + /// + /// Returns `true` if the ticker was notified. + fn remove(&mut self, id: usize) -> bool { + self.count -= 1; + self.free_ids.push(id); + + for i in (0..self.wakers.len()).rev() { + if self.wakers[i].0 == id { + self.wakers.remove(i); + return false; + } + } + true + } + + /// Returns `true` if a sleeping ticker is notified or no tickers are sleeping. + fn is_notified(&self) -> bool { + self.count == 0 || self.count > self.wakers.len() + } + + /// Returns notification waker for a sleeping ticker. + /// + /// If a ticker was notified already or there are no tickers, `None` will be returned. + fn notify(&mut self) -> Option { + if self.wakers.len() == self.count { + self.wakers.pop().map(|item| item.1) + } else { + None + } + } +} + +/// Runs task one by one. +struct Ticker<'a> { + /// The executor state. + state: &'a State, + + /// Set to a non-zero sleeper ID when in sleeping state. + /// + /// States a ticker can be in: + /// 1) Woken. + /// 2a) Sleeping and unnotified. + /// 2b) Sleeping and notified. + sleeping: usize, +} + +impl Ticker<'_> { + /// Creates a ticker. + fn new(state: &State) -> Ticker<'_> { + Ticker { state, sleeping: 0 } + } + + /// Moves the ticker into sleeping and unnotified state. + /// + /// Returns `false` if the ticker was already sleeping and unnotified. + fn sleep(&mut self, waker: &Waker) -> bool { + let mut sleepers = self.state.sleepers.lock().unwrap(); + + match self.sleeping { + // Move to sleeping state. + 0 => { + self.sleeping = sleepers.insert(waker); + } + + // Already sleeping, check if notified. + id => { + if !sleepers.update(id, waker) { + return false; + } + } + } + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + + true + } + + /// Moves the ticker into woken state. + fn wake(&mut self) { + if self.sleeping != 0 { + let mut sleepers = self.state.sleepers.lock().unwrap(); + sleepers.remove(self.sleeping); + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + } + self.sleeping = 0; + } + + /// Waits for the next runnable task to run. + async fn runnable(&mut self) -> Runnable { + self.runnable_with(|| self.state.queue.pop().ok()).await + } + + /// Waits for the next runnable task to run, given a function that searches for a task. + async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { + future::poll_fn(|cx| { + loop { + match search() { + None => { + // Move to sleeping and unnotified state. + if !self.sleep(cx.waker()) { + // If already sleeping and unnotified, return. + return Poll::Pending; + } + } + Some(r) => { + // Wake up. + self.wake(); + + // Notify another ticker now to pick up where this ticker left off, just in + // case running the task takes a long time. + self.state.notify(); + + return Poll::Ready(r); + } + } + } + }) + .await + } +} + +impl Drop for Ticker<'_> { + fn drop(&mut self) { + // If this ticker is in sleeping state, it must be removed from the sleepers list. + if self.sleeping != 0 { + let mut sleepers = self.state.sleepers.lock().unwrap(); + let notified = sleepers.remove(self.sleeping); + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + + // If this ticker was notified, then notify another ticker. + if notified { + drop(sleepers); + self.state.notify(); + } + } + } +} + +/// A worker in a work-stealing executor. +/// +/// This is just a ticker that also has an associated local queue for improved cache locality. +struct Runner<'a> { + /// The executor state. + state: &'a State, + + /// Inner ticker. + ticker: Ticker<'a>, + + /// The local queue. + local: Arc>, + + /// Bumped every time a runnable task is found. + ticks: usize, +} + +impl Runner<'_> { + /// Creates a runner and registers it in the executor state. + fn new(state: &State) -> Runner<'_> { + let runner = Runner { + state, + ticker: Ticker::new(state), + local: Arc::new(ConcurrentQueue::bounded(512)), + ticks: 0, + }; + state + .local_queues + .write() + .unwrap() + .push(runner.local.clone()); + runner + } + + /// Waits for the next runnable task to run. + async fn runnable(&mut self, rng: &mut fastrand::Rng) -> Runnable { + let runnable = self + .ticker + .runnable_with(|| { + // Try the local queue. + if let Ok(r) = self.local.pop() { + return Some(r); + } + + // Try stealing from the global queue. + if let Ok(r) = self.state.queue.pop() { + steal(&self.state.queue, &self.local); + return Some(r); + } + + // Try stealing from other runners. + let local_queues = self.state.local_queues.read().unwrap(); + + // Pick a random starting point in the iterator list and rotate the list. + let n = local_queues.len(); + let start = rng.usize(..n); + let iter = local_queues + .iter() + .chain(local_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = iter.filter(|local| !Arc::ptr_eq(local, &self.local)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(local, &self.local); + if let Ok(r) = self.local.pop() { + return Some(r); + } + } + + None + }) + .await; + + // Bump the tick counter. + self.ticks = self.ticks.wrapping_add(1); + + if self.ticks % 64 == 0 { + // Steal tasks from the global queue to ensure fair task scheduling. + steal(&self.state.queue, &self.local); + } + + runnable + } +} + +impl Drop for Runner<'_> { + fn drop(&mut self) { + // Remove the local queue. + self.state + .local_queues + .write() + .unwrap() + .retain(|local| !Arc::ptr_eq(local, &self.local)); + + // Re-schedule remaining tasks in the local queue. + while let Ok(r) = self.local.pop() { + r.schedule(); + } + } +} + +/// Steals some items from one queue into another. +fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { + // Half of `src`'s length rounded up. + let mut count = (src.len() + 1) / 2; + + if count > 0 { + // Don't steal more than fits into the queue. + if let Some(cap) = dest.capacity() { + count = count.min(cap - dest.len()); + } + + // Steal tasks. + for _ in 0..count { + if let Ok(t) = src.pop() { + assert!(dest.push(t).is_ok()); + } else { + break; + } + } + } +} + +/// Debug implementation for `Executor` and `LocalExecutor`. +fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Get a reference to the state. + let ptr = executor.state.load(Ordering::Acquire); + if ptr.is_null() { + // The executor has not been initialized. + struct Uninitialized; + + impl fmt::Debug for Uninitialized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + + return f.debug_tuple(name).field(&Uninitialized).finish(); + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: If the state pointer is not null, it must have been + // allocated properly by Arc::new and converted via Arc::into_raw + // in state_ptr. + let state = unsafe { &*ptr }; + + debug_state(state, name, f) +} + +/// Debug implementation for `Executor` and `LocalExecutor`. +fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// Debug wrapper for the number of active tasks. + struct ActiveTasks<'a>(&'a Mutex>); + + impl fmt::Debug for ActiveTasks<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_lock() { + Ok(lock) => fmt::Debug::fmt(&lock.len(), f), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(err)) => fmt::Debug::fmt(&err.into_inner().len(), f), + } + } + } + + /// Debug wrapper for the local runners. + struct LocalRunners<'a>(&'a RwLock>>>); + + impl fmt::Debug for LocalRunners<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_read() { + Ok(lock) => f + .debug_list() + .entries(lock.iter().map(|queue| queue.len())) + .finish(), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(_)) => f.write_str(""), + } + } + } + + /// Debug wrapper for the sleepers. + struct SleepCount<'a>(&'a Mutex); + + impl fmt::Debug for SleepCount<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_lock() { + Ok(lock) => fmt::Debug::fmt(&lock.count, f), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(_)) => f.write_str(""), + } + } + } + + f.debug_struct(name) + .field("active", &ActiveTasks(&state.active)) + .field("global_tasks", &state.queue.len()) + .field("local_runners", &LocalRunners(&state.local_queues)) + .field("sleepers", &SleepCount(&state.sleepers)) + .finish() +} + +/// Runs a closure when dropped. +struct CallOnDrop(F); + +impl Drop for CallOnDrop { + fn drop(&mut self) { + (self.0)(); + } +} + +pin_project! { + /// A wrapper around a future, running a closure when dropped. + struct AsyncCallOnDrop { + #[pin] + future: Fut, + cleanup: CallOnDrop, + } +} + +impl AsyncCallOnDrop { + fn new(future: Fut, cleanup: Cleanup) -> Self { + Self { + future, + cleanup: CallOnDrop(cleanup), + } + } +} + +impl Future for AsyncCallOnDrop { + type Output = Fut::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.project().future.poll(cx) + } +} + +fn _ensure_send_and_sync() { + use futures_lite::future::pending; + + fn is_send(_: T) {} + fn is_sync(_: T) {} + fn is_static(_: T) {} + + is_send::>(Executor::new()); + is_sync::>(Executor::new()); + + let ex = Executor::new(); + is_send(ex.run(pending::<()>())); + is_sync(ex.run(pending::<()>())); + is_send(ex.tick()); + is_sync(ex.tick()); + is_send(ex.schedule()); + is_sync(ex.schedule()); + is_static(ex.schedule()); + + /// ```compile_fail + /// use async_executor::LocalExecutor; + /// use futures_lite::future::pending; + /// + /// fn is_send(_: T) {} + /// fn is_sync(_: T) {} + /// + /// is_send::>(LocalExecutor::new()); + /// is_sync::>(LocalExecutor::new()); + /// + /// let ex = LocalExecutor::new(); + /// is_send(ex.run(pending::<()>())); + /// is_sync(ex.run(pending::<()>())); + /// is_send(ex.tick()); + /// is_sync(ex.tick()); + /// ``` + fn _negative_test() {} +} diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 01bbe4a669258..deb8f2fca3cf7 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -15,9 +15,9 @@ use core::{ use derive_more::{Deref, DerefMut}; cfg_if::cfg_if! { - if #[cfg(feature = "async_executor")] { - type ExecutorInner<'a> = async_executor::Executor<'a>; - type LocalExecutorInner<'a> = async_executor::LocalExecutor<'a>; + if #[cfg(feature = "std")] { + type ExecutorInner<'a> = crate::async_executor::Executor<'a>; + type LocalExecutorInner<'a> = crate::async_executor::LocalExecutor<'a>; } else { type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; type LocalExecutorInner<'a> = crate::edge_executor::LocalExecutor<'a, 64>; diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 66899ef36f095..4f23e7954bf36 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -42,7 +42,9 @@ pub type BoxedFuture<'a, T> = core::pin::Pin Date: Sat, 26 Jul 2025 15:18:43 -0700 Subject: [PATCH 02/68] Merge LocalExecutor into the Executor implementation --- crates/bevy_tasks/Cargo.toml | 2 + crates/bevy_tasks/src/async_executor.rs | 608 +++++------------- crates/bevy_tasks/src/executor.rs | 32 +- .../src/single_threaded_task_pool.rs | 67 +- crates/bevy_tasks/src/task_pool.rs | 55 +- crates/bevy_tasks/src/usages.rs | 26 +- 6 files changed, 200 insertions(+), 590 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 1c43f917a6357..a27379e05b08c 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -33,6 +33,7 @@ std = [ "bevy_platform/std", "fastrand/std", "dep:slab", + "dep:thread_local", "dep:concurrent-queue", "dep:pin-project-lite", ] @@ -65,6 +66,7 @@ derive_more = { version = "2", default-features = false, features = [ cfg-if = "1.0.0" slab = { version = "0.4", optional = true } pin-project-lite = { version = "0.2", optional = true } +thread_local = { version = "1.1", optional = true } fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 1aad2a6c133ad..8f862f8f0e662 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -1,43 +1,30 @@ +use std::cell::RefCell; +use std::collections::VecDeque; use std::fmt; use std::marker::PhantomData; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; -use std::rc::Rc; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; -use async_task::{Builder, Runnable}; +use async_task::{Builder, Runnable, Task}; +use bevy_platform::prelude::Vec; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; -use bevy_platform::prelude::Vec; use slab::Slab; +use thread_local::ThreadLocal; + +static LOCAL_QUEUE: ThreadLocal> = ThreadLocal::new(); + +#[derive(Default)] +struct LocalQueue { + queue: VecDeque, + active: Slab, +} /// An async executor. -/// -/// # Examples -/// -/// A multi-threaded executor: -/// -/// ``` -/// use async_channel::unbounded; -/// use async_executor::Executor; -/// use easy_parallel::Parallel; -/// use futures_lite::future; -/// -/// let ex = Executor::new(); -/// let (signal, shutdown) = unbounded::<()>(); -/// -/// Parallel::new() -/// // Run four executor threads. -/// .each(0..4, |_| future::block_on(ex.run(shutdown.recv()))) -/// // Run the main future on the current thread. -/// .finish(|| future::block_on(async { -/// println!("Hello world!"); -/// drop(signal); -/// })); -/// ``` pub struct Executor<'a> { /// The executor state. state: AtomicPtr, @@ -70,14 +57,6 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// ``` pub const fn new() -> Executor<'a> { Executor { state: AtomicPtr::new(std::ptr::null_mut()), @@ -85,141 +64,68 @@ impl<'a> Executor<'a> { } } - /// Returns `true` if there are no unfinished tasks. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// assert!(ex.is_empty()); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(!ex.is_empty()); - /// - /// assert!(ex.try_tick()); - /// assert!(ex.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.state().active().is_empty() - } - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { let mut active = self.state().active(); + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state_as_arc(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + #[expect( unsafe_code, reason = "unsized coercion is an unstable feature for non-std types" )] - // SAFETY: `T` and the future are `Send`. - unsafe { self.spawn_inner(future, &mut active) } - } - - /// Spawns many tasks onto the executor. - /// - /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and - /// spawns all of the tasks in one go. With large amounts of tasks this can improve - /// contention. - /// - /// For very large numbers of tasks the lock is occasionally dropped and re-acquired to - /// prevent runner thread starvation. It is assumed that the iterator provided does not - /// block; blocking iterators can lock up the internal mutex and therefore the entire - /// executor. - /// - /// ## Example - /// - /// ``` - /// use async_executor::Executor; - /// use futures_lite::{stream, prelude::*}; - /// use std::future::ready; - /// - /// # futures_lite::future::block_on(async { - /// let mut ex = Executor::new(); - /// - /// let futures = [ - /// ready(1), - /// ready(2), - /// ready(3) - /// ]; - /// - /// // Spawn all of the futures onto the executor at once. - /// let mut tasks = vec![]; - /// ex.spawn_many(futures, &mut tasks); - /// - /// // Await all of them. - /// let results = ex.run(async move { - /// stream::iter(tasks).then(|x| x).collect::>().await - /// }).await; - /// assert_eq!(results, [1, 2, 3]); - /// # }); - /// ``` - /// - /// [`spawn`]: Executor::spawn - pub fn spawn_many + Send + 'a>( - &self, - futures: impl IntoIterator, - handles: &mut impl Extend>, - ) { - let mut active = Some(self.state().active()); - - // Convert the futures into tasks. - let tasks = futures.into_iter().enumerate().map(move |(i, future)| { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: `T` and the future are `Send`. - let task = unsafe { self.spawn_inner(future, active.as_mut().unwrap()) }; - - // Yield the lock every once in a while to ease contention. - if i.wrapping_sub(1) % 500 == 0 { - drop(active.take()); - active = Some(self.state().active()); - } - - task - }); + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()) + }; + entry.insert(runnable.waker()); - // Push the tasks to the user's collection. - handles.extend(tasks); + runnable.schedule(); + task } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - /// Spawn a future while holding the inner lock. - /// - /// # Safety - /// - /// If this is an `Executor`, `F` and `T` must be `Send`. - unsafe fn spawn_inner( - &self, - future: impl Future + 'a, - active: &mut Slab, - ) -> Task { + /// Spawns a non-Send task onto the executor. + pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let entry = active.vacant_entry(); + let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + let mut local_state = local_queue.borrow_mut(); + let entry = local_state.active.vacant_entry(); let index = entry.key(); - let state = self.state_as_arc(); - let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + let future = AsyncCallOnDrop::new(future, move || { + drop(local_queue.borrow_mut().active.try_remove(index)) + }); + #[expect( + unsafe_code, + reason = "Builder::spawn_local requires a 'static lifetime" + )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -242,11 +148,15 @@ impl<'a> Executor<'a> { // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. // Therefore we do not need to worry about what is done with the // `Waker`. - let (runnable, task) = Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule()); + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule_local()) + }; entry.insert(runnable.waker()); + drop(local_state); + runnable.schedule(); task } @@ -258,7 +168,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// /// let ex = Executor::new(); /// assert!(!ex.try_tick()); // no tasks to run @@ -272,6 +182,16 @@ impl<'a> Executor<'a> { self.state().try_tick() } + pub fn try_tick_local() -> bool { + if let Some(runnable) = LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back() { + // Run the task. + runnable.run(); + true + } else { + false + } + } + /// Runs a single task. /// /// Running a task means simply polling its future once. @@ -281,7 +201,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// use futures_lite::future; /// /// let ex = Executor::new(); @@ -300,7 +220,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// use futures_lite::future; /// /// let ex = Executor::new(); @@ -325,6 +245,16 @@ impl<'a> Executor<'a> { } } + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule_local(&self) -> impl Fn(Runnable) + 'static { + let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + // TODO: If possible, push into the current local queue and notify the ticker. + move |runnable| { + local_queue.borrow_mut().queue.push_back(runnable); + // state.notify(); + } + } + /// Returns a pointer to the inner state. #[inline] fn state_ptr(&self) -> *const State { @@ -367,7 +297,9 @@ impl<'a> Executor<'a> { )] // SAFETY: So long as an Executor lives, it's state pointer will always be valid // when accessed through state_ptr. - unsafe { &*self.state_ptr() } + unsafe { + &*self.state_ptr() + } } // Clones the inner state Arc @@ -417,243 +349,6 @@ impl<'a> Default for Executor<'a> { } } -/// A thread-local executor. -/// -/// The executor can only be run on the thread that created it. -/// -/// # Examples -/// -/// ``` -/// use async_executor::LocalExecutor; -/// use futures_lite::future; -/// -/// let local_ex = LocalExecutor::new(); -/// -/// future::block_on(local_ex.run(async { -/// println!("Hello world!"); -/// })); -/// ``` -pub struct LocalExecutor<'a> { - /// The inner executor. - inner: Executor<'a>, - - /// Makes the type `!Send` and `!Sync`. - _marker: PhantomData>, -} - -impl UnwindSafe for LocalExecutor<'_> {} -impl RefUnwindSafe for LocalExecutor<'_> {} - -impl fmt::Debug for LocalExecutor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - debug_executor(&self.inner, "LocalExecutor", f) - } -} - -impl<'a> LocalExecutor<'a> { - /// Creates a single-threaded executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// ``` - pub const fn new() -> LocalExecutor<'a> { - LocalExecutor { - inner: Executor::new(), - _marker: PhantomData, - } - } - - /// Returns `true` if there are no unfinished tasks. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// assert!(local_ex.is_empty()); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(!local_ex.is_empty()); - /// - /// assert!(local_ex.try_tick()); - /// assert!(local_ex.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.inner().is_empty() - } - - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` - pub fn spawn(&self, future: impl Future + 'a) -> Task { - let mut active = self.inner().state().active(); - - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: This executor is not thread safe, so the future and its result - // cannot be sent to another thread. - unsafe { self.inner().spawn_inner(future, &mut active) } - } - - /// Spawns many tasks onto the executor. - /// - /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and - /// spawns all of the tasks in one go. With large amounts of tasks this can improve - /// contention. - /// - /// It is assumed that the iterator provided does not block; blocking iterators can lock up - /// the internal mutex and therefore the entire executor. Unlike [`Executor::spawn`], the - /// mutex is not released, as there are no other threads that can poll this executor. - /// - /// ## Example - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::{stream, prelude::*}; - /// use std::future::ready; - /// - /// # futures_lite::future::block_on(async { - /// let mut ex = LocalExecutor::new(); - /// - /// let futures = [ - /// ready(1), - /// ready(2), - /// ready(3) - /// ]; - /// - /// // Spawn all of the futures onto the executor at once. - /// let mut tasks = vec![]; - /// ex.spawn_many(futures, &mut tasks); - /// - /// // Await all of them. - /// let results = ex.run(async move { - /// stream::iter(tasks).then(|x| x).collect::>().await - /// }).await; - /// assert_eq!(results, [1, 2, 3]); - /// # }); - /// ``` - /// - /// [`spawn`]: LocalExecutor::spawn - /// [`Executor::spawn_many`]: Executor::spawn_many - pub fn spawn_many + 'a>( - &self, - futures: impl IntoIterator, - handles: &mut impl Extend>, - ) { - let mut active = self.inner().state().active(); - - // Convert all of the futures to tasks. - let tasks = futures.into_iter().map(|future| { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: This executor is not thread safe, so the future and its result - // cannot be sent to another thread. - unsafe { self.inner().spawn_inner(future, &mut active) } - - // As only one thread can spawn or poll tasks at a time, there is no need - // to release lock contention here. - }); - - // Push them to the user's collection. - handles.extend(tasks); - } - - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let ex = LocalExecutor::new(); - /// assert!(!ex.try_tick()); // no tasks to run - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.inner().try_tick() - } - - /// Runs a single task. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::future; - /// - /// let ex = LocalExecutor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// future::block_on(ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.inner().tick().await - } - - /// Runs the executor until the given future completes. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::future; - /// - /// let local_ex = LocalExecutor::new(); - /// - /// let task = local_ex.spawn(async { 1 + 2 }); - /// let res = future::block_on(local_ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` - pub async fn run(&self, future: impl Future) -> T { - self.inner().run(future).await - } - - /// Returns a reference to the inner executor. - fn inner(&self) -> &Executor<'a> { - &self.inner - } -} - -impl<'a> Default for LocalExecutor<'a> { - fn default() -> LocalExecutor<'a> { - LocalExecutor::new() - } -} - /// The state of a executor. struct State { /// The global queue. @@ -709,17 +404,21 @@ impl State { } pub(crate) fn try_tick(&self) -> bool { - match self.queue.pop() { - Err(_) => false, - Ok(runnable) => { - // Notify another ticker now to pick up where this ticker left off, just in case - // running the task takes a long time. - self.notify(); - - // Run the task. - runnable.run(); - true - } + let runnable = self + .queue + .pop() + .ok() + .or_else(|| LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back()); + if let Some(runnable) = runnable { + // Notify another ticker now to pick up where this ticker left off, just in case + // running the task takes a long time. + self.notify(); + + // Run the task. + runnable.run(); + true + } else { + false } } @@ -884,7 +583,15 @@ impl Ticker<'_> { /// Waits for the next runnable task to run. async fn runnable(&mut self) -> Runnable { - self.runnable_with(|| self.state.queue.pop().ok()).await + self.runnable_with(|| { + LOCAL_QUEUE + .get_or_default() + .borrow_mut() + .queue + .pop_back() + .or_else(|| self.state.queue.pop().ok()) + }) + .await } /// Waits for the next runnable task to run, given a function that searches for a task. @@ -951,6 +658,9 @@ struct Runner<'a> { /// Bumped every time a runnable task is found. ticks: usize, + + // The thread local state of the executor for the current thread. + local_state: &'static RefCell, } impl Runner<'_> { @@ -961,6 +671,7 @@ impl Runner<'_> { ticker: Ticker::new(state), local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, + local_state: LOCAL_QUEUE.get_or_default(), }; state .local_queues @@ -975,6 +686,10 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { + if let Some(r) = self.local_state.borrow_mut().queue.pop_back() { + return Some(r); + } + // Try the local queue. if let Ok(r) = self.local.pop() { return Some(r); @@ -1179,40 +894,39 @@ impl Future for AsyncCallOnDrop { } } -fn _ensure_send_and_sync() { - use futures_lite::future::pending; - - fn is_send(_: T) {} - fn is_sync(_: T) {} - fn is_static(_: T) {} - - is_send::>(Executor::new()); - is_sync::>(Executor::new()); - - let ex = Executor::new(); - is_send(ex.run(pending::<()>())); - is_sync(ex.run(pending::<()>())); - is_send(ex.tick()); - is_sync(ex.tick()); - is_send(ex.schedule()); - is_sync(ex.schedule()); - is_static(ex.schedule()); - - /// ```compile_fail - /// use async_executor::LocalExecutor; - /// use futures_lite::future::pending; - /// - /// fn is_send(_: T) {} - /// fn is_sync(_: T) {} - /// - /// is_send::>(LocalExecutor::new()); - /// is_sync::>(LocalExecutor::new()); - /// - /// let ex = LocalExecutor::new(); - /// is_send(ex.run(pending::<()>())); - /// is_sync(ex.run(pending::<()>())); - /// is_send(ex.tick()); - /// is_sync(ex.tick()); - /// ``` - fn _negative_test() {} +#[cfg(test)] +mod test { + fn _ensure_send_and_sync() { + fn is_send(_: T) {} + fn is_sync(_: T) {} + fn is_static(_: T) {} + + is_send::>(Executor::new()); + is_sync::>(Executor::new()); + + let ex = Executor::new(); + is_send(ex.tick()); + is_sync(ex.tick()); + is_send(ex.schedule()); + is_sync(ex.schedule()); + is_static(ex.schedule()); + + /// ```compile_fail + /// use crate::async_executor::LocalExecutor; + /// use futures_lite::future::pending; + /// + /// fn is_send(_: T) {} + /// fn is_sync(_: T) {} + /// + /// is_send::>(LocalExecutor::new()); + /// is_sync::>(LocalExecutor::new()); + /// + /// let ex = LocalExecutor::new(); + /// is_send(ex.run(pending::<()>())); + /// is_sync(ex.run(pending::<()>())); + /// is_send(ex.tick()); + /// is_sync(ex.tick()); + /// ``` + fn _negative_test() {} + } } diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index deb8f2fca3cf7..2c7e142a0a623 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -17,10 +17,8 @@ use derive_more::{Deref, DerefMut}; cfg_if::cfg_if! { if #[cfg(feature = "std")] { type ExecutorInner<'a> = crate::async_executor::Executor<'a>; - type LocalExecutorInner<'a> = crate::async_executor::LocalExecutor<'a>; } else { type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; - type LocalExecutorInner<'a> = crate::edge_executor::LocalExecutor<'a, 64>; } } @@ -36,16 +34,6 @@ pub use async_task::FallibleTask; #[derive(Deref, DerefMut, Default)] pub struct Executor<'a>(ExecutorInner<'a>); -/// Wrapper around a single-threaded async executor. -/// Spawning wont generally require tasks to be `Send` and `Sync`, at the cost of -/// this executor itself not being `Send` or `Sync`. This makes it unsuitable for -/// global statics. -/// -/// If need to store an executor in a global static, or send across threads, -/// consider using [`Executor`] instead. -#[derive(Deref, DerefMut, Default)] -pub struct LocalExecutor<'a>(LocalExecutorInner<'a>); - impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] @@ -53,14 +41,10 @@ impl Executor<'_> { pub const fn new() -> Self { Self(ExecutorInner::new()) } -} -impl LocalExecutor<'_> { - /// Construct a new [`LocalExecutor`] - #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] - #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { - Self(LocalExecutorInner::new()) + #[inline] + pub fn try_tick_local() -> bool { + ExecutorInner::try_tick_local() } } @@ -68,18 +52,8 @@ impl UnwindSafe for Executor<'_> {} impl RefUnwindSafe for Executor<'_> {} -impl UnwindSafe for LocalExecutor<'_> {} - -impl RefUnwindSafe for LocalExecutor<'_> {} - impl fmt::Debug for Executor<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Executor").finish() } } - -impl fmt::Debug for LocalExecutor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("LocalExecutor").finish() - } -} diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 0f9488bcd0f33..33dd8f3c0acd5 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -4,25 +4,12 @@ use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; -#[cfg(feature = "std")] -use std::thread_local; - #[cfg(not(feature = "std"))] use bevy_platform::sync::{Mutex, PoisonError}; -#[cfg(feature = "std")] -use crate::executor::LocalExecutor; - -#[cfg(not(feature = "std"))] -use crate::executor::Executor as LocalExecutor; +use crate::executor::Executor; -#[cfg(feature = "std")] -thread_local! { - static LOCAL_EXECUTOR: LocalExecutor<'static> = const { LocalExecutor::new() }; -} - -#[cfg(not(feature = "std"))] -static LOCAL_EXECUTOR: LocalExecutor<'static> = const { LocalExecutor::new() }; +static EXECUTOR: Executor = Executor::new(); #[cfg(feature = "std")] type ScopeResult = alloc::rc::Rc>>; @@ -147,9 +134,9 @@ impl TaskPool { // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor = &LocalExecutor::new(); + let executor = &Executor::new(); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor: &'env LocalExecutor<'env> = unsafe { mem::transmute(executor) }; + let executor: &'env Executor<'env> = unsafe { mem::transmute(executor) }; let results: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -203,21 +190,17 @@ impl TaskPool { if #[cfg(all(target_arch = "wasm32", feature = "web"))] { Task::wrap_future(future) } else if #[cfg(feature = "std")] { - LOCAL_EXECUTOR.with(|executor| { - let task = executor.spawn(future); - // Loop until all tasks are done - while executor.try_tick() {} + let task = EXECUTOR.spawn_local(future); + // Loop until all tasks are done + while EXECUTOR.try_tick() {} - Task::new(task) - }) + Task::new(task) } else { - { - let task = LOCAL_EXECUTOR.spawn(future); - // Loop until all tasks are done - while LOCAL_EXECUTOR.try_tick() {} + EXECUTOR.spawn_local(future); + // Loop until all tasks are done + while EXECUTOR.try_tick() {} - Task::new(task) - } + Task::new(task) } } } @@ -232,28 +215,6 @@ impl TaskPool { { self.spawn(future) } - - /// Runs a function with the local executor. Typically used to tick - /// the local executor on the main thread as it needs to share time with - /// other things. - /// - /// ``` - /// use bevy_tasks::TaskPool; - /// - /// TaskPool::new().with_local_executor(|local_executor| { - /// local_executor.try_tick(); - /// }); - /// ``` - pub fn with_local_executor(&self, f: F) -> R - where - F: FnOnce(&LocalExecutor) -> R, - { - #[cfg(feature = "std")] - return LOCAL_EXECUTOR.with(f); - - #[cfg(not(feature = "std"))] - return f(&LOCAL_EXECUTOR); - } } /// A `TaskPool` scope for running one or more non-`'static` futures. @@ -261,7 +222,7 @@ impl TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor: &'scope LocalExecutor<'scope>, + executor: &'scope Executor<'scope>, // Vector to gather results of all futures spawned during scope run results: &'env RefCell>>, @@ -313,7 +274,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; - self.executor.spawn(f).detach(); + self.executor.spawn_local(f).detach(); } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 25255a1e5d1d3..45c098adbcd1f 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -143,7 +143,6 @@ pub struct TaskPool { impl TaskPool { thread_local! { - static LOCAL_EXECUTOR: crate::executor::LocalExecutor<'static> = const { crate::executor::LocalExecutor::new() }; static THREAD_EXECUTOR: Arc> = Arc::new(ThreadExecutor::new()); } @@ -187,28 +186,20 @@ impl TaskPool { thread_builder .spawn(move || { - TaskPool::LOCAL_EXECUTOR.with(|local_executor| { - if let Some(on_thread_spawn) = on_thread_spawn { - on_thread_spawn(); - drop(on_thread_spawn); - } - let _destructor = CallOnDrop(on_thread_destroy); - loop { - let res = std::panic::catch_unwind(|| { - let tick_forever = async move { - loop { - local_executor.tick().await; - } - }; - block_on(ex.run(tick_forever.or(shutdown_rx.recv()))) - }); - if let Ok(value) = res { - // Use unwrap_err because we expect a Closed error - value.unwrap_err(); - break; - } + if let Some(on_thread_spawn) = on_thread_spawn { + on_thread_spawn(); + drop(on_thread_spawn); + } + let _destructor = CallOnDrop(on_thread_destroy); + loop { + let res = + std::panic::catch_unwind(|| block_on(ex.run(shutdown_rx.recv()))); + if let Ok(value) = res { + // Use unwrap_err because we expect a Closed error + value.unwrap_err(); + break; } - }); + } }) .expect("Failed to spawn thread.") }) @@ -578,25 +569,7 @@ impl TaskPool { where T: 'static, { - Task::new(TaskPool::LOCAL_EXECUTOR.with(|executor| executor.spawn(future))) - } - - /// Runs a function with the local executor. Typically used to tick - /// the local executor on the main thread as it needs to share time with - /// other things. - /// - /// ``` - /// use bevy_tasks::TaskPool; - /// - /// TaskPool::new().with_local_executor(|local_executor| { - /// local_executor.try_tick(); - /// }); - /// ``` - pub fn with_local_executor(&self, f: F) -> R - where - F: FnOnce(&crate::executor::LocalExecutor) -> R, - { - Self::LOCAL_EXECUTOR.with(f) + Task::new(self.executor.spawn_local(future)) } } diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 8b08d5941c6bd..eb8429fe0948f 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,6 +1,7 @@ use super::TaskPool; use bevy_platform::sync::OnceLock; use core::ops::Deref; +use crate::executor::Executor; macro_rules! taskpool { ($(#[$attr:meta])* ($static:ident, $type:ident)) => { @@ -83,24 +84,9 @@ taskpool! { /// This function *must* be called on the main thread, or the task pools will not be updated appropriately. #[cfg(not(all(target_arch = "wasm32", feature = "web")))] pub fn tick_global_task_pools_on_main_thread() { - COMPUTE_TASK_POOL - .get() - .unwrap() - .with_local_executor(|compute_local_executor| { - ASYNC_COMPUTE_TASK_POOL - .get() - .unwrap() - .with_local_executor(|async_local_executor| { - IO_TASK_POOL - .get() - .unwrap() - .with_local_executor(|io_local_executor| { - for _ in 0..100 { - compute_local_executor.try_tick(); - async_local_executor.try_tick(); - io_local_executor.try_tick(); - } - }); - }); - }); + for _ in 0..100 { + if !Executor::try_tick_local() { + break; + } + } } From 2b8ed66ceee2c38b3be662ba9c227fa565feed1d Mon Sep 17 00:00:00 2001 From: James Liu Date: Sat, 26 Jul 2025 20:12:54 -0700 Subject: [PATCH 03/68] Merge ThreadEecutors into the new forked async_executor --- .../src/schedule/executor/multi_threaded.rs | 9 +- crates/bevy_render/src/pipelined_rendering.rs | 2 +- crates/bevy_tasks/src/async_executor.rs | 270 ++++++++++-------- crates/bevy_tasks/src/lib.rs | 5 +- crates/bevy_tasks/src/task_pool.rs | 234 +++------------ crates/bevy_tasks/src/thread_executor.rs | 133 --------- crates/bevy_tasks/src/usages.rs | 2 +- 7 files changed, 209 insertions(+), 446 deletions(-) delete mode 100644 crates/bevy_tasks/src/thread_executor.rs diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index bd99344f498fa..5637a12ab9fa2 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -1,7 +1,6 @@ use alloc::{boxed::Box, vec::Vec}; use bevy_platform::cell::SyncUnsafeCell; -use bevy_platform::sync::Arc; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadExecutor}; +use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadSpawner}; use concurrent_queue::ConcurrentQueue; use core::{any::Any, panic::AssertUnwindSafe}; use fixedbitset::FixedBitSet; @@ -272,12 +271,10 @@ impl SystemExecutor for MultiThreadedExecutor { let thread_executor = world .get_resource::() .map(|e| e.0.clone()); - let thread_executor = thread_executor.as_deref(); let environment = &Environment::new(self, schedule, world); ComputeTaskPool::get_or_init(TaskPool::default).scope_with_executor( - false, thread_executor, |scope| { let context = Context { @@ -866,7 +863,7 @@ unsafe fn evaluate_and_fold_conditions( /// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadExecutor(pub Arc>); +pub struct MainThreadExecutor(pub ThreadSpawner<'static>); impl Default for MainThreadExecutor { fn default() -> Self { @@ -877,7 +874,7 @@ impl Default for MainThreadExecutor { impl MainThreadExecutor { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadExecutor(TaskPool::get_thread_executor()) + MainThreadExecutor(ComputeTaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 00dfc4ba0e19c..9769b051bd341 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -186,7 +186,7 @@ fn renderer_extract(app_world: &mut World, _world: &mut World) { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. if let Some(mut render_app) = ComputeTaskPool::get() - .scope_with_executor(true, Some(&*main_thread_executor.0), |s| { + .scope_with_executor(Some(main_thread_executor.0.clone()), |s| { s.spawn(async { render_channels.recv().await }); }) .pop() diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 8f862f8f0e662..c577fa60f26cc 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -7,6 +7,7 @@ use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; +use std::thread::{Thread, ThreadId}; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; @@ -16,12 +17,93 @@ use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; -static LOCAL_QUEUE: ThreadLocal> = ThreadLocal::new(); +static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); -#[derive(Default)] -struct LocalQueue { - queue: VecDeque, - active: Slab, +struct ThreadLocalState { + thread_id: ThreadId, + thread_locked_queue: ConcurrentQueue, + local_queue: RefCell>, + local_active: RefCell>, +} + +impl Default for ThreadLocalState { + fn default() -> Self { + Self { + thread_id: std::thread::current().id(), + thread_locked_queue: ConcurrentQueue::unbounded(), + local_queue: RefCell::new(VecDeque::new()), + local_active: RefCell::new(Slab::new()), + } + } +} + +#[derive(Clone, Debug)] +pub struct ThreadSpawner<'a> { + thread_id: ThreadId, + target_queue: &'static ConcurrentQueue, + state: Arc, + _marker: PhantomData<&'a ()>, +} + +impl<'a> ThreadSpawner<'a> { + /// Spawns a task onto the executor. + pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + let mut active = self.state.active(); + + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state.clone(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()) + }; + entry.insert(runnable.waker()); + + runnable.schedule(); + task + } + + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { + let thread_id = self.thread_id; + let queue: &'static ConcurrentQueue = self.target_queue; + let state = self.state.clone(); + + move |runnable| { + queue.push(runnable).unwrap(); + state.notify_specific_thread(thread_id); + } + } } /// An async executor. @@ -114,12 +196,12 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); - let mut local_state = local_queue.borrow_mut(); - let entry = local_state.active.vacant_entry(); + let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); + let mut local_active = local_state.local_active.borrow_mut(); + let entry = local_active.vacant_entry(); let index = entry.key(); let future = AsyncCallOnDrop::new(future, move || { - drop(local_queue.borrow_mut().active.try_remove(index)) + drop(local_state.local_active.borrow_mut().try_remove(index)) }); #[expect( @@ -155,35 +237,28 @@ impl<'a> Executor<'a> { }; entry.insert(runnable.waker()); - drop(local_state); + drop(local_active); runnable.schedule(); task } - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// - /// let ex = Executor::new(); - /// assert!(!ex.try_tick()); // no tasks to run - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.state().try_tick() + pub fn current_thread_spawner(&self) -> ThreadSpawner<'a> { + ThreadSpawner { + thread_id: std::thread::current().id(), + target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, + state: self.state_as_arc(), + _marker: PhantomData, + } } pub fn try_tick_local() -> bool { - if let Some(runnable) = LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back() { + if let Some(runnable) = THREAD_LOCAL_STATE + .get_or_default() + .local_queue + .borrow_mut() + .pop_back() + { // Run the task. runnable.run(); true @@ -192,44 +267,7 @@ impl<'a> Executor<'a> { } } - /// Runs a single task. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// use futures_lite::future; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// future::block_on(ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.state().tick().await; - } - /// Runs the executor until the given future completes. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// use futures_lite::future; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { 1 + 2 }); - /// let res = future::block_on(ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await } @@ -247,11 +285,12 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + let state = self.state_as_arc(); + let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { - local_queue.borrow_mut().queue.push_back(runnable); - // state.notify(); + local_state.local_queue.borrow_mut().push_back(runnable); + state.notify_specific_thread(local_state.thread_id); } } @@ -403,30 +442,19 @@ impl State { } } - pub(crate) fn try_tick(&self) -> bool { - let runnable = self - .queue - .pop() - .ok() - .or_else(|| LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back()); - if let Some(runnable) = runnable { - // Notify another ticker now to pick up where this ticker left off, just in case - // running the task takes a long time. - self.notify(); - - // Run the task. - runnable.run(); - true - } else { - false + /// Notifies a sleeping ticker. + #[inline] + fn notify_specific_thread(&self, thread_id: ThreadId) { + let waker = self + .sleepers + .lock() + .unwrap() + .notify_specific_thread(thread_id); + if let Some(w) = waker { + w.wake(); } } - pub(crate) async fn tick(&self) { - let runnable = Ticker::new(self).runnable().await; - runnable.run(); - } - pub async fn run(&self, future: impl Future) -> T { let mut runner = Runner::new(self); let mut rng = fastrand::Rng::new(); @@ -447,6 +475,12 @@ impl State { } } +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_state(self, "State", f) + } +} + /// A list of sleeping tickers. struct Sleepers { /// Number of sleeping tickers (both notified and unnotified). @@ -455,7 +489,7 @@ struct Sleepers { /// IDs and wakers of sleeping unnotified tickers. /// /// A sleeping ticker is notified when its waker is missing from this list. - wakers: Vec<(usize, Waker)>, + wakers: Vec<(usize, ThreadId, Waker)>, /// Reclaimed IDs. free_ids: Vec, @@ -469,7 +503,8 @@ impl Sleepers { None => self.count + 1, }; self.count += 1; - self.wakers.push((id, waker.clone())); + self.wakers + .push((id, std::thread::current().id(), waker.clone())); id } @@ -479,12 +514,13 @@ impl Sleepers { fn update(&mut self, id: usize, waker: &Waker) -> bool { for item in &mut self.wakers { if item.0 == id { - item.1.clone_from(waker); + item.2.clone_from(waker); return false; } } - self.wakers.push((id, waker.clone())); + self.wakers + .push((id, std::thread::current().id(), waker.clone())); true } @@ -514,11 +550,24 @@ impl Sleepers { /// If a ticker was notified already or there are no tickers, `None` will be returned. fn notify(&mut self) -> Option { if self.wakers.len() == self.count { - self.wakers.pop().map(|item| item.1) + self.wakers.pop().map(|item| item.2) } else { None } } + + /// Returns notification waker for a sleeping ticker. + /// + /// If a ticker was notified already or there are no tickers, `None` will be returned. + fn notify_specific_thread(&mut self, thread_id: ThreadId) -> Option { + for i in (0..self.wakers.len()).rev() { + if self.wakers[i].1 == thread_id { + let (_, _, waker) = self.wakers.remove(i); + return Some(waker); + } + } + None + } } /// Runs task one by one. @@ -581,19 +630,6 @@ impl Ticker<'_> { self.sleeping = 0; } - /// Waits for the next runnable task to run. - async fn runnable(&mut self) -> Runnable { - self.runnable_with(|| { - LOCAL_QUEUE - .get_or_default() - .borrow_mut() - .queue - .pop_back() - .or_else(|| self.state.queue.pop().ok()) - }) - .await - } - /// Waits for the next runnable task to run, given a function that searches for a task. async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { future::poll_fn(|cx| { @@ -660,7 +696,7 @@ struct Runner<'a> { ticks: usize, // The thread local state of the executor for the current thread. - local_state: &'static RefCell, + local_state: &'static ThreadLocalState, } impl Runner<'_> { @@ -671,7 +707,7 @@ impl Runner<'_> { ticker: Ticker::new(state), local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, - local_state: LOCAL_QUEUE.get_or_default(), + local_state: THREAD_LOCAL_STATE.get_or_default(), }; state .local_queues @@ -686,7 +722,7 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - if let Some(r) = self.local_state.borrow_mut().queue.pop_back() { + if let Some(r) = self.local_state.local_queue.borrow_mut().pop_back() { return Some(r); } @@ -724,6 +760,12 @@ impl Runner<'_> { } } + if let Ok(r) = self.local_state.thread_locked_queue.pop() { + // Do not steal from this queue. If other threads steal + // from this current thread, the task will be moved. + return Some(r); + } + None }) .await; @@ -896,6 +938,8 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { + use super::Executor; + fn _ensure_send_and_sync() { fn is_send(_: T) {} fn is_sync(_: T) {} @@ -910,6 +954,8 @@ mod test { is_send(ex.schedule()); is_sync(ex.schedule()); is_static(ex.schedule()); + is_send(ex.current_thread_spawner()); + is_sync(ex.current_thread_spawner()); /// ```compile_fail /// use crate::async_executor::LocalExecutor; diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 4f23e7954bf36..2e04705b13fb0 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -60,14 +60,13 @@ pub use task::Task; cfg_if::cfg_if! { if #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { mod task_pool; - mod thread_executor; pub use task_pool::{Scope, TaskPool, TaskPoolBuilder}; - pub use thread_executor::{ThreadExecutor, ThreadExecutorTicker}; + pub use async_executor::ThreadSpawner; } else if #[cfg(any(target_arch = "wasm32", not(feature = "multi_threaded")))] { mod single_threaded_task_pool; - pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadExecutor}; + pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 45c098adbcd1f..bf9da3d2ed454 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -1,20 +1,14 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; -use std::{ - thread::{self, JoinHandle}, - thread_local, -}; +use std::thread::{self, JoinHandle}; +use crate::async_executor::ThreadSpawner; use crate::executor::FallibleTask; use bevy_platform::sync::Arc; use concurrent_queue::ConcurrentQueue; use futures_lite::FutureExt; -use crate::{ - block_on, - thread_executor::{ThreadExecutor, ThreadExecutorTicker}, - Task, -}; +use crate::{block_on, Task}; struct CallOnDrop(Option>); @@ -142,13 +136,9 @@ pub struct TaskPool { } impl TaskPool { - thread_local! { - static THREAD_EXECUTOR: Arc> = Arc::new(ThreadExecutor::new()); - } - - /// Each thread should only create one `ThreadExecutor`, otherwise, there are good chances they will deadlock - pub fn get_thread_executor() -> Arc> { - Self::THREAD_EXECUTOR.with(Clone::clone) + /// Each thread will only have one `ThreadExecutor`, otherwise, there are good chances they will deadlock + pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + self.executor.current_thread_spawner() } /// Create a `TaskPool` with the default configuration. @@ -303,57 +293,47 @@ impl TaskPool { F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - Self::THREAD_EXECUTOR.with(|scope_executor| { - self.scope_with_executor_inner(true, scope_executor, scope_executor, f) - }) + let scope_spawner = self.current_thread_spawner(); + self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } /// This allows passing an external executor to spawn tasks on. When you pass an external executor /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. /// - /// When `tick_task_pool_executor` is set to `true`, the multithreaded task stealing executor is ticked on the scope - /// thread. Disabling this can be useful when finishing the scope is latency sensitive. Pulling tasks from - /// global executor can run tasks unrelated to the scope and delay when the scope returns. - /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, - tick_task_pool_executor: bool, - external_executor: Option<&ThreadExecutor>, + external_spawner: Option>, f: F, ) -> Vec where F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - Self::THREAD_EXECUTOR.with(|scope_executor| { - // If an `external_executor` is passed, use that. Otherwise, get the executor stored - // in the `THREAD_EXECUTOR` thread local. - if let Some(external_executor) = external_executor { - self.scope_with_executor_inner( - tick_task_pool_executor, - external_executor, - scope_executor, - f, - ) - } else { - self.scope_with_executor_inner( - tick_task_pool_executor, - scope_executor, - scope_executor, - f, - ) - } - }) + let scope_spawner = self.executor.current_thread_spawner(); + // If an `external_executor` is passed, use that. Otherwise, get the executor stored + // in the `THREAD_EXECUTOR` thread local. + if let Some(external_spawner) = external_spawner { + self.scope_with_executor_inner( + external_spawner, + scope_spawner, + f, + ) + } else { + self.scope_with_executor_inner( + scope_spawner.clone(), + scope_spawner, + f, + ) + } } #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] fn scope_with_executor_inner<'env, F, T>( &self, - tick_task_pool_executor: bool, - external_executor: &ThreadExecutor, - scope_executor: &ThreadExecutor, + external_spawner: ThreadSpawner<'env>, + scope_spawner: ThreadSpawner<'env>, f: F, ) -> Vec where @@ -370,11 +350,6 @@ impl TaskPool { let executor: &crate::executor::Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let external_executor: &'env ThreadExecutor<'env> = - unsafe { mem::transmute(external_executor) }; - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let scope_executor: &'env ThreadExecutor<'env> = unsafe { mem::transmute(scope_executor) }; let spawned: ConcurrentQueue>>> = ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function @@ -385,8 +360,8 @@ impl TaskPool { let scope = Scope { executor, - external_executor, - scope_executor, + external_spawner, + scope_spawner, spawned, scope: PhantomData, env: PhantomData, @@ -401,144 +376,23 @@ impl TaskPool { if spawned.is_empty() { Vec::new() } else { - block_on(async move { - let get_results = async { - let mut results = Vec::with_capacity(spawned.len()); - while let Ok(task) = spawned.pop() { - if let Some(res) = task.await { - match res { - Ok(res) => results.push(res), - Err(payload) => std::panic::resume_unwind(payload), - } - } else { - panic!("Failed to catch panic!"); + block_on(self.executor.run(async move { + let mut results = Vec::with_capacity(spawned.len()); + while let Ok(task) = spawned.pop() { + if let Some(res) = task.await { + match res { + Ok(res) => results.push(res), + Err(payload) => std::panic::resume_unwind(payload), } + } else { + panic!("Failed to catch panic!"); } - results - }; - - let tick_task_pool_executor = tick_task_pool_executor || self.threads.is_empty(); - - // we get this from a thread local so we should always be on the scope executors thread. - // note: it is possible `scope_executor` and `external_executor` is the same executor, - // in that case, we should only tick one of them, otherwise, it may cause deadlock. - let scope_ticker = scope_executor.ticker().unwrap(); - let external_ticker = if !external_executor.is_same(scope_executor) { - external_executor.ticker() - } else { - None - }; - - match (external_ticker, tick_task_pool_executor) { - (Some(external_ticker), true) => { - Self::execute_global_external_scope( - executor, - external_ticker, - scope_ticker, - get_results, - ) - .await - } - (Some(external_ticker), false) => { - Self::execute_external_scope(external_ticker, scope_ticker, get_results) - .await - } - // either external_executor is none or it is same as scope_executor - (None, true) => { - Self::execute_global_scope(executor, scope_ticker, get_results).await - } - (None, false) => Self::execute_scope(scope_ticker, get_results).await, } - }) + results + })) } } - #[inline] - async fn execute_global_external_scope<'scope, 'ticker, T>( - executor: &'scope crate::executor::Executor<'scope>, - external_ticker: ThreadExecutorTicker<'scope, 'ticker>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - // we restart the executors if a task errors. if a scoped - // task errors it will panic the scope on the call to get_results - let execute_forever = async move { - loop { - let tick_forever = async { - loop { - external_ticker.tick().or(scope_ticker.tick()).await; - } - }; - // we don't care if it errors. If a scoped task errors it will propagate - // to get_results - let _result = AssertUnwindSafe(executor.run(tick_forever)) - .catch_unwind() - .await - .is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_external_scope<'scope, 'ticker, T>( - external_ticker: ThreadExecutorTicker<'scope, 'ticker>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - external_ticker.tick().or(scope_ticker.tick()).await; - } - }; - let _result = AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_global_scope<'scope, 'ticker, T>( - executor: &'scope crate::executor::Executor<'scope>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - scope_ticker.tick().await; - } - }; - let _result = AssertUnwindSafe(executor.run(tick_forever)) - .catch_unwind() - .await - .is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_scope<'scope, 'ticker, T>( - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - scope_ticker.tick().await; - } - }; - let _result = AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); - } - }; - get_results.or(execute_forever).await - } - /// Spawns a static future onto the thread pool. The returned [`Task`] is a /// future that can be polled for the result. It can also be canceled and /// "detached", allowing the task to continue running even if dropped. In @@ -599,8 +453,8 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope crate::executor::Executor<'scope>, - external_executor: &'scope ThreadExecutor<'scope>, - scope_executor: &'scope ThreadExecutor<'scope>, + external_spawner: ThreadSpawner<'scope>, + scope_spawner: ThreadSpawner<'scope>, spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -634,7 +488,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { let task = self - .scope_executor + .scope_spawner .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); // ConcurrentQueue only errors when closed or full, but we never @@ -651,7 +505,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_external + 'scope + Send>(&self, f: Fut) { let task = self - .external_executor + .external_spawner .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); // ConcurrentQueue only errors when closed or full, but we never diff --git a/crates/bevy_tasks/src/thread_executor.rs b/crates/bevy_tasks/src/thread_executor.rs deleted file mode 100644 index 86d2ab280d87c..0000000000000 --- a/crates/bevy_tasks/src/thread_executor.rs +++ /dev/null @@ -1,133 +0,0 @@ -use core::marker::PhantomData; -use std::thread::{self, ThreadId}; - -use crate::executor::Executor; -use async_task::Task; -use futures_lite::Future; - -/// An executor that can only be ticked on the thread it was instantiated on. But -/// can spawn `Send` tasks from other threads. -/// -/// # Example -/// ``` -/// # use std::sync::{Arc, atomic::{AtomicI32, Ordering}}; -/// use bevy_tasks::ThreadExecutor; -/// -/// let thread_executor = ThreadExecutor::new(); -/// let count = Arc::new(AtomicI32::new(0)); -/// -/// // create some owned values that can be moved into another thread -/// let count_clone = count.clone(); -/// -/// std::thread::scope(|scope| { -/// scope.spawn(|| { -/// // we cannot get the ticker from another thread -/// let not_thread_ticker = thread_executor.ticker(); -/// assert!(not_thread_ticker.is_none()); -/// -/// // but we can spawn tasks from another thread -/// thread_executor.spawn(async move { -/// count_clone.fetch_add(1, Ordering::Relaxed); -/// }).detach(); -/// }); -/// }); -/// -/// // the tasks do not make progress unless the executor is manually ticked -/// assert_eq!(count.load(Ordering::Relaxed), 0); -/// -/// // tick the ticker until task finishes -/// let thread_ticker = thread_executor.ticker().unwrap(); -/// thread_ticker.try_tick(); -/// assert_eq!(count.load(Ordering::Relaxed), 1); -/// ``` -#[derive(Debug)] -pub struct ThreadExecutor<'task> { - executor: Executor<'task>, - thread_id: ThreadId, -} - -impl<'task> Default for ThreadExecutor<'task> { - fn default() -> Self { - Self { - executor: Executor::new(), - thread_id: thread::current().id(), - } - } -} - -impl<'task> ThreadExecutor<'task> { - /// create a new [`ThreadExecutor`] - pub fn new() -> Self { - Self::default() - } - - /// Spawn a task on the thread executor - pub fn spawn( - &self, - future: impl Future + Send + 'task, - ) -> Task { - self.executor.spawn(future) - } - - /// Gets the [`ThreadExecutorTicker`] for this executor. - /// Use this to tick the executor. - /// It only returns the ticker if it's on the thread the executor was created on - /// and returns `None` otherwise. - pub fn ticker<'ticker>(&'ticker self) -> Option> { - if thread::current().id() == self.thread_id { - return Some(ThreadExecutorTicker { - executor: self, - _marker: PhantomData, - }); - } - None - } - - /// Returns true if `self` and `other`'s executor is same - pub fn is_same(&self, other: &Self) -> bool { - core::ptr::eq(self, other) - } -} - -/// Used to tick the [`ThreadExecutor`]. The executor does not -/// make progress unless it is manually ticked on the thread it was -/// created on. -#[derive(Debug)] -pub struct ThreadExecutorTicker<'task, 'ticker> { - executor: &'ticker ThreadExecutor<'task>, - // make type not send or sync - _marker: PhantomData<*const ()>, -} - -impl<'task, 'ticker> ThreadExecutorTicker<'task, 'ticker> { - /// Tick the thread executor. - pub async fn tick(&self) { - self.executor.executor.tick().await; - } - - /// Synchronously try to tick a task on the executor. - /// Returns false if does not find a task to tick. - pub fn try_tick(&self) -> bool { - self.executor.executor.try_tick() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::sync::Arc; - - #[test] - fn test_ticker() { - let executor = Arc::new(ThreadExecutor::new()); - let ticker = executor.ticker(); - assert!(ticker.is_some()); - - thread::scope(|s| { - s.spawn(|| { - let ticker = executor.ticker(); - assert!(ticker.is_none()); - }); - }); - } -} diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index eb8429fe0948f..8bd0bd3182cd2 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,7 +1,7 @@ use super::TaskPool; +use crate::executor::Executor; use bevy_platform::sync::OnceLock; use core::ops::Deref; -use crate::executor::Executor; macro_rules! taskpool { ($(#[$attr:meta])* ($static:ident, $type:ident)) => { From 1121542bdbbde3873731705b9e9aaf192590be4f Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 27 Jul 2025 09:10:47 -0700 Subject: [PATCH 04/68] Move stealer queues to TLS storage. Avoid extra Arc allocations --- crates/bevy_tasks/src/async_executor.rs | 244 ++++++++++++++---------- crates/bevy_tasks/src/task_pool.rs | 12 +- 2 files changed, 148 insertions(+), 108 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index c577fa60f26cc..f6f396b2200da 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,20 +19,32 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); +std::thread_local! { + static LOCAL_QUEUE: LocalQueue = const { + LocalQueue { + local_queue: RefCell::new(VecDeque::new()), + local_active: RefCell::new(Slab::new()), + } + }; +} + +struct LocalQueue { + local_queue: RefCell>, + local_active: RefCell>, +} + struct ThreadLocalState { thread_id: ThreadId, + stealable_queue: ConcurrentQueue, thread_locked_queue: ConcurrentQueue, - local_queue: RefCell>, - local_active: RefCell>, } impl Default for ThreadLocalState { fn default() -> Self { Self { thread_id: std::thread::current().id(), + stealable_queue: ConcurrentQueue::bounded(512), thread_locked_queue: ConcurrentQueue::unbounded(), - local_queue: RefCell::new(VecDeque::new()), - local_active: RefCell::new(Slab::new()), } } } @@ -101,7 +113,7 @@ impl<'a> ThreadSpawner<'a> { move |runnable| { queue.push(runnable).unwrap(); - state.notify_specific_thread(thread_id); + state.notify_specific_thread(thread_id, false); } } } @@ -196,49 +208,52 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); - let mut local_active = local_state.local_active.borrow_mut(); - let entry = local_active.vacant_entry(); - let index = entry.key(); - let future = AsyncCallOnDrop::new(future, move || { - drop(local_state.local_active.borrow_mut().try_remove(index)) + let (runnable, task) = LOCAL_QUEUE.with(|tls| { + let mut local_active = tls.local_active.borrow_mut(); + let entry = local_active.vacant_entry(); + let index = entry.key(); + let future = AsyncCallOnDrop::new(future, move || { + LOCAL_QUEUE.with(|tls| { + drop(tls.local_active.borrow_mut().try_remove(index)) + }); + }); + + #[expect( + unsafe_code, + reason = "Builder::spawn_local requires a 'static lifetime" + )] + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule_local()) + }; + entry.insert(runnable.waker()); + + (runnable, task) }); - #[expect( - unsafe_code, - reason = "Builder::spawn_local requires a 'static lifetime" - )] - // Create the task and register it in the set of active tasks. - // - // SAFETY: - // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. - let (runnable, task) = unsafe { - Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule_local()) - }; - entry.insert(runnable.waker()); - - drop(local_active); - runnable.schedule(); task } @@ -253,18 +268,19 @@ impl<'a> Executor<'a> { } pub fn try_tick_local() -> bool { - if let Some(runnable) = THREAD_LOCAL_STATE - .get_or_default() - .local_queue - .borrow_mut() - .pop_back() - { - // Run the task. - runnable.run(); - true - } else { - false - } + LOCAL_QUEUE.with(|tls| { + if let Some(runnable) = tls + .local_queue + .borrow_mut() + .pop_back() + { + // Run the task. + runnable.run(); + true + } else { + false + } + }) } /// Runs the executor until the given future completes. @@ -276,8 +292,21 @@ impl<'a> Executor<'a> { fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { let state = self.state_as_arc(); - // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { + // Attempt to push onto the local queue first, because we know that + // this thread is awake. + // let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + // match local_state.stealable_queue.push(runnable) { + // Ok(()) => { + // state.notify_specific_thread(local_state.thread_id, true); + // return; + // } + // Err(r) => r.into_inner(), + // } + // } else { + // runnable + // }; + // Otherwise push onto the global queue instead. state.queue.push(runnable).unwrap(); state.notify(); } @@ -289,8 +318,10 @@ impl<'a> Executor<'a> { let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { - local_state.local_queue.borrow_mut().push_back(runnable); - state.notify_specific_thread(local_state.thread_id); + LOCAL_QUEUE.with(|tls| { + tls.local_queue.borrow_mut().push_back(runnable); + }); + state.notify_specific_thread(local_state.thread_id, false); } } @@ -394,7 +425,7 @@ struct State { queue: ConcurrentQueue, /// Local queues created by runners. - local_queues: RwLock>>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -411,7 +442,7 @@ impl State { const fn new() -> State { State { queue: ConcurrentQueue::unbounded(), - local_queues: RwLock::new(Vec::new()), + stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { count: 0, @@ -444,12 +475,18 @@ impl State { /// Notifies a sleeping ticker. #[inline] - fn notify_specific_thread(&self, thread_id: ThreadId) { - let waker = self - .sleepers - .lock() - .unwrap() - .notify_specific_thread(thread_id); + fn notify_specific_thread(&self, thread_id: ThreadId, allow_stealing: bool) { + let mut sleepers = self.sleepers.lock().unwrap(); + let mut waker = sleepers.notify_specific_thread(thread_id); + if waker.is_none() + && allow_stealing + && self + .notified + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + waker = sleepers.notify(); + } if let Some(w) = waker { w.wake(); } @@ -689,9 +726,6 @@ struct Runner<'a> { /// Inner ticker. ticker: Ticker<'a>, - /// The local queue. - local: Arc>, - /// Bumped every time a runnable task is found. ticks: usize, @@ -702,18 +736,18 @@ struct Runner<'a> { impl Runner<'_> { /// Creates a runner and registers it in the executor state. fn new(state: &State) -> Runner<'_> { + let local_state = THREAD_LOCAL_STATE.get_or_default(); let runner = Runner { state, ticker: Ticker::new(state), - local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, - local_state: THREAD_LOCAL_STATE.get_or_default(), + local_state, }; state - .local_queues + .stealer_queues .write() .unwrap() - .push(runner.local.clone()); + .push(&local_state.stealable_queue); runner } @@ -722,40 +756,46 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - if let Some(r) = self.local_state.local_queue.borrow_mut().pop_back() { - return Some(r); + { + let local_pop = LOCAL_QUEUE.with(|tls| { + tls.local_queue.borrow_mut().pop_back() + }); + if let Some(r) = local_pop { + return Some(r); + } } // Try the local queue. - if let Ok(r) = self.local.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. if let Ok(r) = self.state.queue.pop() { - steal(&self.state.queue, &self.local); + steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } // Try stealing from other runners. - let local_queues = self.state.local_queues.read().unwrap(); + let stealer_queues = self.state.stealer_queues.read().unwrap(); // Pick a random starting point in the iterator list and rotate the list. - let n = local_queues.len(); + let n = stealer_queues.len(); let start = rng.usize(..n); - let iter = local_queues + let iter = stealer_queues .iter() - .chain(local_queues.iter()) + .chain(stealer_queues.iter()) .skip(start) .take(n); // Remove this runner's local queue. - let iter = iter.filter(|local| !Arc::ptr_eq(local, &self.local)); + let iter = + iter.filter(|local| !std::ptr::eq(**local, &self.local_state.stealable_queue)); // Try stealing from each local queue in the list. for local in iter { - steal(local, &self.local); - if let Ok(r) = self.local.pop() { + steal(local, &self.local_state.stealable_queue); + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } } @@ -775,7 +815,7 @@ impl Runner<'_> { if self.ticks % 64 == 0 { // Steal tasks from the global queue to ensure fair task scheduling. - steal(&self.state.queue, &self.local); + steal(&self.state.queue, &self.local_state.stealable_queue); } runnable @@ -785,14 +825,20 @@ impl Runner<'_> { impl Drop for Runner<'_> { fn drop(&mut self) { // Remove the local queue. - self.state - .local_queues - .write() - .unwrap() - .retain(|local| !Arc::ptr_eq(local, &self.local)); + { + let mut stealer_queues = self.state.stealer_queues.write().unwrap(); + if let Some((idx, _)) = stealer_queues + .iter() + .enumerate() + .rev() + .find(|(_, local)| !std::ptr::eq(**local, &self.local_state.stealable_queue)) + { + stealer_queues.remove(idx); + } + } // Re-schedule remaining tasks in the local queue. - while let Ok(r) = self.local.pop() { + while let Ok(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } @@ -865,7 +911,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -896,7 +942,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re f.debug_struct(name) .field("active", &ActiveTasks(&state.active)) .field("global_tasks", &state.queue.len()) - .field("local_runners", &LocalRunners(&state.local_queues)) + .field("stealer_queues", &LocalRunners(&state.stealer_queues)) .field("sleepers", &SleepCount(&state.sleepers)) .finish() } @@ -939,6 +985,8 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { use super::Executor; + use super::ThreadLocalState; + use super::THREAD_LOCAL_STATE; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -949,13 +997,13 @@ mod test { is_sync::>(Executor::new()); let ex = Executor::new(); - is_send(ex.tick()); - is_sync(ex.tick()); is_send(ex.schedule()); is_sync(ex.schedule()); is_static(ex.schedule()); is_send(ex.current_thread_spawner()); is_sync(ex.current_thread_spawner()); + is_send(THREAD_LOCAL_STATE.get_or_default()); + is_sync(THREAD_LOCAL_STATE.get_or_default()); /// ```compile_fail /// use crate::async_executor::LocalExecutor; diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index bf9da3d2ed454..19a1e6887447d 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -315,17 +315,9 @@ impl TaskPool { // If an `external_executor` is passed, use that. Otherwise, get the executor stored // in the `THREAD_EXECUTOR` thread local. if let Some(external_spawner) = external_spawner { - self.scope_with_executor_inner( - external_spawner, - scope_spawner, - f, - ) + self.scope_with_executor_inner(external_spawner, scope_spawner, f) } else { - self.scope_with_executor_inner( - scope_spawner.clone(), - scope_spawner, - f, - ) + self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } } From 01ac223cf17293701efb4ec5653670f5cd05492d Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 27 Jul 2025 19:21:05 -0700 Subject: [PATCH 05/68] Queue tasks directly back onto local queues to avoid global injector queue contention --- crates/bevy_tasks/src/async_executor.rs | 54 +++++++++++++------------ crates/bevy_tasks/src/task_pool.rs | 2 + 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f6f396b2200da..705cc9060655d 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,6 +19,11 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); +pub(crate) fn install_runtime_into_current_thread() { + let tls = THREAD_LOCAL_STATE.get_or_default(); + tls.executor_thread.store(true, Ordering::Relaxed); +} + std::thread_local! { static LOCAL_QUEUE: LocalQueue = const { LocalQueue { @@ -34,6 +39,7 @@ struct LocalQueue { } struct ThreadLocalState { + executor_thread: AtomicBool, thread_id: ThreadId, stealable_queue: ConcurrentQueue, thread_locked_queue: ConcurrentQueue, @@ -42,6 +48,7 @@ struct ThreadLocalState { impl Default for ThreadLocalState { fn default() -> Self { Self { + executor_thread: AtomicBool::new(false), thread_id: std::thread::current().id(), stealable_queue: ConcurrentQueue::bounded(512), thread_locked_queue: ConcurrentQueue::unbounded(), @@ -213,9 +220,7 @@ impl<'a> Executor<'a> { let entry = local_active.vacant_entry(); let index = entry.key(); let future = AsyncCallOnDrop::new(future, move || { - LOCAL_QUEUE.with(|tls| { - drop(tls.local_active.borrow_mut().try_remove(index)) - }); + LOCAL_QUEUE.with(|tls| drop(tls.local_active.borrow_mut().try_remove(index))); }); #[expect( @@ -269,11 +274,7 @@ impl<'a> Executor<'a> { pub fn try_tick_local() -> bool { LOCAL_QUEUE.with(|tls| { - if let Some(runnable) = tls - .local_queue - .borrow_mut() - .pop_back() - { + if let Some(runnable) = tls.local_queue.borrow_mut().pop_back() { // Run the task. runnable.run(); true @@ -293,19 +294,23 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); move |runnable| { - // Attempt to push onto the local queue first, because we know that - // this thread is awake. - // let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - // match local_state.stealable_queue.push(runnable) { - // Ok(()) => { - // state.notify_specific_thread(local_state.thread_id, true); - // return; - // } - // Err(r) => r.into_inner(), - // } - // } else { - // runnable - // }; + // Attempt to push onto the local queue first in dedicated executor threads, + // because we know that this thread is awake and always processing new tasks. + let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + if local_state.executor_thread.load(Ordering::Relaxed) { + match local_state.stealable_queue.push(runnable) { + Ok(()) => { + state.notify_specific_thread(local_state.thread_id, true); + return; + } + Err(r) => r.into_inner(), + } + } else { + runnable + } + } else { + runnable + }; // Otherwise push onto the global queue instead. state.queue.push(runnable).unwrap(); state.notify(); @@ -316,7 +321,6 @@ impl<'a> Executor<'a> { fn schedule_local(&self) -> impl Fn(Runnable) + 'static { let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); - // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { LOCAL_QUEUE.with(|tls| { tls.local_queue.borrow_mut().push_back(runnable); @@ -757,9 +761,7 @@ impl Runner<'_> { .ticker .runnable_with(|| { { - let local_pop = LOCAL_QUEUE.with(|tls| { - tls.local_queue.borrow_mut().pop_back() - }); + let local_pop = LOCAL_QUEUE.with(|tls| tls.local_queue.borrow_mut().pop_back()); if let Some(r) = local_pop { return Some(r); } @@ -831,7 +833,7 @@ impl Drop for Runner<'_> { .iter() .enumerate() .rev() - .find(|(_, local)| !std::ptr::eq(**local, &self.local_state.stealable_queue)) + .find(|(_, local)| std::ptr::eq(**local, &self.local_state.stealable_queue)) { stealer_queues.remove(idx); } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 19a1e6887447d..ac31c86359b35 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -176,6 +176,8 @@ impl TaskPool { thread_builder .spawn(move || { + crate::async_executor::install_runtime_into_current_thread(); + if let Some(on_thread_spawn) = on_thread_spawn { on_thread_spawn(); drop(on_thread_spawn); From 1dab08e37a1d1a260f1324621c6a69b37d4014f2 Mon Sep 17 00:00:00 2001 From: James Liu Date: Mon, 28 Jul 2025 13:58:54 -0700 Subject: [PATCH 06/68] Use UnsafeCell instead of RefCell --- crates/bevy_tasks/src/async_executor.rs | 223 ++++++++++++------------ 1 file changed, 112 insertions(+), 111 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 705cc9060655d..32645c190eb59 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -1,4 +1,8 @@ -use std::cell::RefCell; +#![expect( + unsafe_code, + reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" +)] + use std::collections::VecDeque; use std::fmt; use std::marker::PhantomData; @@ -7,7 +11,7 @@ use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; -use std::thread::{Thread, ThreadId}; +use std::thread::ThreadId; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; @@ -24,18 +28,54 @@ pub(crate) fn install_runtime_into_current_thread() { tls.executor_thread.store(true, Ordering::Relaxed); } -std::thread_local! { - static LOCAL_QUEUE: LocalQueue = const { - LocalQueue { - local_queue: RefCell::new(VecDeque::new()), - local_active: RefCell::new(Slab::new()), +// Do not access this directly, use `with_local_queue` instead. +cfg_if::cfg_if! { + if #[cfg(all(debug_assertions, not(miri)))] { + use std::cell::RefCell; + + std::thread_local! { + static LOCAL_QUEUE: RefCell = const { + RefCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; + } + } else { + use std::cell::UnsafeCell; + + std::thread_local! { + static LOCAL_QUEUE: UnsafeCell = const { + UnsafeCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; + } + } +} + +/// # Safety +/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +#[inline(always)] +unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { + LOCAL_QUEUE.with(|tls| { + cfg_if::cfg_if! { + if #[cfg(all(debug_assertions, not(miri)))] { + f(&mut *tls.borrow_mut()) + } else { + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. The caller guarantees that this function is not used with + // LOCAL_QUEUE in any way. + f(unsafe { &mut *tls.get() }) + } } - }; + }) } struct LocalQueue { - local_queue: RefCell>, - local_active: RefCell>, + local_queue: VecDeque, + local_active: Slab, } struct ThreadLocalState { @@ -75,10 +115,6 @@ impl<'a> ThreadSpawner<'a> { let state = self.state.clone(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -131,22 +167,9 @@ pub struct Executor<'a> { state: AtomicPtr, /// Makes the `'a` lifetime invariant. - _marker: PhantomData>, + _marker: PhantomData<&'a ()>, } -#[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" -)] -// SAFETY: Executor stores no thread local state that can be accessed via other thread. -unsafe impl Send for Executor<'_> {} -#[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" -)] -// SAFETY: Executor internally synchronizes all of it's operations internally. -unsafe impl Sync for Executor<'_> {} - impl UnwindSafe for Executor<'_> {} impl RefUnwindSafe for Executor<'_> {} @@ -175,10 +198,6 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -215,49 +234,51 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let (runnable, task) = LOCAL_QUEUE.with(|tls| { - let mut local_active = tls.local_active.borrow_mut(); - let entry = local_active.vacant_entry(); - let index = entry.key(); - let future = AsyncCallOnDrop::new(future, move || { - LOCAL_QUEUE.with(|tls| drop(tls.local_active.borrow_mut().try_remove(index))); - }); - - #[expect( - unsafe_code, - reason = "Builder::spawn_local requires a 'static lifetime" - )] - // Create the task and register it in the set of active tasks. - // - // SAFETY: - // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. - let (runnable, task) = unsafe { - Builder::new() + // + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let (runnable, task) = unsafe { + with_local_queue(|tls| { + let entry = tls.local_active.vacant_entry(); + let index = entry.key(); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. This AsyncCallOnDrop will be + // invoked after the surrounding scope has exited in either a + // `try_tick_local` or `run` call. + let future = AsyncCallOnDrop::new(future, move || { + with_local_queue(|tls| drop(tls.local_active.try_remove(index))); + }); + + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = Builder::new() .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule_local()) - }; - entry.insert(runnable.waker()); + .spawn_unchecked(|()| future, self.schedule_local()); + entry.insert(runnable.waker()); - (runnable, task) - }); + (runnable, task) + }) + }; runnable.schedule(); task @@ -273,15 +294,13 @@ impl<'a> Executor<'a> { } pub fn try_tick_local() -> bool { - LOCAL_QUEUE.with(|tls| { - if let Some(runnable) = tls.local_queue.borrow_mut().pop_back() { - // Run the task. - runnable.run(); - true - } else { - false - } - }) + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) } + .map(|runnable| runnable.run()) + .is_some() } /// Runs the executor until the given future completes. @@ -322,9 +341,12 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { - LOCAL_QUEUE.with(|tls| { - tls.local_queue.borrow_mut().push_back(runnable); - }); + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { + with_local_queue(|tls| tls.local_queue.push_back(runnable)); + } state.notify_specific_thread(local_state.thread_id, false); } } @@ -335,18 +357,13 @@ impl<'a> Executor<'a> { #[cold] fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { let state = Arc::new(State::new()); - // TODO: Switch this to use cast_mut once the MSRV can be bumped past 1.65 - let ptr = Arc::into_raw(state) as *mut State; + let ptr = Arc::into_raw(state).cast_mut(); if let Err(actual) = atomic_ptr.compare_exchange( std::ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire, ) { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: This was just created from Arc::into_raw. drop(unsafe { Arc::from_raw(ptr) }); actual @@ -365,24 +382,14 @@ impl<'a> Executor<'a> { /// Returns a reference to the inner state. #[inline] fn state(&self) -> &State { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: So long as an Executor lives, it's state pointer will always be valid // when accessed through state_ptr. - unsafe { - &*self.state_ptr() - } + unsafe { &*self.state_ptr() } } // Clones the inner state Arc #[inline] fn state_as_arc(&self) -> Arc { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: So long as an Executor lives, it's state pointer will always be a valid // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; @@ -399,10 +406,6 @@ impl Drop for Executor<'_> { return; } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: As ptr is not null, it was allocated via Arc::new and converted // via Arc::into_raw in state_ptr. let state = unsafe { Arc::from_raw(ptr) }; @@ -761,7 +764,9 @@ impl Runner<'_> { .ticker .runnable_with(|| { { - let local_pop = LOCAL_QUEUE.with(|tls| tls.local_queue.borrow_mut().pop_back()); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) }; if let Some(r) = local_pop { return Some(r); } @@ -885,10 +890,6 @@ fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_ return f.debug_tuple(name).field(&Uninitialized).finish(); } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: If the state pointer is not null, it must have been // allocated properly by Arc::new and converted via Arc::into_raw // in state_ptr. From 0b036fe4936d028d763454e39e31a191712959be Mon Sep 17 00:00:00 2001 From: James Liu Date: Mon, 28 Jul 2025 14:29:31 -0700 Subject: [PATCH 07/68] Optimize access of thread locked tasks --- crates/bevy_tasks/src/async_executor.rs | 65 +++++++++++++++++++------ 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 32645c190eb59..6271598e66765 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -144,18 +144,25 @@ impl<'a> ThreadSpawner<'a> { }; entry.insert(runnable.waker()); - runnable.schedule(); + // Instead of directly scheduling this task, it's put into the onto the + // thread locked queue to be moved to the target thread, where it will + // either be run immediately or flushed into the thread's local queue. + self.target_queue.push(runnable).unwrap(); task } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { let thread_id = self.thread_id; - let queue: &'static ConcurrentQueue = self.target_queue; let state = self.state.clone(); move |runnable| { - queue.push(runnable).unwrap(); + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { + with_local_queue(|tls| tls.local_queue.push_back(runnable)); + } state.notify_specific_thread(thread_id, false); } } @@ -298,7 +305,7 @@ impl<'a> Executor<'a> { // from multiple locations simultaneously. As the Runnable is run after // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) } + unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } .map(|runnable| runnable.run()) .is_some() } @@ -763,13 +770,11 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - { - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. - let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) }; - if let Some(r) = local_pop { - return Some(r); - } + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) }; + if let Some(r) = local_pop { + return Some(r); } // Try the local queue. @@ -810,6 +815,13 @@ impl Runner<'_> { if let Ok(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. + // + // Instead, flush all queued tasks into the local queue to + // minimize the effort required to scan for these tasks. + // + // SAFETY: This is not being used at the same time as any + // access to LOCAL_QUEUE. + unsafe { flush_to_local(&self.local_state.thread_locked_queue) }; return Some(r); } @@ -864,15 +876,38 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Steal tasks. for _ in 0..count { - if let Ok(t) = src.pop() { - assert!(dest.push(t).is_ok()); - } else { - break; + match src.pop() { + Ok(t) => assert!(dest.push(t).is_ok()), + Err(_) => break, } } } } +/// Flushes all of the items from a queue into the thread local queue. +/// +/// # Safety +/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +unsafe fn flush_to_local(src: &ConcurrentQueue) { + let count = src.len(); + + if count > 0 { + // SAFETY: Caller assures that LOCAL_QUEUE does not have any + // overlapping accesses. + unsafe { + with_local_queue(|tls| { + // Steal tasks. + for _ in 0..count { + match src.pop() { + Ok(t) => tls.local_queue.push_front(t), + Err(_) => break, + } + } + }) + } + } +} + /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Get a reference to the state. From 45b5e15a78f3b797e58ab68f7bcd156dfe1d9b67 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 29 Jul 2025 05:28:36 -0700 Subject: [PATCH 08/68] Clean up unnecessary unsafe use --- crates/bevy_tasks/src/async_executor.rs | 110 +++--------------------- crates/bevy_tasks/src/executor.rs | 4 +- 2 files changed, 16 insertions(+), 98 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 6271598e66765..446ce57306bcb 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -8,7 +8,7 @@ use std::fmt; use std::marker::PhantomData; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; use std::thread::ThreadId; @@ -171,7 +171,7 @@ impl<'a> ThreadSpawner<'a> { /// An async executor. pub struct Executor<'a> { /// The executor state. - state: AtomicPtr, + state: Arc, /// Makes the `'a` lifetime invariant. _marker: PhantomData<&'a ()>, @@ -188,21 +188,21 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - pub const fn new() -> Executor<'a> { + pub fn new() -> Executor<'a> { Executor { - state: AtomicPtr::new(std::ptr::null_mut()), + state: Arc::new(State::new()), _marker: PhantomData, } } /// Spawns a task onto the executor. pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state().active(); + let mut active = self.state.active(); // Remove the task from the set of active tasks when the future finishes. let entry = active.vacant_entry(); let index = entry.key(); - let state = self.state_as_arc(); + let state = self.state.clone(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); // Create the task and register it in the set of active tasks. @@ -295,7 +295,7 @@ impl<'a> Executor<'a> { ThreadSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, - state: self.state_as_arc(), + state: self.state.clone(), _marker: PhantomData, } } @@ -312,12 +312,12 @@ impl<'a> Executor<'a> { /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { - self.state().run(future).await + self.state.run(future).await } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { - let state = self.state_as_arc(); + let state = self.state.clone(); move |runnable| { // Attempt to push onto the local queue first in dedicated executor threads, @@ -345,7 +345,7 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let state = self.state_as_arc(); + let state = self.state.clone(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accesed @@ -357,79 +357,17 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, false); } } - - /// Returns a pointer to the inner state. - #[inline] - fn state_ptr(&self) -> *const State { - #[cold] - fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { - let state = Arc::new(State::new()); - let ptr = Arc::into_raw(state).cast_mut(); - if let Err(actual) = atomic_ptr.compare_exchange( - std::ptr::null_mut(), - ptr, - Ordering::AcqRel, - Ordering::Acquire, - ) { - // SAFETY: This was just created from Arc::into_raw. - drop(unsafe { Arc::from_raw(ptr) }); - actual - } else { - ptr - } - } - - let mut ptr = self.state.load(Ordering::Acquire); - if ptr.is_null() { - ptr = alloc_state(&self.state); - } - ptr - } - - /// Returns a reference to the inner state. - #[inline] - fn state(&self) -> &State { - // SAFETY: So long as an Executor lives, it's state pointer will always be valid - // when accessed through state_ptr. - unsafe { &*self.state_ptr() } - } - - // Clones the inner state Arc - #[inline] - fn state_as_arc(&self) -> Arc { - // SAFETY: So long as an Executor lives, it's state pointer will always be a valid - // Arc when accessed through state_ptr. - let arc = unsafe { Arc::from_raw(self.state_ptr()) }; - let clone = arc.clone(); - std::mem::forget(arc); - clone - } } impl Drop for Executor<'_> { fn drop(&mut self) { - let ptr = *self.state.get_mut(); - if ptr.is_null() { - return; - } - - // SAFETY: As ptr is not null, it was allocated via Arc::new and converted - // via Arc::into_raw in state_ptr. - let state = unsafe { Arc::from_raw(ptr) }; - - let mut active = state.active(); + let mut active = self.state.active(); for w in active.drain() { w.wake(); } drop(active); - while state.queue.pop().is_ok() {} - } -} - -impl<'a> Default for Executor<'a> { - fn default() -> Executor<'a> { - Executor::new() + while self.state.queue.pop().is_ok() {} } } @@ -885,7 +823,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { } /// Flushes all of the items from a queue into the thread local queue. -/// +/// /// # Safety /// This must not be accessed at the same time as LOCAL_QUEUE in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { @@ -910,27 +848,7 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Get a reference to the state. - let ptr = executor.state.load(Ordering::Acquire); - if ptr.is_null() { - // The executor has not been initialized. - struct Uninitialized; - - impl fmt::Debug for Uninitialized { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("") - } - } - - return f.debug_tuple(name).field(&Uninitialized).finish(); - } - - // SAFETY: If the state pointer is not null, it must have been - // allocated properly by Arc::new and converted via Arc::into_raw - // in state_ptr. - let state = unsafe { &*ptr }; - - debug_state(state, name, f) + debug_state(&executor.state, name, f) } /// Debug implementation for `Executor` and `LocalExecutor`. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 2c7e142a0a623..118fc1f6f2865 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -32,13 +32,13 @@ pub use async_task::FallibleTask; /// If you require an executor _without_ the `Send` and `Sync` requirements, consider /// using [`LocalExecutor`] instead. #[derive(Deref, DerefMut, Default)] -pub struct Executor<'a>(ExecutorInner<'a>); +pub(crate) struct Executor<'a>(ExecutorInner<'a>); impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { + pub fn new() -> Self { Self(ExecutorInner::new()) } From ab45573f7d44ebe91437ffcd8b7c1a5f659d7a27 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 29 Jul 2025 18:54:57 -0700 Subject: [PATCH 09/68] Address potential unsoundness with ThreadSpawner --- crates/bevy_tasks/src/async_executor.rs | 99 +++++++++++-------------- crates/bevy_tasks/src/executor.rs | 2 +- crates/bevy_tasks/src/task_pool.rs | 30 ++++++-- 3 files changed, 65 insertions(+), 66 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 446ce57306bcb..f76c3399aca57 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -105,8 +105,23 @@ pub struct ThreadSpawner<'a> { } impl<'a> ThreadSpawner<'a> { + /// Spawns a task onto the specific target thread. + pub fn spawn( + &self, + future: impl Future + Send + 'static, + ) -> Task { + // SAFETY: T and `future` are both 'static, so the Task is guaranteed to not outlive it. + unsafe { self.spawn_scoped(future) } + } + /// Spawns a task onto the executor. - pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + /// + /// # Safety + /// The caller must ensure that the returned Task does not outlive 'a. + pub unsafe fn spawn_scoped( + &self, + future: impl Future + Send + 'a, + ) -> Task { let mut active = self.state.active(); // Remove the task from the set of active tasks when the future finishes. @@ -119,24 +134,13 @@ impl<'a> ThreadSpawner<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is `Send`. Therefore we do not need to worry about what thread + // the produced `Runnable` is used and dropped from. + // - `future` is not `'static`, but the caller must make sure that the Task + // and thus the `Runnable` will not outlive `'a`. + // - `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. let (runnable, task) = unsafe { Builder::new() .propagate_panic(true) @@ -209,24 +213,16 @@ impl<'a> Executor<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is `Send`. Therefore we do not need to worry about what thread + // the produced `Runnable` is used and dropped from. + // - `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // - `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. let (runnable, task) = unsafe { Builder::new() .propagate_panic(true) @@ -239,7 +235,7 @@ impl<'a> Executor<'a> { } /// Spawns a non-Send task onto the executor. - pub fn spawn_local(&self, future: impl Future + 'a) -> Task { + pub fn spawn_local(&self, future: impl Future + 'static) -> Task { // Remove the task from the set of active tasks when the future finishes. // // SAFETY: There are no instances where the value is accessed mutably @@ -260,24 +256,13 @@ impl<'a> Executor<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is not `Send`, but the produced `Runnable` does is bound + // to thread-local storage and thus cannot leave this thread of execution. + // - `future` is `'static`. + // - `self.schedule_local()` is not `Send` or `Sync` so all instances + // must not leave the current thread of execution, and it does not + // all of them are bound vy use of thread-local storage. + // - `self.schedule_local()` is `'static`, as checked below. let (runnable, task) = Builder::new() .propagate_panic(true) .spawn_unchecked(|()| future, self.schedule_local()); @@ -823,7 +808,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { } /// Flushes all of the items from a queue into the thread local queue. -/// +/// /// # Safety /// This must not be accessed at the same time as LOCAL_QUEUE in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 118fc1f6f2865..ff40adc1befd5 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -31,7 +31,7 @@ pub use async_task::FallibleTask; /// /// If you require an executor _without_ the `Send` and `Sync` requirements, consider /// using [`LocalExecutor`] instead. -#[derive(Deref, DerefMut, Default)] +#[derive(Deref, DerefMut)] pub(crate) struct Executor<'a>(ExecutorInner<'a>); impl Executor<'_> { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index ac31c86359b35..49b9d4ec28aa4 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -474,6 +474,10 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { self.spawned.push(task).unwrap(); } + #[expect( + unsafe_code, + reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + )] /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of /// [`TaskPool::scope`]'s return value. Users should generally prefer to use @@ -481,15 +485,22 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { - let task = self - .scope_spawner - .spawn(AssertUnwindSafe(f).catch_unwind()) - .fallible(); + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + let task = unsafe { + self.scope_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .fallible() + }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); } + #[expect( + unsafe_code, + reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + )] /// Spawns a scoped future onto the thread of the external thread executor. /// This is typically the main thread. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of @@ -498,10 +509,13 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_external + 'scope + Send>(&self, f: Fut) { - let task = self - .external_spawner - .spawn(AssertUnwindSafe(f).catch_unwind()) - .fallible(); + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + let task = unsafe { + self.external_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .fallible() + }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); From 05f1f40c3fe74bd5798875bd80d84bbe6b2eb9a7 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:18:34 -0700 Subject: [PATCH 10/68] Fix some CI errors --- crates/bevy_tasks/src/async_executor.rs | 42 +++++++++++++------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f76c3399aca57..c6b0d1a0bc765 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -3,18 +3,18 @@ reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] -use std::collections::VecDeque; -use std::fmt; -use std::marker::PhantomData; -use std::panic::{RefUnwindSafe, UnwindSafe}; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; -use std::task::{Context, Poll, Waker}; +use core::marker::PhantomData; +use core::panic::{RefUnwindSafe, UnwindSafe}; +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll, Waker}; use std::thread::ThreadId; +use alloc::collections::VecDeque; +use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; +use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; @@ -31,7 +31,7 @@ pub(crate) fn install_runtime_into_current_thread() { // Do not access this directly, use `with_local_queue` instead. cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { - use std::cell::RefCell; + use core::cell::RefCell; std::thread_local! { static LOCAL_QUEUE: RefCell = const { @@ -42,7 +42,7 @@ cfg_if::cfg_if! { }; } } else { - use std::cell::UnsafeCell; + use core::cell::UnsafeCell; std::thread_local! { static LOCAL_QUEUE: UnsafeCell = const { @@ -56,13 +56,13 @@ cfg_if::cfg_if! { } /// # Safety -/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { LOCAL_QUEUE.with(|tls| { cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { - f(&mut *tls.borrow_mut()) + f(&mut tls.borrow_mut()) } else { // SAFETY: This value is in thread local storage and thus can only be accesed // from one thread. The caller guarantees that this function is not used with @@ -96,6 +96,8 @@ impl Default for ThreadLocalState { } } +/// A task spawner for a specific thread. Must be created by calling [`TaskPool::current_thread_spawner`] +/// from the target thread. #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, @@ -291,7 +293,7 @@ impl<'a> Executor<'a> { // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } - .map(|runnable| runnable.run()) + .map(Runnable::run) .is_some() } @@ -392,7 +394,7 @@ impl State { /// Returns a reference to currently active tasks. fn active(&self) -> MutexGuard<'_, Slab> { - self.active.lock().unwrap_or_else(|e| e.into_inner()) + self.active.lock().unwrap_or_else(PoisonError::into_inner) } /// Notifies a sleeping ticker. @@ -725,7 +727,7 @@ impl Runner<'_> { // Remove this runner's local queue. let iter = - iter.filter(|local| !std::ptr::eq(**local, &self.local_state.stealable_queue)); + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); // Try stealing from each local queue in the list. for local in iter { @@ -773,7 +775,7 @@ impl Drop for Runner<'_> { .iter() .enumerate() .rev() - .find(|(_, local)| std::ptr::eq(**local, &self.local_state.stealable_queue)) + .find(|(_, local)| core::ptr::eq(**local, &self.local_state.stealable_queue)) { stealer_queues.remove(idx); } @@ -789,7 +791,7 @@ impl Drop for Runner<'_> { /// Steals some items from one queue into another. fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Half of `src`'s length rounded up. - let mut count = (src.len() + 1) / 2; + let mut count = src.len().div_ceil(2); if count > 0 { // Don't steal more than fits into the queue. @@ -810,12 +812,12 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { /// Flushes all of the items from a queue into the thread local queue. /// /// # Safety -/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { let count = src.len(); if count > 0 { - // SAFETY: Caller assures that LOCAL_QUEUE does not have any + // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any // overlapping accesses. unsafe { with_local_queue(|tls| { @@ -826,7 +828,7 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { Err(_) => break, } } - }) + }); } } } From 0f226bfd6992232355844926ce5f91867970effb Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:20:38 -0700 Subject: [PATCH 11/68] Update docs --- docs/cargo_features.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/cargo_features.md b/docs/cargo_features.md index 6e409cf998bb7..e23ecf84eb969 100644 --- a/docs/cargo_features.md +++ b/docs/cargo_features.md @@ -14,7 +14,6 @@ The default feature set enables most of the expected features of a game engine, |android-game-activity|Android GameActivity support. Default, choose between this and `android-native-activity`.| |android_shared_stdcxx|Enable using a shared stdlib for cxx on Android| |animation|Enable animation support, and glTF animation loading| -|async_executor|Uses `async-executor` as a task execution backend.| |bevy_animation|Provides animation functionality| |bevy_anti_aliasing|Provides various anti aliasing solutions| |bevy_asset|Provides asset functionality| From 9c2847391368a0a5c3eb91fb625e380656ed9341 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:21:09 -0700 Subject: [PATCH 12/68] Format TOML files --- crates/bevy_tasks/Cargo.toml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index a27379e05b08c..54202a0db7e77 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -28,8 +28,8 @@ multi_threaded = [ ## on `no_std` targets, but provides access to certain additional features on ## supported platforms. std = [ - "futures-lite/std", - "async-task/std", + "futures-lite/std", + "async-task/std", "bevy_platform/std", "fastrand/std", "dep:slab", @@ -44,11 +44,7 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = [ - "bevy_platform/web", - "dep:wasm-bindgen-futures", - "dep:futures-channel", -] +web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel"] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ From 7f8c93204ca444c1193ce953dd6586e12d03aac4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:24:14 -0700 Subject: [PATCH 13/68] Make note on ThreadLocal soundness hole --- crates/bevy_tasks/src/async_executor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index c6b0d1a0bc765..cbf83e0daa77e 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -21,6 +21,8 @@ use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; +// ThreadLocalState *must* stay `Sync` due to a currently existing soundness hole. +// See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); pub(crate) fn install_runtime_into_current_thread() { From f84028e0c4efbe7e3358ee472f1dc96cd7b5370f Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 22:51:50 -0700 Subject: [PATCH 14/68] Allow clippy warning --- crates/bevy_ecs/src/schedule/executor/mod.rs | 2 +- .../bevy_ecs/src/schedule/executor/multi_threaded.rs | 12 ++++++------ crates/bevy_render/src/pipelined_rendering.rs | 8 ++++---- crates/bevy_tasks/src/async_executor.rs | 2 +- crates/bevy_tasks/src/executor.rs | 5 +---- crates/bevy_tasks/src/task_pool.rs | 9 +++++---- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/crates/bevy_ecs/src/schedule/executor/mod.rs b/crates/bevy_ecs/src/schedule/executor/mod.rs index 655a71caa70e6..3dce10640d333 100644 --- a/crates/bevy_ecs/src/schedule/executor/mod.rs +++ b/crates/bevy_ecs/src/schedule/executor/mod.rs @@ -11,7 +11,7 @@ use core::any::TypeId; pub use self::{simple::SimpleExecutor, single_threaded::SingleThreadedExecutor}; #[cfg(feature = "std")] -pub use self::multi_threaded::{MainThreadExecutor, MultiThreadedExecutor}; +pub use self::multi_threaded::{MainThreadSpawner, MultiThreadedExecutor}; use fixedbitset::FixedBitSet; diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index 5637a12ab9fa2..b16e6e22a7b53 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -269,7 +269,7 @@ impl SystemExecutor for MultiThreadedExecutor { } let thread_executor = world - .get_resource::() + .get_resource::() .map(|e| e.0.clone()); let environment = &Environment::new(self, schedule, world); @@ -861,20 +861,20 @@ unsafe fn evaluate_and_fold_conditions( .fold(true, |acc, res| acc && res) } -/// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread +/// New-typed [`ThreadSpawner`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadExecutor(pub ThreadSpawner<'static>); +pub struct MainThreadSpawner(pub ThreadSpawner<'static>); -impl Default for MainThreadExecutor { +impl Default for MainThreadSpawner { fn default() -> Self { Self::new() } } -impl MainThreadExecutor { +impl MainThreadSpawner { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadExecutor(ComputeTaskPool::get().current_thread_spawner()) + MainThreadSpawner(ComputeTaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 9769b051bd341..61f9a446a2150 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -3,7 +3,7 @@ use async_channel::{Receiver, Sender}; use bevy_app::{App, AppExit, AppLabel, Plugin, SubApp}; use bevy_ecs::{ resource::Resource, - schedule::MainThreadExecutor, + schedule::MainThreadSpawner, world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; @@ -114,7 +114,7 @@ impl Plugin for PipelinedRenderingPlugin { if app.get_sub_app(RenderApp).is_none() { return; } - app.insert_resource(MainThreadExecutor::new()); + app.insert_resource(MainThreadSpawner::new()); let mut sub_app = SubApp::new(); sub_app.set_extract(renderer_extract); @@ -136,7 +136,7 @@ impl Plugin for PipelinedRenderingPlugin { .expect("Unable to get RenderApp. Another plugin may have removed the RenderApp before PipelinedRenderingPlugin"); // clone main thread executor to render world - let executor = app.world().get_resource::().unwrap(); + let executor = app.world().get_resource::().unwrap(); render_app.world_mut().insert_resource(executor.clone()); render_to_app_sender.send_blocking(render_app).unwrap(); @@ -181,7 +181,7 @@ impl Plugin for PipelinedRenderingPlugin { // This function waits for the rendering world to be received, // runs extract, and then sends the rendering world back to the render thread. fn renderer_extract(app_world: &mut World, _world: &mut World) { - app_world.resource_scope(|world, main_thread_executor: Mut| { + app_world.resource_scope(|world, main_thread_executor: Mut| { world.resource_scope(|world, mut render_channels: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index cbf83e0daa77e..be64d4a851805 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -902,6 +902,7 @@ impl Drop for CallOnDrop { } pin_project! { + #[expect(clippy::unused_unit)] /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] @@ -930,7 +931,6 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { use super::Executor; - use super::ThreadLocalState; use super::THREAD_LOCAL_STATE; fn _ensure_send_and_sync() { diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index ff40adc1befd5..2ac1fe9bde025 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -26,11 +26,8 @@ cfg_if::cfg_if! { pub use async_task::FallibleTask; /// Wrapper around a multi-threading-aware async executor. -/// Spawning will generally require tasks to be `Send` and `Sync` to allow multiple +/// spawning will generally require tasks to be `send` and `sync` to allow multiple /// threads to send/receive/advance tasks. -/// -/// If you require an executor _without_ the `Send` and `Sync` requirements, consider -/// using [`LocalExecutor`] instead. #[derive(Deref, DerefMut)] pub(crate) struct Executor<'a>(ExecutorInner<'a>); diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 49b9d4ec28aa4..866f3deeb65e5 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -136,7 +136,8 @@ pub struct TaskPool { } impl TaskPool { - /// Each thread will only have one `ThreadExecutor`, otherwise, there are good chances they will deadlock + /// Creates a [`ThreadSpawner`] for this current thread of execution. + /// Can be used to spawn new tasks to execute exclusively on this thread. pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { self.executor.current_thread_spawner() } @@ -299,9 +300,9 @@ impl TaskPool { self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } - /// This allows passing an external executor to spawn tasks on. When you pass an external executor - /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. - /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. + /// This allows passing an external [`ThreadSpawner`] to spawn tasks to. When you pass an external spawner + /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadSpawner`] originated from. + /// If [`None`] is passed the scope will use a [`ThreadSpawner`] that is ticked on the current thread. /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( From f44d302edbad638a584b55bae323ea6e9788a75d Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 22:57:48 -0700 Subject: [PATCH 15/68] Fix typos --- crates/bevy_tasks/src/async_executor.rs | 6 +++--- crates/bevy_tasks/src/task_pool.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index be64d4a851805..d95e6569256d1 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -66,7 +66,7 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { if #[cfg(all(debug_assertions, not(miri)))] { f(&mut tls.borrow_mut()) } else { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. The caller guarantees that this function is not used with // LOCAL_QUEUE in any way. f(unsafe { &mut *tls.get() }) @@ -165,7 +165,7 @@ impl<'a> ThreadSpawner<'a> { let state = self.state.clone(); move |runnable| { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. unsafe { @@ -337,7 +337,7 @@ impl<'a> Executor<'a> { let state = self.state.clone(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. unsafe { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 866f3deeb65e5..d3d9801aaa711 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -477,7 +477,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + reason = "ThreadSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of @@ -500,7 +500,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + reason = "ThreadSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread of the external thread executor. /// This is typically the main thread. The scope *must* outlive From 227258ebce857aaabd247f2d74df6da0d1885c59 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 30 Jul 2025 00:50:43 -0700 Subject: [PATCH 16/68] Try to fix CI outside of miri --- crates/bevy_tasks/src/async_executor.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index d95e6569256d1..ea79e448b4008 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -2,6 +2,10 @@ unsafe_code, reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] +#![expect( + clippy::unused_unit, + reason = "False positive detection on {Async}CallOnDrop" +)] use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; @@ -100,6 +104,8 @@ impl Default for ThreadLocalState { /// A task spawner for a specific thread. Must be created by calling [`TaskPool::current_thread_spawner`] /// from the target thread. +/// +/// [`TaskPool::current_thread_spawner`]: crate::TaskPool::current_thread_spawner #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, @@ -902,7 +908,6 @@ impl Drop for CallOnDrop { } pin_project! { - #[expect(clippy::unused_unit)] /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] From d88035d5f830e6f2263eb79239836e9c43464d10 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 16:32:18 -0700 Subject: [PATCH 17/68] Remove depnedency on concurrent-queue --- crates/bevy_tasks/Cargo.toml | 3 - crates/bevy_tasks/src/async_executor.rs | 91 ++++++++++++++++--------- crates/bevy_tasks/src/task_pool.rs | 27 +++----- 3 files changed, 68 insertions(+), 53 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 54202a0db7e77..407505e5855b0 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -18,7 +18,6 @@ default = ["std"] multi_threaded = [ "std", "dep:async-channel", - "dep:concurrent-queue", "dep:fastrand", ] @@ -34,7 +33,6 @@ std = [ "fastrand/std", "dep:slab", "dep:thread_local", - "dep:concurrent-queue", "dep:pin-project-lite", ] @@ -66,7 +64,6 @@ thread_local = { version = "1.1", optional = true } fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } -concurrent-queue = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } crossbeam-queue = { version = "0.3", default-features = false, features = [ "alloc", diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index ea79e448b4008..a413444ba1468 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,7 +19,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use concurrent_queue::ConcurrentQueue; +use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; use slab::Slab; @@ -87,8 +87,8 @@ struct LocalQueue { struct ThreadLocalState { executor_thread: AtomicBool, thread_id: ThreadId, - stealable_queue: ConcurrentQueue, - thread_locked_queue: ConcurrentQueue, + stealable_queue: ArrayQueue, + thread_locked_queue: SegQueue, } impl Default for ThreadLocalState { @@ -96,8 +96,8 @@ impl Default for ThreadLocalState { Self { executor_thread: AtomicBool::new(false), thread_id: std::thread::current().id(), - stealable_queue: ConcurrentQueue::bounded(512), - thread_locked_queue: ConcurrentQueue::unbounded(), + stealable_queue: ArrayQueue::new(512), + thread_locked_queue: SegQueue::new(), } } } @@ -109,7 +109,7 @@ impl Default for ThreadLocalState { #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, - target_queue: &'static ConcurrentQueue, + target_queue: &'static SegQueue, state: Arc, _marker: PhantomData<&'a ()>, } @@ -161,7 +161,7 @@ impl<'a> ThreadSpawner<'a> { // Instead of directly scheduling this task, it's put into the onto the // thread locked queue to be moved to the target thread, where it will // either be run immediately or flushed into the thread's local queue. - self.target_queue.push(runnable).unwrap(); + self.target_queue.push(runnable); task } @@ -324,7 +324,7 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, true); return; } - Err(r) => r.into_inner(), + Err(r) => r, } } else { runnable @@ -333,7 +333,7 @@ impl<'a> Executor<'a> { runnable }; // Otherwise push onto the global queue instead. - state.queue.push(runnable).unwrap(); + state.queue.push(runnable); state.notify(); } } @@ -362,17 +362,17 @@ impl Drop for Executor<'_> { } drop(active); - while self.state.queue.pop().is_ok() {} + while self.state.queue.pop().is_some() {} } } /// The state of a executor. struct State { /// The global queue. - queue: ConcurrentQueue, + queue: SegQueue, /// Local queues created by runners. - stealer_queues: RwLock>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -388,7 +388,7 @@ impl State { /// Creates state for a new executor. const fn new() -> State { State { - queue: ConcurrentQueue::unbounded(), + queue: SegQueue::new(), stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { @@ -711,12 +711,12 @@ impl Runner<'_> { } // Try the local queue. - if let Ok(r) = self.local_state.stealable_queue.pop() { + if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. - if let Ok(r) = self.state.queue.pop() { + if let Some(r) = self.state.queue.pop() { steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } @@ -739,13 +739,13 @@ impl Runner<'_> { // Try stealing from each local queue in the list. for local in iter { - steal(local, &self.local_state.stealable_queue); - if let Ok(r) = self.local_state.stealable_queue.pop() { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } } - if let Ok(r) = self.local_state.thread_locked_queue.pop() { + if let Some(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. // @@ -790,29 +790,54 @@ impl Drop for Runner<'_> { } // Re-schedule remaining tasks in the local queue. - while let Ok(r) = self.local_state.stealable_queue.pop() { + while let Some(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } } +trait WorkQueue { + fn stealable_count(&self) -> usize; + fn queue_pop(&self) -> Option; +} + +impl WorkQueue for ArrayQueue { + #[inline] + fn stealable_count(&self) -> usize { + self.len().div_ceil(2) + } + + #[inline] + fn queue_pop(&self) -> Option { + self.pop() + } +} + +impl WorkQueue for SegQueue { + #[inline] + fn stealable_count(&self) -> usize { + self.len() + } + + #[inline] + fn queue_pop(&self) -> Option { + self.pop() + } +} + /// Steals some items from one queue into another. -fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { +fn steal>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. - let mut count = src.len().div_ceil(2); + let mut count = src.stealable_count(); if count > 0 { // Don't steal more than fits into the queue. - if let Some(cap) = dest.capacity() { - count = count.min(cap - dest.len()); - } + count = count.min(dest.capacity() - dest.len()); // Steal tasks. for _ in 0..count { - match src.pop() { - Ok(t) => assert!(dest.push(t).is_ok()), - Err(_) => break, - } + let Some(val) = src.queue_pop() else { break }; + assert!(dest.push(val).is_ok()); } } } @@ -821,7 +846,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { /// /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. -unsafe fn flush_to_local(src: &ConcurrentQueue) { +unsafe fn flush_to_local(src: &SegQueue) { let count = src.len(); if count > 0 { @@ -831,10 +856,8 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { with_local_queue(|tls| { // Steal tasks. for _ in 0..count { - match src.pop() { - Ok(t) => tls.local_queue.push_front(t), - Err(_) => break, - } + let Some(val) = src.queue_pop() else { break }; + tls.local_queue.push_front(val); } }); } @@ -862,7 +885,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index d3d9801aaa711..972b554783b98 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -5,7 +5,7 @@ use std::thread::{self, JoinHandle}; use crate::async_executor::ThreadSpawner; use crate::executor::FallibleTask; use bevy_platform::sync::Arc; -use concurrent_queue::ConcurrentQueue; +use crossbeam_queue::SegQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; @@ -345,13 +345,12 @@ impl TaskPool { let executor: &crate::executor::Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - let spawned: ConcurrentQueue>>> = - ConcurrentQueue::unbounded(); + let spawned: SegQueue>>> = + SegQueue::new(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env ConcurrentQueue< - FallibleTask>>, - > = unsafe { mem::transmute(&spawned) }; + let spawned: &'env SegQueue>>> = + unsafe { mem::transmute(&spawned) }; let scope = Scope { executor, @@ -373,7 +372,7 @@ impl TaskPool { } else { block_on(self.executor.run(async move { let mut results = Vec::with_capacity(spawned.len()); - while let Ok(task) = spawned.pop() { + while let Some(task) = spawned.pop() { if let Some(res) = task.await { match res { Ok(res) => results.push(res), @@ -450,7 +449,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope crate::executor::Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope ConcurrentQueue>>>, + spawned: &'scope SegQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, @@ -470,9 +469,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .executor .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); - // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } #[expect( @@ -493,9 +490,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) .fallible() }; - // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } #[expect( @@ -519,7 +514,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } } @@ -529,7 +524,7 @@ where { fn drop(&mut self) { block_on(async { - while let Ok(task) = self.spawned.pop() { + while let Some(task) = self.spawned.pop() { task.cancel().await; } }); From 202873615b32082712e0a218431e167596ec066c Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 16:40:22 -0700 Subject: [PATCH 18/68] Revert "Clean up unnecessary unsafe use" This reverts commit 8f1eaa71bcd28e79676e5930f29a2afe7daa6b79. --- crates/bevy_tasks/src/async_executor.rs | 101 +++++++++++++++++++++--- crates/bevy_tasks/src/executor.rs | 2 +- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index a413444ba1468..916ac6878a129 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -10,9 +10,8 @@ use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; -use std::thread::ThreadId; use alloc::collections::VecDeque; use alloc::fmt; @@ -185,7 +184,7 @@ impl<'a> ThreadSpawner<'a> { /// An async executor. pub struct Executor<'a> { /// The executor state. - state: Arc, + state: AtomicPtr, /// Makes the `'a` lifetime invariant. _marker: PhantomData<&'a ()>, @@ -202,21 +201,21 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - pub fn new() -> Executor<'a> { + pub const fn new() -> Executor<'a> { Executor { - state: Arc::new(State::new()), + state: AtomicPtr::new(std::ptr::null_mut()), _marker: PhantomData, } } /// Spawns a task onto the executor. pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state.active(); + let mut active = self.state().active(); // Remove the task from the set of active tasks when the future finishes. let entry = active.vacant_entry(); let index = entry.key(); - let state = self.state.clone(); + let state = self.state_as_arc(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); // Create the task and register it in the set of active tasks. @@ -290,7 +289,7 @@ impl<'a> Executor<'a> { ThreadSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, - state: self.state.clone(), + state: self.state_as_arc(), _marker: PhantomData, } } @@ -307,12 +306,12 @@ impl<'a> Executor<'a> { /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { - self.state.run(future).await + self.state().run(future).await } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { - let state = self.state.clone(); + let state = self.state_as_arc(); move |runnable| { // Attempt to push onto the local queue first in dedicated executor threads, @@ -340,7 +339,7 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let state = self.state.clone(); + let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed @@ -352,11 +351,67 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, false); } } + + /// Returns a pointer to the inner state. + #[inline] + fn state_ptr(&self) -> *const State { + #[cold] + fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { + let state = Arc::new(State::new()); + let ptr = Arc::into_raw(state).cast_mut(); + if let Err(actual) = atomic_ptr.compare_exchange( + std::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) { + // SAFETY: This was just created from Arc::into_raw. + drop(unsafe { Arc::from_raw(ptr) }); + actual + } else { + ptr + } + } + + let mut ptr = self.state.load(Ordering::Acquire); + if ptr.is_null() { + ptr = alloc_state(&self.state); + } + ptr + } + + /// Returns a reference to the inner state. + #[inline] + fn state(&self) -> &State { + // SAFETY: So long as an Executor lives, it's state pointer will always be valid + // when accessed through state_ptr. + unsafe { &*self.state_ptr() } + } + + // Clones the inner state Arc + #[inline] + fn state_as_arc(&self) -> Arc { + // SAFETY: So long as an Executor lives, it's state pointer will always be a valid + // Arc when accessed through state_ptr. + let arc = unsafe { Arc::from_raw(self.state_ptr()) }; + let clone = arc.clone(); + std::mem::forget(arc); + clone + } } impl Drop for Executor<'_> { fn drop(&mut self) { - let mut active = self.state.active(); + let ptr = *self.state.get_mut(); + if ptr.is_null() { + return; + } + + // SAFETY: As ptr is not null, it was allocated via Arc::new and converted + // via Arc::into_raw in state_ptr. + let state = unsafe { Arc::from_raw(ptr) }; + + let mut active = state.active(); for w in active.drain() { w.wake(); } @@ -866,7 +921,27 @@ unsafe fn flush_to_local(src: &SegQueue) { /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { - debug_state(&executor.state, name, f) + // Get a reference to the state. + let ptr = executor.state.load(Ordering::Acquire); + if ptr.is_null() { + // The executor has not been initialized. + struct Uninitialized; + + impl fmt::Debug for Uninitialized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + + return f.debug_tuple(name).field(&Uninitialized).finish(); + } + + // SAFETY: If the state pointer is not null, it must have been + // allocated properly by Arc::new and converted via Arc::into_raw + // in state_ptr. + let state = unsafe { &*ptr }; + + debug_state(state, name, f) } /// Debug implementation for `Executor` and `LocalExecutor`. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 2ac1fe9bde025..d8f6f862ed66d 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -35,7 +35,7 @@ impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] #[allow(dead_code, reason = "not all feature flags require this function")] - pub fn new() -> Self { + pub const fn new() -> Self { Self(ExecutorInner::new()) } From da66005b058efc5d49787913571d31be689699d9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 17:02:18 -0700 Subject: [PATCH 19/68] Fix single-threaded builds --- crates/bevy_tasks/src/async_executor.rs | 44 ++++++++++++++++++- .../src/single_threaded_task_pool.rs | 5 ++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 916ac6878a129..f48670943984f 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -6,12 +6,17 @@ clippy::unused_unit, reason = "False positive detection on {Async}CallOnDrop" )] +#![expect( + dead_code, + reason = "Not every function is used in all feature combinations." +)] use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; +use std::thread::ThreadId; use alloc::collections::VecDeque; use alloc::fmt; @@ -245,6 +250,18 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'static) -> Task { + // SAFETY: future is 'static + unsafe { self.spawn_local_scoped(future) } + } + + /// Spawns a non-'static and non-Send task onto the executor. + /// + /// # Safety + /// The caller must ensure that the returned Task does not outlive 'a. + pub unsafe fn spawn_local_scoped( + &self, + future: impl Future + 'a, + ) -> Task { // Remove the task from the set of active tasks when the future finishes. // // SAFETY: There are no instances where the value is accessed mutably @@ -267,7 +284,9 @@ impl<'a> Executor<'a> { // // - `future` is not `Send`, but the produced `Runnable` does is bound // to thread-local storage and thus cannot leave this thread of execution. - // - `future` is `'static`. + // - `future` may not be `'static`, but the caller is required to ensure that + // the future does not outlive the borrowed non-metadata variables of the + // task. // - `self.schedule_local()` is not `Send` or `Sync` so all instances // must not leave the current thread of execution, and it does not // all of them are bound vy use of thread-local storage. @@ -294,6 +313,27 @@ impl<'a> Executor<'a> { } } + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + pub fn try_tick(&self) -> bool { + let state = self.state(); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } + .or_else(|| state.queue.pop()) + .or_else(|| { + THREAD_LOCAL_STATE + .get_or_default() + .thread_locked_queue + .pop() + }) + .map(Runnable::run) + .is_some() + } + pub fn try_tick_local() -> bool { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. As the Runnable is run after @@ -417,7 +457,7 @@ impl Drop for Executor<'_> { } drop(active); - while self.state.queue.pop().is_some() {} + while state.queue.pop().is_some() {} } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 33dd8f3c0acd5..4755c95efd31c 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -254,6 +254,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -274,7 +275,9 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; - self.executor.spawn_local(f).detach(); + // SAFETY: The surrounding scope will not terminate until all local tasks are done + // ensuring that the borrowed variables do not outlive the detatched task. + unsafe { self.executor.spawn_local_scoped(f).detach() }; } } From 8d4e5cabb7aca09f9d23d6e14aef0a9c20a90b6d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:00:17 -0700 Subject: [PATCH 20/68] Fix builds for no_std builds --- crates/bevy_tasks/Cargo.toml | 6 +- crates/bevy_tasks/src/async_executor.rs | 25 +------- crates/bevy_tasks/src/edge_executor.rs | 9 +++ crates/bevy_tasks/src/executor.rs | 5 -- crates/bevy_tasks/src/lib.rs | 5 +- .../src/single_threaded_task_pool.rs | 60 +++++++++++-------- crates/bevy_tasks/src/task_pool.rs | 9 ++- crates/bevy_tasks/src/usages.rs | 3 +- 8 files changed, 58 insertions(+), 64 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 407505e5855b0..b4830a45674e7 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -15,11 +15,7 @@ default = ["std"] ## Enables multi-threading support. ## Without this feature, all tasks will be run on a single thread. -multi_threaded = [ - "std", - "dep:async-channel", - "dep:fastrand", -] +multi_threaded = ["std", "dep:async-channel", "dep:fastrand"] # Platform Compatibility diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f48670943984f..9d6677118951b 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -6,9 +6,9 @@ clippy::unused_unit, reason = "False positive detection on {Async}CallOnDrop" )] -#![expect( +#![allow( dead_code, - reason = "Not every function is used in all feature combinations." + reason = "Not all functions are used with every feature combination" )] use core::marker::PhantomData; @@ -313,27 +313,6 @@ impl<'a> Executor<'a> { } } - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - pub fn try_tick(&self) -> bool { - let state = self.state(); - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. As the Runnable is run after - // this scope closes, the AsyncCallOnDrop around the future will be invoked - // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } - .or_else(|| state.queue.pop()) - .or_else(|| { - THREAD_LOCAL_STATE - .get_or_default() - .thread_locked_queue - .pop() - }) - .map(Runnable::run) - .is_some() - } - pub fn try_tick_local() -> bool { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. As the Runnable is run after diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 70e11c8a433cf..c182fc230770e 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -97,6 +97,15 @@ impl<'a, const C: usize> Executor<'a, C> { unsafe { self.spawn_unchecked(fut) } } + pub fn spawn_local(&self, fut: F) -> Task + where + F: Future + Send + 'a, + F::Output: Send + 'a, + { + // SAFETY: Original implementation missing safety documentation + unsafe { self.spawn_unchecked(fut) } + } + /// Attempts to run a task if at least one is scheduled. /// /// Running a scheduled task means simply polling its future once. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index d8f6f862ed66d..e8cb779e0355d 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -38,11 +38,6 @@ impl Executor<'_> { pub const fn new() -> Self { Self(ExecutorInner::new()) } - - #[inline] - pub fn try_tick_local() -> bool { - ExecutorInner::try_tick_local() - } } impl UnwindSafe for Executor<'_> {} diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 2e04705b13fb0..2ca5e8c50c241 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -61,12 +61,11 @@ cfg_if::cfg_if! { if #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { mod task_pool; - pub use task_pool::{Scope, TaskPool, TaskPoolBuilder}; - pub use async_executor::ThreadSpawner; + pub use task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; } else if #[cfg(any(target_arch = "wasm32", not(feature = "multi_threaded")))] { mod single_threaded_task_pool; - pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; + pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 4755c95efd31c..bab9a7c4ce8e1 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -4,18 +4,19 @@ use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; -#[cfg(not(feature = "std"))] -use bevy_platform::sync::{Mutex, PoisonError}; - use crate::executor::Executor; static EXECUTOR: Executor = Executor::new(); -#[cfg(feature = "std")] -type ScopeResult = alloc::rc::Rc>>; +cfg_if::cfg_if! { + if #[cfg(feature = "std")] { + type ScopeResult = alloc::rc::Rc>>; + } else { + use bevy_platform::sync::{Mutex, PoisonError}; -#[cfg(not(feature = "std"))] -type ScopeResult = Arc>>; + type ScopeResult = Arc>>; + } +} /// Used to create a [`TaskPool`]. #[derive(Debug, Default, Clone)] @@ -27,8 +28,8 @@ pub struct TaskPoolBuilder {} /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread /// and so the [`ThreadExecutor`] does nothing. #[derive(Default)] -pub struct ThreadExecutor<'a>(PhantomData<&'a ()>); -impl<'a> ThreadExecutor<'a> { +pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); +impl<'a> ThreadSpawner<'a> { /// Creates a new `ThreadExecutor` pub fn new() -> Self { Self::default() @@ -79,8 +80,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn get_thread_executor() -> Arc> { - Arc::new(ThreadExecutor::new()) + pub fn get_thread_executor() -> Arc> { + Arc::new(ThreadSpawner::new()) } /// Create a `TaskPool` with the default configuration. @@ -119,7 +120,7 @@ impl TaskPool { pub fn scope_with_executor<'env, F, T>( &self, _tick_task_pool_executor: bool, - _thread_executor: Option<&ThreadExecutor>, + _thread_executor: Option<&ThreadSpawner>, f: F, ) -> Vec where @@ -155,7 +156,7 @@ impl TaskPool { f(scope_ref); // Loop until all tasks are done - while executor.try_tick() {} + while Self::try_tick_local() {} let results = scope.results.borrow(); results @@ -189,16 +190,10 @@ impl TaskPool { cfg_if::cfg_if! { if #[cfg(all(target_arch = "wasm32", feature = "web"))] { Task::wrap_future(future) - } else if #[cfg(feature = "std")] { - let task = EXECUTOR.spawn_local(future); - // Loop until all tasks are done - while EXECUTOR.try_tick() {} - - Task::new(task) } else { - EXECUTOR.spawn_local(future); + let task = EXECUTOR.spawn_local(future); // Loop until all tasks are done - while EXECUTOR.try_tick() {} + while Self::try_tick_local() {} Task::new(task) } @@ -215,6 +210,16 @@ impl TaskPool { { self.spawn(future) } + + pub(crate) fn try_tick_local() -> bool { + cfg_if::cfg_if! { + if #[cfg(feature = "std")] { + crate::async_executor::Executor::try_tick_local() + } else { + EXECUTOR.try_tick() + } + } + } } /// A `TaskPool` scope for running one or more non-`'static` futures. @@ -254,7 +259,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } - #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] + #[allow(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -275,9 +280,16 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; + + #[cfg(feature = "std")] // SAFETY: The surrounding scope will not terminate until all local tasks are done - // ensuring that the borrowed variables do not outlive the detatched task. - unsafe { self.executor.spawn_local_scoped(f).detach() }; + // ensuring that the borrowed variables do not outlive the detached task. + unsafe { + self.executor.spawn_local_scoped(f).detach() + }; + + #[cfg(not(feature = "std"))] + self.executor.spawn(f).detach(); } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 972b554783b98..ab827f95812e3 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,14 +2,15 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; use std::thread::{self, JoinHandle}; -use crate::async_executor::ThreadSpawner; -use crate::executor::FallibleTask; +use crate::{async_executor::Executor, executor::FallibleTask}; use bevy_platform::sync::Arc; use crossbeam_queue::SegQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; +pub use crate::async_executor::ThreadSpawner; + struct CallOnDrop(Option>); impl Drop for CallOnDrop { @@ -419,6 +420,10 @@ impl TaskPool { { Task::new(self.executor.spawn_local(future)) } + + pub(crate) fn try_tick_local() -> bool { + Executor::try_tick_local() + } } impl Default for TaskPool { diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 8bd0bd3182cd2..a4d614a297f0c 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,5 +1,4 @@ use super::TaskPool; -use crate::executor::Executor; use bevy_platform::sync::OnceLock; use core::ops::Deref; @@ -85,7 +84,7 @@ taskpool! { #[cfg(not(all(target_arch = "wasm32", feature = "web")))] pub fn tick_global_task_pools_on_main_thread() { for _ in 0..100 { - if !Executor::try_tick_local() { + if !TaskPool::try_tick_local() { break; } } From 3cb694f598f320c42496fbcab51fc507aa116852 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:04:20 -0700 Subject: [PATCH 21/68] Shut up Clippy --- crates/bevy_tasks/src/async_executor.rs | 6 +++--- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 9d6677118951b..dde1c114f1e3b 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -208,7 +208,7 @@ impl<'a> Executor<'a> { /// Creates a new executor. pub const fn new() -> Executor<'a> { Executor { - state: AtomicPtr::new(std::ptr::null_mut()), + state: AtomicPtr::new(core::ptr::null_mut()), _marker: PhantomData, } } @@ -379,7 +379,7 @@ impl<'a> Executor<'a> { let state = Arc::new(State::new()); let ptr = Arc::into_raw(state).cast_mut(); if let Err(actual) = atomic_ptr.compare_exchange( - std::ptr::null_mut(), + core::ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire, @@ -414,7 +414,7 @@ impl<'a> Executor<'a> { // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; let clone = arc.clone(); - std::mem::forget(arc); + core::mem::forget(arc); clone } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index bab9a7c4ce8e1..b839b180cd654 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -27,7 +27,7 @@ pub struct TaskPoolBuilder {} /// tasks on a specific thread. But the wasm task pool just calls /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread /// and so the [`ThreadExecutor`] does nothing. -#[derive(Default)] +#[derive(Default, Clone)] pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); impl<'a> ThreadSpawner<'a> { /// Creates a new `ThreadExecutor` From d0ef3f4584272af875097094ff62079964919c80 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:11:47 -0700 Subject: [PATCH 22/68] Fix bevy_ecs builds --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index b839b180cd654..b3009ab2b9381 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -108,7 +108,7 @@ impl TaskPool { F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), T: Send + 'static, { - self.scope_with_executor(false, None, f) + self.scope_with_executor(None, f) } /// Allows spawning non-`'static` futures on the thread pool. The function takes a callback, @@ -119,7 +119,6 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] pub fn scope_with_executor<'env, F, T>( &self, - _tick_task_pool_executor: bool, _thread_executor: Option<&ThreadSpawner>, f: F, ) -> Vec From 25233ffa53ce554b5f4ab22832ce1f44fcf2b11d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:14:50 -0700 Subject: [PATCH 23/68] Whoops --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index b3009ab2b9381..45d9a6720dac8 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -80,8 +80,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn get_thread_executor() -> Arc> { - Arc::new(ThreadSpawner::new()) + pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + ThreadSpawner::new() } /// Create a `TaskPool` with the default configuration. @@ -119,7 +119,7 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] pub fn scope_with_executor<'env, F, T>( &self, - _thread_executor: Option<&ThreadSpawner>, + _thread_executor: Option, f: F, ) -> Vec where From 69a13893d4df1b7b1b139f25711b90e02fdc8ce4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 19:05:42 -0700 Subject: [PATCH 24/68] Arc is only used when no_std --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 45d9a6720dac8..e23e18f9a7307 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,5 +1,4 @@ use alloc::{string::String, vec::Vec}; -use bevy_platform::sync::Arc; use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; @@ -13,6 +12,7 @@ cfg_if::cfg_if! { type ScopeResult = alloc::rc::Rc>>; } else { use bevy_platform::sync::{Mutex, PoisonError}; + use bevy_platform::sync::Arc; type ScopeResult = Arc>>; } From 943f13511f7ee5e5ee74aca71d1aea9eec6b9512 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 19:51:45 -0700 Subject: [PATCH 25/68] Remove now unused mentions of LocalExecutor --- crates/bevy_tasks/src/async_executor.rs | 23 +--- crates/bevy_tasks/src/edge_executor.rs | 175 ------------------------ 2 files changed, 2 insertions(+), 196 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index dde1c114f1e3b..932962d4563f6 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -82,7 +82,6 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { } }) } - struct LocalQueue { local_queue: VecDeque, local_active: Slab, @@ -938,7 +937,7 @@ unsafe fn flush_to_local(src: &SegQueue) { } } -/// Debug implementation for `Executor` and `LocalExecutor`. +/// Debug implementation for `Executor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Get a reference to the state. let ptr = executor.state.load(Ordering::Acquire); @@ -963,7 +962,7 @@ fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_ debug_state(state, name, f) } -/// Debug implementation for `Executor` and `LocalExecutor`. +/// Debug implementation for `Executor`. fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// Debug wrapper for the number of active tasks. struct ActiveTasks<'a>(&'a Mutex>); @@ -1071,23 +1070,5 @@ mod test { is_sync(ex.current_thread_spawner()); is_send(THREAD_LOCAL_STATE.get_or_default()); is_sync(THREAD_LOCAL_STATE.get_or_default()); - - /// ```compile_fail - /// use crate::async_executor::LocalExecutor; - /// use futures_lite::future::pending; - /// - /// fn is_send(_: T) {} - /// fn is_sync(_: T) {} - /// - /// is_send::>(LocalExecutor::new()); - /// is_sync::>(LocalExecutor::new()); - /// - /// let ex = LocalExecutor::new(); - /// is_send(ex.run(pending::<()>())); - /// is_sync(ex.run(pending::<()>())); - /// is_send(ex.tick()); - /// is_sync(ex.tick()); - /// ``` - fn _negative_test() {} } } diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index c182fc230770e..800ae57a70798 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -307,141 +307,6 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} -/// A thread-local executor. -/// -/// The executor can only be run on the thread that created it. -/// -/// # Examples -/// -/// ```ignore -/// use edge_executor::{LocalExecutor, block_on}; -/// -/// let local_ex: LocalExecutor = Default::default(); -/// -/// block_on(local_ex.run(async { -/// println!("Hello world!"); -/// })); -/// ``` -pub struct LocalExecutor<'a, const C: usize = 64> { - executor: Executor<'a, C>, - _not_send: PhantomData>>, -} - -impl<'a, const C: usize> LocalExecutor<'a, C> { - /// Creates a single-threaded executor. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// ``` - pub const fn new() -> Self { - Self { - executor: Executor::::new(), - _not_send: PhantomData, - } - } - - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` - /// - /// Note that if the executor's queue size is equal to the number of currently - /// spawned and running tasks, spawning this additional task might cause the executor to panic - /// later, when the task is scheduled for polling. - pub fn spawn(&self, fut: F) -> Task - where - F: Future + 'a, - F::Output: 'a, - { - // SAFETY: Original implementation missing safety documentation - unsafe { self.executor.spawn_unchecked(fut) } - } - - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// assert!(!local_ex.try_tick()); // no tasks to run - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(local_ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.executor.try_tick() - } - - /// Runs a single task asynchronously. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::{LocalExecutor, block_on}; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// block_on(local_ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.executor.tick().await; - } - - /// Runs the executor asynchronously until the given future completes. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::{LocalExecutor, block_on}; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { 1 + 2 }); - /// let res = block_on(local_ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` - pub async fn run(&self, fut: F) -> F::Output - where - F: Future, - { - // SAFETY: Original implementation missing safety documentation - unsafe { self.executor.run_unchecked(fut) }.await - } -} - -impl<'a, const C: usize> Default for LocalExecutor<'a, C> { - fn default() -> Self { - Self::new() - } -} - struct State { #[cfg(all( target_has_atomic = "8", @@ -486,46 +351,6 @@ impl State { } } -#[cfg(test)] -mod different_executor_tests { - use core::cell::Cell; - - use futures_lite::future::{block_on, pending, poll_once}; - use futures_lite::pin; - - use super::LocalExecutor; - - #[test] - fn shared_queue_slot() { - block_on(async { - let was_polled = Cell::new(false); - let future = async { - was_polled.set(true); - pending::<()>().await; - }; - - let ex1: LocalExecutor = Default::default(); - let ex2: LocalExecutor = Default::default(); - - // Start the futures for running forever. - let (run1, run2) = (ex1.run(pending::<()>()), ex2.run(pending::<()>())); - pin!(run1); - pin!(run2); - assert!(poll_once(run1.as_mut()).await.is_none()); - assert!(poll_once(run2.as_mut()).await.is_none()); - - // Spawn the future on executor one and then poll executor two. - ex1.spawn(future).detach(); - assert!(poll_once(run2).await.is_none()); - assert!(!was_polled.get()); - - // Poll the first one. - assert!(poll_once(run1).await.is_none()); - assert!(was_polled.get()); - }); - } -} - #[cfg(test)] mod drop_tests { use alloc::string::String; From 5b9a40eb011f4ec8f85d1bf49ac13a63f89fce7e Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:06:05 -0700 Subject: [PATCH 26/68] Remove unused import --- crates/bevy_tasks/src/edge_executor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 800ae57a70798..b2cd152e79c84 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -13,7 +13,6 @@ // TODO: Create a more tailored replacement, possibly integrating [Fotre](https://github.com/NthTensor/Forte) -use alloc::rc::Rc; use core::{ future::{poll_fn, Future}, marker::PhantomData, From 355c96f9fdd32d8a8b843cdfddcf995345e33a69 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:19:10 -0700 Subject: [PATCH 27/68] Use pin_project_lite for web builds instead of pin_project --- crates/bevy_tasks/Cargo.toml | 2 +- crates/bevy_tasks/src/async_executor.rs | 3 +-- crates/bevy_tasks/src/wasm_task.rs | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index b4830a45674e7..77c385ae5e279 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -38,7 +38,7 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel"] +web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel", "dep:pin-project-lite"] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 932962d4563f6..a90a193f323cf 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -25,7 +25,6 @@ use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future, prelude::*}; -use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; @@ -1023,7 +1022,7 @@ impl Drop for CallOnDrop { } } -pin_project! { +pin_project_lite::pin_project! { /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index 0cc569c47913d..f2f804a1d80b1 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -74,8 +74,9 @@ impl Future for Task { type Panic = Box; -#[pin_project::pin_project] -struct CatchUnwind(#[pin] F); +pin_project_lite::pin_project! { + struct CatchUnwind(#[pin] F); +} impl Future for CatchUnwind { type Output = Result; From 564daf6d4822dc3751a80c7fe6f0da55e2b734f3 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:21:31 -0700 Subject: [PATCH 28/68] Fix TOML formatting --- crates/bevy_tasks/Cargo.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 77c385ae5e279..f83dcaf041db5 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -38,7 +38,12 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel", "dep:pin-project-lite"] +web = [ + "bevy_platform/web", + "dep:wasm-bindgen-futures", + "dep:futures-channel", + "dep:pin-project-lite", +] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ From b100634a65f66371d8850719715eb3bfa164f7c9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:41:51 -0700 Subject: [PATCH 29/68] Make CatchUnwind pin_project_lite friendly --- crates/bevy_tasks/src/wasm_task.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index f2f804a1d80b1..a85d0dd8b0f8d 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -21,7 +21,7 @@ impl Task { wasm_bindgen_futures::spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind(AssertUnwindSafe(future)).await; + let value = CatchUnwind { inner: future }.await; let _ = sender.send(value); }); Self(receiver.into_future()) @@ -75,13 +75,16 @@ impl Future for Task { type Panic = Box; pin_project_lite::pin_project! { - struct CatchUnwind(#[pin] F); + struct CatchUnwind { + #[pin] + inner: F + } } impl Future for CatchUnwind { type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let f = AssertUnwindSafe(|| self.project().0.poll(cx)); + let f = AssertUnwindSafe(|| self.project().inner.poll(cx)); #[cfg(feature = "std")] let result = std::panic::catch_unwind(f)?; From 8700f7eb12474bff6ae3b379336fac00c8909ebc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 22:02:54 -0700 Subject: [PATCH 30/68] Add missing AssertUnwindSafe --- crates/bevy_tasks/src/wasm_task.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index a85d0dd8b0f8d..feee8184542d1 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -21,7 +21,10 @@ impl Task { wasm_bindgen_futures::spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind { inner: future }.await; + let value = CatchUnwind { + inner: AssertUnwindSafe(future), + } + .await; let _ = sender.send(value); }); Self(receiver.into_future()) From 6246ec841e16f12b1d374e2f14a1a91e590d2ace Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 4 Aug 2025 00:49:47 -0700 Subject: [PATCH 31/68] Another attempt at fixing CI --- crates/bevy_tasks/Cargo.toml | 6 +----- crates/bevy_tasks/src/single_threaded_task_pool.rs | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 1ff2e0e2cb4cd..308efd1be21e6 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -13,11 +13,7 @@ default = ["futures-lite"] # Enables multi-threading support. # Without this feature, all tasks will be run on a single thread. -multi_threaded = [ - "bevy_platform/std", - "dep:async-channel", - "bevy_executor", -] +multi_threaded = ["bevy_platform/std", "dep:async-channel", "bevy_executor"] # Uses a Bevy-specific fork of `async-executor` as a task execution backend. # This backend is incompatible with `no_std` targets. diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5b867c1c72088..664bce4607a31 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -5,7 +5,11 @@ use crate::Task; use crate::executor::Executor; -static EXECUTOR: Executor = Executor::new(); +crate::cfg::web! { + if {} else { + static EXECUTOR: Executor = Executor::new(); + } +} crate::cfg::std! { if { From 93c696b45d680c18f0bcf3334404576b17a77da1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 5 Aug 2025 02:16:06 -0700 Subject: [PATCH 32/68] Properly handle thread destruction and recycling --- crates/bevy_tasks/src/bevy_executor.rs | 64 +++++++++++++++++--------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index a90a193f323cf..a230d19240929 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -16,7 +16,7 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; -use std::thread::ThreadId; +use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; use alloc::fmt; @@ -24,7 +24,7 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; -use futures_lite::{future, prelude::*}; +use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -33,8 +33,11 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); pub(crate) fn install_runtime_into_current_thread() { - let tls = THREAD_LOCAL_STATE.get_or_default(); - tls.executor_thread.store(true, Ordering::Relaxed); + // Use LOCAL_QUEUE here to set the thread destructor + LOCAL_QUEUE.with(|_| { + let tls = THREAD_LOCAL_STATE.get_or_default(); + tls.executor_thread.store(true, Ordering::Relaxed); + }) } // Do not access this directly, use `with_local_queue` instead. @@ -67,8 +70,8 @@ cfg_if::cfg_if! { /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] -unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { - LOCAL_QUEUE.with(|tls| { +unsafe fn try_with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> Result { + LOCAL_QUEUE.try_with(|tls| { cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { f(&mut tls.borrow_mut()) @@ -81,14 +84,29 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { } }) } + struct LocalQueue { local_queue: VecDeque, local_active: Slab, } +impl Drop for LocalQueue { + fn drop(&mut self) { + // Unset the executor thread flag if it's been set. + if let Some(tls) = THREAD_LOCAL_STATE.get() { + tls.executor_thread.store(false, Ordering::Relaxed); + } + + for waker in self.local_active.drain() { + waker.wake(); + } + + while self.local_queue.pop_front().is_some() {} + } +} + struct ThreadLocalState { executor_thread: AtomicBool, - thread_id: ThreadId, stealable_queue: ArrayQueue, thread_locked_queue: SegQueue, } @@ -97,7 +115,6 @@ impl Default for ThreadLocalState { fn default() -> Self { Self { executor_thread: AtomicBool::new(false), - thread_id: std::thread::current().id(), stealable_queue: ArrayQueue::new(512), thread_locked_queue: SegQueue::new(), } @@ -176,10 +193,9 @@ impl<'a> ThreadSpawner<'a> { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - unsafe { - with_local_queue(|tls| tls.local_queue.push_back(runnable)); + if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { + state.notify_specific_thread(thread_id, false); } - state.notify_specific_thread(thread_id, false); } } } @@ -265,7 +281,7 @@ impl<'a> Executor<'a> { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. let (runnable, task) = unsafe { - with_local_queue(|tls| { + try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); // SAFETY: There are no instances where the value is accessed mutably @@ -273,7 +289,7 @@ impl<'a> Executor<'a> { // invoked after the surrounding scope has exited in either a // `try_tick_local` or `run` call. let future = AsyncCallOnDrop::new(future, move || { - with_local_queue(|tls| drop(tls.local_active.try_remove(index))); + try_with_local_queue(|tls| drop(tls.local_active.try_remove(index))).ok(); }); // Create the task and register it in the set of active tasks. @@ -295,7 +311,7 @@ impl<'a> Executor<'a> { entry.insert(runnable.waker()); (runnable, task) - }) + }).unwrap() }; runnable.schedule(); @@ -316,7 +332,9 @@ impl<'a> Executor<'a> { // from multiple locations simultaneously. As the Runnable is run after // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } + unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) } + .ok() + .flatten() .map(Runnable::run) .is_some() } @@ -337,7 +355,7 @@ impl<'a> Executor<'a> { if local_state.executor_thread.load(Ordering::Relaxed) { match local_state.stealable_queue.push(runnable) { Ok(()) => { - state.notify_specific_thread(local_state.thread_id, true); + state.notify_specific_thread(std::thread::current().id(), true); return; } Err(r) => r, @@ -357,15 +375,13 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { let state = self.state_as_arc(); - let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - unsafe { - with_local_queue(|tls| tls.local_queue.push_back(runnable)); + if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { + state.notify_specific_thread(std::thread::current().id(), false); } - state.notify_specific_thread(local_state.thread_id, false); } } @@ -777,8 +793,8 @@ impl Runner<'_> { .runnable_with(|| { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. - let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) }; - if let Some(r) = local_pop { + let local_pop = unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) }; + if let Ok(Some(r)) = local_pop { return Some(r); } @@ -925,7 +941,9 @@ unsafe fn flush_to_local(src: &SegQueue) { // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any // overlapping accesses. unsafe { - with_local_queue(|tls| { + // It's OK to ignore this error, no point in pushing to a + // queue for a thread that is already terminating. + let _ = try_with_local_queue(|tls| { // Steal tasks. for _ in 0..count { let Some(val) = src.queue_pop() else { break }; From 686d4294e8c016f897d8b1042f80418f95d2c5a4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 6 Aug 2025 00:48:11 -0700 Subject: [PATCH 33/68] Switch to a solution that doesn't require TLS access in thread destructors --- crates/bevy_tasks/src/bevy_executor.rs | 52 ++++++++----------- .../src/single_threaded_task_pool.rs | 12 ++--- crates/bevy_tasks/src/task_pool.rs | 2 +- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index a230d19240929..8833e5354364e 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -32,41 +32,40 @@ use thread_local::ThreadLocal; // See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); -pub(crate) fn install_runtime_into_current_thread() { +pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { // Use LOCAL_QUEUE here to set the thread destructor LOCAL_QUEUE.with(|_| { let tls = THREAD_LOCAL_STATE.get_or_default(); - tls.executor_thread.store(true, Ordering::Relaxed); - }) + let state = executor.state_as_arc(); + let state_ptr = Arc::into_raw(state).cast_mut(); + let old_ptr = tls.executor.swap(state_ptr, Ordering::Relaxed); + if !old_ptr.is_null() { + // SAFETY: If this pointer was not null, it was initialized from Arc::into_raw. + drop(unsafe { Arc::from_raw(old_ptr) }) + } + }); } // Do not access this directly, use `with_local_queue` instead. cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { use core::cell::RefCell; - - std::thread_local! { - static LOCAL_QUEUE: RefCell = const { - RefCell::new(LocalQueue { - local_queue: VecDeque::new(), - local_active:Slab::new(), - }) - }; - } + type LocalCell = RefCell; } else { use core::cell::UnsafeCell; - - std::thread_local! { - static LOCAL_QUEUE: UnsafeCell = const { - UnsafeCell::new(LocalQueue { - local_queue: VecDeque::new(), - local_active:Slab::new(), - }) - }; - } + type LocalCell = UnsafeCell; } } +std::thread_local! { + static LOCAL_QUEUE: LocalCell = const { + LocalCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; +} + /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] @@ -92,11 +91,6 @@ struct LocalQueue { impl Drop for LocalQueue { fn drop(&mut self) { - // Unset the executor thread flag if it's been set. - if let Some(tls) = THREAD_LOCAL_STATE.get() { - tls.executor_thread.store(false, Ordering::Relaxed); - } - for waker in self.local_active.drain() { waker.wake(); } @@ -106,7 +100,7 @@ impl Drop for LocalQueue { } struct ThreadLocalState { - executor_thread: AtomicBool, + executor: AtomicPtr, stealable_queue: ArrayQueue, thread_locked_queue: SegQueue, } @@ -114,7 +108,7 @@ struct ThreadLocalState { impl Default for ThreadLocalState { fn default() -> Self { Self { - executor_thread: AtomicBool::new(false), + executor: AtomicPtr::new(core::ptr::null_mut()), stealable_queue: ArrayQueue::new(512), thread_locked_queue: SegQueue::new(), } @@ -352,7 +346,7 @@ impl<'a> Executor<'a> { // Attempt to push onto the local queue first in dedicated executor threads, // because we know that this thread is awake and always processing new tasks. let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - if local_state.executor_thread.load(Ordering::Relaxed) { + if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), Arc::as_ptr(&state)) { match local_state.stealable_queue.push(runnable) { Ok(()) => { state.notify_specific_thread(std::thread::current().id(), true); diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 664bce4607a31..c5327ffa0bf67 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -30,15 +30,9 @@ pub struct TaskPoolBuilder {} /// task pool. In the case of the multithreaded task pool this struct is used to spawn /// tasks on a specific thread. But the wasm task pool just calls /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread -/// and so the [`ThreadExecutor`] does nothing. -#[derive(Default, Clone)] +/// and so the [`ThreadSpawner`] does nothing. +#[derive(Clone)] pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); -impl<'a> ThreadSpawner<'a> { - /// Creates a new `ThreadExecutor` - pub fn new() -> Self { - Self::default() - } -} impl TaskPoolBuilder { /// Creates a new `TaskPoolBuilder` instance @@ -85,7 +79,7 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { - ThreadSpawner::new() + ThreadSpawner(PhantomData) } /// Create a `TaskPool` with the default configuration. diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index e0a31f3fdedf7..cace1736b4902 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -178,7 +178,7 @@ impl TaskPool { thread_builder .spawn(move || { - crate::bevy_executor::install_runtime_into_current_thread(); + crate::bevy_executor::install_runtime_into_current_thread(&ex); if let Some(on_thread_spawn) = on_thread_spawn { on_thread_spawn(); From aa64f0336f2c4d80d68700908bf85967687601ea Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 6 Aug 2025 02:21:24 -0700 Subject: [PATCH 34/68] Try to provide a blocking solution to TaskPool::scope in single threaded spaces --- crates/bevy_tasks/src/bevy_executor.rs | 55 +++++++++++-------- .../src/single_threaded_task_pool.rs | 12 +++- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 8833e5354364e..85c8ad01f0a67 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -16,6 +16,7 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; +use core::cell::UnsafeCell; use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; @@ -24,6 +25,7 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; +use futures_lite::future::block_on; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -41,25 +43,14 @@ pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { let old_ptr = tls.executor.swap(state_ptr, Ordering::Relaxed); if !old_ptr.is_null() { // SAFETY: If this pointer was not null, it was initialized from Arc::into_raw. - drop(unsafe { Arc::from_raw(old_ptr) }) + drop(unsafe { Arc::from_raw(old_ptr) }); } }); } -// Do not access this directly, use `with_local_queue` instead. -cfg_if::cfg_if! { - if #[cfg(all(debug_assertions, not(miri)))] { - use core::cell::RefCell; - type LocalCell = RefCell; - } else { - use core::cell::UnsafeCell; - type LocalCell = UnsafeCell; - } -} - std::thread_local! { - static LOCAL_QUEUE: LocalCell = const { - LocalCell::new(LocalQueue { + static LOCAL_QUEUE: UnsafeCell = const { + UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), local_active:Slab::new(), }) @@ -71,16 +62,10 @@ std::thread_local! { #[inline(always)] unsafe fn try_with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> Result { LOCAL_QUEUE.try_with(|tls| { - cfg_if::cfg_if! { - if #[cfg(all(debug_assertions, not(miri)))] { - f(&mut tls.borrow_mut()) - } else { - // SAFETY: This value is in thread local storage and thus can only be accessed - // from one thread. The caller guarantees that this function is not used with - // LOCAL_QUEUE in any way. - f(unsafe { &mut *tls.get() }) - } - } + // SAFETY: This value is in thread local storage and thus can only be accessed + // from one thread. The caller guarantees that this function is not used with + // LOCAL_QUEUE in any way. + f(unsafe { &mut *tls.get() }) }) } @@ -333,6 +318,28 @@ impl<'a> Executor<'a> { .is_some() } + pub fn flush_local() { + // If this is called during the thread destructor, there's nothing to run anyway. + // All of the tasks will be dropped soon. + // + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + let _ = LOCAL_QUEUE.try_with(|local| unsafe { + block_on(async { + while !(&*local.get()).local_active.is_empty() { + match (&mut *local.get()).local_queue.pop_front() { + Some(runnable) => { + runnable.run(); + }, + None => future::yield_now().await, + } + } + }) + }); + } + /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index c5327ffa0bf67..195154597069c 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -153,7 +153,7 @@ impl TaskPool { f(scope_ref); // Loop until all tasks are done - while Self::try_tick_local() {} + Self::flush_local(); let results = scope.results.borrow(); results @@ -217,6 +217,16 @@ impl TaskPool { } } } + + fn flush_local() { + crate::cfg::bevy_executor! { + if { + crate::bevy_executor::Executor::flush_local(); + } else { + while EXECUTOR.try_tick() {} + } + } + } } /// A `TaskPool` scope for running one or more non-`'static` futures. From 79c31b7e6d9fcf480007f7b55362be706f7f87b1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 7 Aug 2025 23:46:37 -0700 Subject: [PATCH 35/68] CI fixes --- crates/bevy_tasks/src/bevy_executor.rs | 4 ++-- .../bevy_tasks/src/single_threaded_task_pool.rs | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 85c8ad01f0a67..d0815c3198598 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -336,7 +336,7 @@ impl<'a> Executor<'a> { None => future::yield_now().await, } } - }) + }); }); } @@ -915,7 +915,7 @@ impl WorkQueue for SegQueue { } /// Steals some items from one queue into another. -fn steal>(src: &Q, dest: &ArrayQueue) { +fn steal_and_pop>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. let mut count = src.stealable_count(); diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 195154597069c..62e21ef970ddf 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -208,12 +208,16 @@ impl TaskPool { self.spawn(future) } - pub(crate) fn try_tick_local() -> bool { - crate::cfg::bevy_executor! { - if { - crate::bevy_executor::Executor::try_tick_local() - } else { - EXECUTOR.try_tick() + crate::cfg::web! { + if {} else { + pub(crate) fn try_tick_local() -> bool { + crate::cfg::bevy_executor! { + if { + crate::bevy_executor::Executor::try_tick_local() + } else { + EXECUTOR.try_tick() + } + } } } } From 4dcb37c941323fdebf432ef69497bf460a3e0f9d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:21:54 -0700 Subject: [PATCH 36/68] Fix up the build, and hopefully CI --- crates/bevy_tasks/src/bevy_executor.rs | 24 +-------- crates/bevy_tasks/src/edge_executor.rs | 4 +- crates/bevy_tasks/src/executor.rs | 52 ------------------- crates/bevy_tasks/src/lib.rs | 1 - .../src/single_threaded_task_pool.rs | 38 +++++--------- crates/bevy_tasks/src/task_pool.rs | 19 +++---- 6 files changed, 28 insertions(+), 110 deletions(-) delete mode 100644 crates/bevy_tasks/src/executor.rs diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index d0815c3198598..eeea29ef9c006 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -318,28 +318,6 @@ impl<'a> Executor<'a> { .is_some() } - pub fn flush_local() { - // If this is called during the thread destructor, there's nothing to run anyway. - // All of the tasks will be dropped soon. - // - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. As the Runnable is run after - // this scope closes, the AsyncCallOnDrop around the future will be invoked - // without overlapping mutable accssses. - let _ = LOCAL_QUEUE.try_with(|local| unsafe { - block_on(async { - while !(&*local.get()).local_active.is_empty() { - match (&mut *local.get()).local_queue.pop_front() { - Some(runnable) => { - runnable.run(); - }, - None => future::yield_now().await, - } - } - }); - }); - } - /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await @@ -915,7 +893,7 @@ impl WorkQueue for SegQueue { } /// Steals some items from one queue into another. -fn steal_and_pop>(src: &Q, dest: &ArrayQueue) { +fn steal>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. let mut count = src.stealable_count(); diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index e063a47533774..b8f72ba31f40c 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -48,6 +48,7 @@ use futures_lite::FutureExt; /// drop(signal); /// })); /// ``` +#[derive(Debug)] pub struct Executor<'a, const C: usize = 64> { state: LazyLock>>, _invariant: PhantomData>, @@ -170,7 +171,7 @@ impl<'a, const C: usize> Executor<'a, C> { /// ``` pub async fn run(&self, fut: F) -> F::Output where - F: Future + Send + 'a, + F: Future + 'a, { // SAFETY: Original implementation missing safety documentation unsafe { self.run_unchecked(fut).await } @@ -306,6 +307,7 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} +#[derive(Debug)] struct State { #[cfg(all( target_has_atomic = "8", diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs deleted file mode 100644 index 7ea6e64e32229..0000000000000 --- a/crates/bevy_tasks/src/executor.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! Provides a fundamental executor primitive appropriate for the target platform -//! and feature set selected. -//! By default, the `async_executor` feature will be enabled, which will rely on -//! [`async-executor`] for the underlying implementation. This requires `std`, -//! so is not suitable for `no_std` contexts. Instead, you must use `edge_executor`, -//! which relies on the alternate [`edge-executor`] backend. -//! -//! [`async-executor`]: https://crates.io/crates/async-executor -//! [`edge-executor`]: https://crates.io/crates/edge-executor - -use core::{ - fmt, - panic::{RefUnwindSafe, UnwindSafe}, -}; -use derive_more::{Deref, DerefMut}; - -crate::cfg::bevy_executor! { - if { - type ExecutorInner<'a> = crate::bevy_executor::Executor<'a>; - } else { - type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; - } -} - -crate::cfg::multi_threaded! { - pub use async_task::FallibleTask; -} - -/// Wrapper around a multi-threading-aware async executor. -/// spawning will generally require tasks to be `send` and `sync` to allow multiple -/// threads to send/receive/advance tasks. -#[derive(Deref, DerefMut)] -pub(crate) struct Executor<'a>(ExecutorInner<'a>); - -impl Executor<'_> { - /// Construct a new [`Executor`] - #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] - #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { - Self(ExecutorInner::new()) - } -} - -impl UnwindSafe for Executor<'_> {} - -impl RefUnwindSafe for Executor<'_> {} - -impl fmt::Debug for Executor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Executor").finish() - } -} diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 2854655354ab3..d03665c94bc86 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -72,7 +72,6 @@ use alloc::boxed::Box; pub type BoxedFuture<'a, T> = core::pin::Pin + 'a>>; // Modules -mod executor; pub mod futures; mod iter; mod slice; diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 59921e8281393..434895a7bf3d0 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,28 +1,18 @@ use alloc::{string::String, vec::Vec}; -use bevy_platform::sync::Arc; use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; -use crate::executor::LocalExecutor; use crate::{block_on, Task}; -crate::cfg::std! { - if { - use std::thread_local; - - use crate::executor::LocalExecutor as Executor; - - thread_local! { - static LOCAL_EXECUTOR: Executor<'static> = const { Executor::new() }; - } +crate::cfg::bevy_executor! { + if { + use crate::bevy_executor::Executor; } else { - - // Because we do not have thread-locals without std, we cannot use LocalExecutor here. - use crate::executor::Executor; - - static LOCAL_EXECUTOR: Executor<'static> = const { Executor::new() }; + use crate::edge_executor::Executor; } } +static EXECUTOR: Executor<'static> = const { Executor::new() }; + /// Used to create a [`TaskPool`]. #[derive(Debug, Default, Clone)] pub struct TaskPoolBuilder {} @@ -133,9 +123,8 @@ impl TaskPool { // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor = LocalExecutor::new(); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor_ref: &'env LocalExecutor<'env> = unsafe { mem::transmute(&executor) }; + let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; let results: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -159,7 +148,8 @@ impl TaskPool { f(scope_ref); // Wait until the scope is complete - block_on(executor.run(async { + block_on(executor_ref.run(async { + std::println!("Pending: {}", pending_tasks.get()); while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } @@ -215,7 +205,7 @@ impl TaskPool { pub(crate) fn try_tick_local() -> bool { crate::cfg::bevy_executor! { if { - crate::bevy_executor::Executor::try_tick_local() + Executor::try_tick_local() } else { EXECUTOR.try_tick() } @@ -227,7 +217,7 @@ impl TaskPool { fn flush_local() { crate::cfg::bevy_executor! { if { - crate::bevy_executor::Executor::flush_local(); + Executor::flush_local(); } else { while EXECUTOR.try_tick() {} } @@ -240,7 +230,7 @@ impl TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor_ref: &'scope LocalExecutor<'scope>, + executor_ref: &'scope Executor<'scope>, // The number of pending tasks spawned on the scope pending_tasks: &'scope Cell, // Vector to gather results of all futures spawned during scope run @@ -310,10 +300,10 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. unsafe { - self.executor.spawn_local_scoped(f).detach() + self.executor_ref.spawn_local_scoped(f).detach() }; } else { - self.executor.spawn_local(f).detach(); + self.executor_ref.spawn_local(f).detach(); } } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index cace1736b4902..086e05d4e0614 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,7 +2,8 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; use std::thread::{self, JoinHandle}; -use crate::{bevy_executor::Executor, executor::FallibleTask}; +use crate::bevy_executor::Executor; +use async_task::FallibleTask; use bevy_platform::sync::Arc; use crossbeam_queue::SegQueue; use futures_lite::FutureExt; @@ -129,7 +130,7 @@ impl TaskPoolBuilder { #[derive(Debug)] pub struct TaskPool { /// The executor for the pool. - executor: Arc>, + executor: Arc>, // The inner state of the pool. threads: Vec>, @@ -151,7 +152,7 @@ impl TaskPool { fn new_internal(builder: TaskPoolBuilder) -> Self { let (shutdown_tx, shutdown_rx) = async_channel::unbounded::<()>(); - let executor = Arc::new(crate::executor::Executor::new()); + let executor = Arc::new(Executor::new()); let num_threads = builder .num_threads @@ -343,14 +344,14 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor: &crate::executor::Executor = &self.executor; + let executor: &Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - let spawned: SegQueue>>> = + let executor: &'env Executor = unsafe { mem::transmute(executor) }; + let spawned: SegQueue>>> = SegQueue::new(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env SegQueue>>> = + let spawned: &'env SegQueue>>> = unsafe { mem::transmute(&spawned) }; let scope = Scope { @@ -451,10 +452,10 @@ impl Drop for TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor: &'scope crate::executor::Executor<'scope>, + executor: &'scope Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope SegQueue>>>, + spawned: &'scope SegQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, From 0923f0268a23dd7966f17f0e2d9b8ccd579b1cfc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:33:17 -0700 Subject: [PATCH 37/68] Shut up Clippy --- crates/bevy_tasks/Cargo.toml | 1 - crates/bevy_tasks/src/bevy_executor.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index f3a155e325bc1..50c88da9ba3e1 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -45,7 +45,6 @@ derive_more = { version = "2", default-features = false, features = [ "deref", "deref_mut", ] } -cfg-if = "1.0.0" slab = { version = "0.4", optional = true } pin-project-lite = { version = "0.2", optional = true } thread_local = { version = "1.1", optional = true } diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index eeea29ef9c006..2fb68aad68d12 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -2,10 +2,6 @@ unsafe_code, reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] -#![expect( - clippy::unused_unit, - reason = "False positive detection on {Async}CallOnDrop" -)] #![allow( dead_code, reason = "Not all functions are used with every feature combination" @@ -25,7 +21,6 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; -use futures_lite::future::block_on; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; From 43a09d7688eb41cda4951ee4aca971b2802346ca Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:46:34 -0700 Subject: [PATCH 38/68] Remove test println --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 434895a7bf3d0..5207e8452439d 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -149,7 +149,6 @@ impl TaskPool { // Wait until the scope is complete block_on(executor_ref.run(async { - std::println!("Pending: {}", pending_tasks.get()); while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } From 8f313f77838f73d073d3f3e549420d04d1a0f00b Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 02:05:50 -0700 Subject: [PATCH 39/68] Fix for portable atomics --- crates/bevy_tasks/src/edge_executor.rs | 2 -- crates/bevy_tasks/src/single_threaded_task_pool.rs | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index b8f72ba31f40c..fc0cb5d770d5d 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -48,7 +48,6 @@ use futures_lite::FutureExt; /// drop(signal); /// })); /// ``` -#[derive(Debug)] pub struct Executor<'a, const C: usize = 64> { state: LazyLock>>, _invariant: PhantomData>, @@ -307,7 +306,6 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} -#[derive(Debug)] struct State { #[cfg(all( target_has_atomic = "8", diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5207e8452439d..139f180bd832f 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,4 +1,4 @@ -use alloc::{string::String, vec::Vec}; +use alloc::{string::String, vec::Vec, fmt}; use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; use crate::{block_on, Task}; @@ -227,7 +227,6 @@ impl TaskPool { /// A `TaskPool` scope for running one or more non-`'static` futures. /// /// For more information, see [`TaskPool::scope`]. -#[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor_ref: &'scope Executor<'scope>, // The number of pending tasks spawned on the scope @@ -308,6 +307,17 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { } } +impl <'scope, 'env: 'scope, T> fmt::Debug for Scope<'scope, 'env, T> +where T: fmt::Debug +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("pending_tasks", &self.pending_tasks) + .field("results_ref", &self.results_ref) + .finish() + } +} + crate::cfg::std! { if { pub trait MaybeSend {} From 4a8b6b0c7d7a550144885e14738df58be9de8ab6 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 10:42:52 -0700 Subject: [PATCH 40/68] Fix for web builds --- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/single_threaded_task_pool.rs | 10 ---------- crates/bevy_tasks/src/task.rs | 14 ++++++++++---- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 50c88da9ba3e1..ce5e38c184c7d 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -58,6 +58,7 @@ crossbeam-queue = { version = "0.3", default-features = false, features = [ [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" +pin-project-lite = "0.2" [target.'cfg(not(all(target_has_atomic = "8", target_has_atomic = "16", target_has_atomic = "32", target_has_atomic = "64", target_has_atomic = "ptr")))'.dependencies] async-task = { version = "4.4.0", default-features = false, features = [ diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 139f180bd832f..e7a5d4dc46125 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -212,16 +212,6 @@ impl TaskPool { } } } - - fn flush_local() { - crate::cfg::bevy_executor! { - if { - Executor::flush_local(); - } else { - while EXECUTOR.try_tick() {} - } - } - } } /// A `TaskPool` scope for running one or more non-`'static` futures. diff --git a/crates/bevy_tasks/src/task.rs b/crates/bevy_tasks/src/task.rs index dd649ba47dca3..4e9826a82f709 100644 --- a/crates/bevy_tasks/src/task.rs +++ b/crates/bevy_tasks/src/task.rs @@ -38,7 +38,9 @@ cfg::web! { spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind(AssertUnwindSafe(future)).await; + let value = CatchUnwind { + inner: AssertUnwindSafe(future) + }.await; let _ = sender.send(value); }); Self(receiver) @@ -173,13 +175,17 @@ cfg::web! { type Panic = Box; - #[pin_project::pin_project] - struct CatchUnwind(#[pin] F); + pin_project_lite::pin_project! { + struct CatchUnwind { + #[pin] + inner: F + } + } impl Future for CatchUnwind { type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let f = AssertUnwindSafe(|| self.project().0.poll(cx)); + let f = AssertUnwindSafe(|| self.project().inner.poll(cx)); let result = cfg::std! { if { From 7e5cf6caa7297f2bec6faa5812701b332feac9dc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:05:55 -0700 Subject: [PATCH 41/68] Reduce async type genreation, code indirection, and handle single-threaded cancellation --- crates/bevy_tasks/src/bevy_executor.rs | 133 ++++++++---------- .../src/single_threaded_task_pool.rs | 42 +++--- 2 files changed, 79 insertions(+), 96 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 2fb68aad68d12..317e333f9f16c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -314,8 +314,23 @@ impl<'a> Executor<'a> { } /// Runs the executor until the given future completes. - pub async fn run(&self, future: impl Future) -> T { - self.state().run(future).await + pub fn run<'b, T>(&'b self, future: impl Future + 'b) -> impl Future + 'b { + let mut runner = Runner::new(self.state()); + + // A future that runs tasks forever. + let run_forever = async move { + let mut rng = fastrand::Rng::new(); + loop { + for _ in 0..200 { + let runnable = runner.runnable(&mut rng).await; + runnable.run(); + } + future::yield_now().await; + } + }; + + // Run `future` and `run_forever` concurrently until `future` completes. + future.or(run_forever) } /// Returns a function that schedules a runnable task when it gets woken up. @@ -500,25 +515,6 @@ impl State { w.wake(); } } - - pub async fn run(&self, future: impl Future) -> T { - let mut runner = Runner::new(self); - let mut rng = fastrand::Rng::new(); - - // A future that runs tasks forever. - let run_forever = async { - loop { - for _ in 0..200 { - let runnable = runner.runnable(&mut rng).await; - runnable.run(); - } - future::yield_now().await; - } - }; - - // Run `future` and `run_forever` concurrently until `future` completes. - future.or(run_forever).await - } } impl fmt::Debug for State { @@ -677,31 +673,39 @@ impl Ticker<'_> { } /// Waits for the next runnable task to run, given a function that searches for a task. - async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { - future::poll_fn(|cx| { - loop { - match search() { - None => { - // Move to sleeping and unnotified state. - if !self.sleep(cx.waker()) { - // If already sleeping and unnotified, return. - return Poll::Pending; + /// + /// # Safety + /// Caller must not access LOCAL_QUEUE either directly or with try_with_local_queue` in any way inside `search`. + unsafe fn runnable_with(&mut self, mut search: impl FnMut(&mut LocalQueue) -> Option) -> impl Future { + future::poll_fn(move |cx| { + // SAFETY: Caller must ensure that there's no instances where LOCAL_QUEUE is accessed mutably + // from multiple locations simultaneously. + unsafe { + try_with_local_queue(|tls| { + loop { + match search(tls) { + None => { + // Move to sleeping and unnotified state. + if !self.sleep(cx.waker()) { + // If already sleeping and unnotified, return. + return Poll::Pending; + } + } + Some(r) => { + // Wake up. + self.wake(); + + // Notify another ticker now to pick up where this ticker left off, just in + // case running the task takes a long time. + self.state.notify(); + + return Poll::Ready(r); + } } } - Some(r) => { - // Wake up. - self.wake(); - - // Notify another ticker now to pick up where this ticker left off, just in - // case running the task takes a long time. - self.state.notify(); - - return Poll::Ready(r); - } - } + }).unwrap() } }) - .await } } @@ -739,7 +743,7 @@ struct Runner<'a> { ticks: usize, // The thread local state of the executor for the current thread. - local_state: &'static ThreadLocalState, + local_state: &'a ThreadLocalState, } impl Runner<'_> { @@ -761,14 +765,14 @@ impl Runner<'_> { } /// Waits for the next runnable task to run. - async fn runnable(&mut self, rng: &mut fastrand::Rng) -> Runnable { - let runnable = self + fn runnable(&mut self, rng: &mut fastrand::Rng) -> impl Future { + // SAFETY: The provided search function does not access LOCAL_QUEUE in any way, and thus cannot + // alias. + let runnable = unsafe { + self .ticker - .runnable_with(|| { - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. - let local_pop = unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) }; - if let Ok(Some(r)) = local_pop { + .runnable_with(|tls| { + if let Some(r) = tls.local_queue.pop_back() { return Some(r); } @@ -813,16 +817,13 @@ impl Runner<'_> { // // Instead, flush all queued tasks into the local queue to // minimize the effort required to scan for these tasks. - // - // SAFETY: This is not being used at the same time as any - // access to LOCAL_QUEUE. - unsafe { flush_to_local(&self.local_state.thread_locked_queue) }; + flush_to_local(&self.local_state.thread_locked_queue, tls); return Some(r); } None }) - .await; + }; // Bump the tick counter. self.ticks = self.ticks.wrapping_add(1); @@ -904,26 +905,14 @@ fn steal>(src: &Q, dest: &ArrayQueue) { } } -/// Flushes all of the items from a queue into the thread local queue. -/// -/// # Safety -/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. -unsafe fn flush_to_local(src: &SegQueue) { +fn flush_to_local(src: &SegQueue, dst: &mut LocalQueue) { let count = src.len(); if count > 0 { - // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any - // overlapping accesses. - unsafe { - // It's OK to ignore this error, no point in pushing to a - // queue for a thread that is already terminating. - let _ = try_with_local_queue(|tls| { - // Steal tasks. - for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; - tls.local_queue.push_front(val); - } - }); + // Steal tasks. + for _ in 0..count { + let Some(val) = src.queue_pop() else { break }; + dst.local_queue.push_front(val); } } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index e7a5d4dc46125..5c892f97b5629 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,5 +1,5 @@ use alloc::{string::String, vec::Vec, fmt}; -use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; +use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem, task::{Poll, Context, Waker}, pin::Pin}; use crate::{block_on, Task}; @@ -126,9 +126,10 @@ impl TaskPool { // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; - let results: RefCell>> = RefCell::new(Vec::new()); + // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks. + let tasks: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let results_ref: &'env RefCell>> = unsafe { mem::transmute(&results) }; + let tasks_ref: &'env RefCell>> = unsafe { mem::transmute(&tasks) }; let pending_tasks: Cell = Cell::new(0); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -136,8 +137,8 @@ impl TaskPool { let mut scope = Scope { executor_ref, + tasks_ref, pending_tasks, - results_ref, scope: PhantomData, env: PhantomData, }; @@ -154,10 +155,14 @@ impl TaskPool { } })); - results + let mut context = Context::from_waker(Waker::noop()); + tasks .take() .into_iter() - .map(|result| result.unwrap()) + .map(|mut task| match Pin::new(&mut task).poll(&mut context) { + Poll::Ready(result) => result, + Poll::Pending => unreachable!(), + }) .collect() } @@ -222,7 +227,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { // The number of pending tasks spawned on the scope pending_tasks: &'scope Cell, // Vector to gather results of all futures spawned during scope run - results_ref: &'env RefCell>>, + tasks_ref: &'env RefCell>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -263,35 +268,24 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { let pending_tasks = self.pending_tasks; pending_tasks.update(|i| i + 1); - // add a spot to keep the result, and record the index - let results_ref = self.results_ref; - let mut results = results_ref.borrow_mut(); - let task_number = results.len(); - results.push(None); - drop(results); - // create the job closure let f = async move { let result = f.await; - // store the result in the allocated slot - let mut results = results_ref.borrow_mut(); - results[task_number] = Some(result); - drop(results); - // decrement the pending tasks count pending_tasks.update(|i| i - 1); + + result }; + let mut tasks = self.tasks_ref.borrow_mut(); crate::cfg::bevy_executor! { if { // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. - unsafe { - self.executor_ref.spawn_local_scoped(f).detach() - }; + tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); } else { - self.executor_ref.spawn_local(f).detach(); + tasks.push(self.executor_ref.spawn_local(f)); } } } @@ -303,7 +297,7 @@ where T: fmt::Debug fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Scope") .field("pending_tasks", &self.pending_tasks) - .field("results_ref", &self.results_ref) + .field("tasks_ref", &self.tasks_ref) .finish() } } From eb689b59831ac324840d1a49f112903c2a94b37a Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:14:50 -0700 Subject: [PATCH 42/68] Fix docs and move expect attribute --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 317e333f9f16c..453252eb7f35a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -675,7 +675,7 @@ impl Ticker<'_> { /// Waits for the next runnable task to run, given a function that searches for a task. /// /// # Safety - /// Caller must not access LOCAL_QUEUE either directly or with try_with_local_queue` in any way inside `search`. + /// Caller must not access `LOCAL_QUEUE` either directly or with `try_with_local_queue` in any way inside `search`. unsafe fn runnable_with(&mut self, mut search: impl FnMut(&mut LocalQueue) -> Option) -> impl Future { future::poll_fn(move |cx| { // SAFETY: Caller must ensure that there's no instances where LOCAL_QUEUE is accessed mutably diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5c892f97b5629..8931856ed8cdc 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -257,7 +257,6 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } - #[allow(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -281,6 +280,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { let mut tasks = self.tasks_ref.borrow_mut(); crate::cfg::bevy_executor! { if { + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); From 7ec07af71b82df333da0913014fb7e7d63e0e999 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:35:04 -0700 Subject: [PATCH 43/68] Complete the comment --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 8931856ed8cdc..62f9f75e1fa23 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -126,7 +126,7 @@ impl TaskPool { // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; - // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks. + // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks are cancelled. let tasks: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime let tasks_ref: &'env RefCell>> = unsafe { mem::transmute(&tasks) }; From 4a4434c2b731662b25f241a08c247d9dd517151f Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 23:37:42 -0700 Subject: [PATCH 44/68] Add async_executor's tests --- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/bevy_executor.rs | 203 ++++++++++++++++++++++++- 2 files changed, 203 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index ce5e38c184c7d..5b12d0c817372 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -76,6 +76,7 @@ futures-lite = { version = "2.0.1", default-features = false, features = [ "std", ] } async-channel = "2.3.0" +async-io = "2.0.0" [lints] workspace = true diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 453252eb7f35a..34ebfcebe5f0c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1031,8 +1031,16 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { - use super::Executor; + use super::*; use super::THREAD_LOCAL_STATE; + use alloc::{string::String, boxed::Box}; + use futures_lite::{future, pin}; + use std::panic::catch_unwind; + use core::sync::atomic::{AtomicUsize, Ordering}; + use bevy_platform::sync::Mutex; + use core::task::{Poll, Waker}; + use async_task::Task; + use std::time::Duration; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -1051,4 +1059,197 @@ mod test { is_send(THREAD_LOCAL_STATE.get_or_default()); is_sync(THREAD_LOCAL_STATE.get_or_default()); } + + #[test] + fn executor_cancels_everything() { + static DROP: AtomicUsize = AtomicUsize::new(0); + static WAKER: Mutex> = Mutex::new(None); + + let ex = Executor::new(); + + let task = ex.spawn(async { + let _guard = CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }); + + future::poll_fn(|cx| { + *WAKER.lock().unwrap() = Some(cx.waker().clone()); + Poll::Pending::<()> + }) + .await; + }); + + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert!(WAKER.lock().unwrap().is_some()); + assert_eq!(DROP.load(Ordering::SeqCst), 0); + + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + + assert!(catch_unwind(|| future::block_on(task)).is_err()); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + #[test] + fn await_task_after_dropping_executor() { + let s: String = "hello".into(); + + let ex = Executor::new(); + let task: Task<&str> = ex.spawn(async { &*s }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + + drop(ex); + assert_eq!(future::block_on(task), "hello"); + drop(s); + } + + #[test] + fn drop_executor_and_then_drop_finished_task() { + static DROP: AtomicUsize = AtomicUsize::new(0); + + let ex = Executor::new(); + let task = ex.spawn(async { + CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }) + }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(task); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + #[test] + fn drop_finished_task_and_then_drop_executor() { + static DROP: AtomicUsize = AtomicUsize::new(0); + + let ex = Executor::new(); + let task = ex.spawn(async { + CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }) + }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(task); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + fn do_run>(mut f: impl FnMut(Arc>) -> Fut) { + // This should not run for longer than two minutes. + #[cfg(not(miri))] + let _stop_timeout = { + let (stop_timeout, stopper) = async_channel::bounded::<()>(1); + std::thread::spawn(move || { + future::block_on(async move { + let timeout = async { + async_io::Timer::after(Duration::from_secs(2 * 60)).await; + std::eprintln!("test timed out after 2m"); + std::process::exit(1) + }; + + let _ = stopper.recv().or(timeout).await; + }) + }); + stop_timeout + }; + + let ex = Arc::new(Executor::new()); + + // Test 1: Use the `run` command. + future::block_on(ex.run(f(ex.clone()))); + + // Test 2: Run on many threads. + std::thread::scope(|scope| { + let (_signal, shutdown) = async_channel::bounded::<()>(1); + + for _ in 0..16 { + let shutdown = shutdown.clone(); + let ex = &ex; + scope.spawn(move || future::block_on(ex.run(shutdown.recv()))); + } + + future::block_on(f(ex.clone())); + }); + } + + #[test] + fn smoke() { + do_run(|ex| async move { ex.spawn(async {}).await }); + } + + #[test] + fn yield_now() { + do_run(|ex| async move { ex.spawn(future::yield_now()).await }) + } + + #[test] + fn timer() { + do_run(|ex| async move { + ex.spawn(async_io::Timer::after(Duration::from_millis(5))) + .await; + }) + } + + #[test] + fn test_panic_propagation() { + let ex = Executor::new(); + let task = ex.spawn(async { panic!("should be caught by the task") }); + + // Running the executor should not panic. + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + // Polling the task should. + assert!(future::block_on(task.catch_unwind()).is_err()); + } + + #[test] + fn two_queues() { + future::block_on(async { + // Create an executor with two runners. + let ex = Executor::new(); + let (run1, run2) = ( + ex.run(future::pending::<()>()), + ex.run(future::pending::<()>()), + ); + let mut run1 = Box::pin(run1); + pin!(run2); + + // Poll them both. + assert!(future::poll_once(run1.as_mut()).await.is_none()); + assert!(future::poll_once(run2.as_mut()).await.is_none()); + + // Drop the first one, which should leave the local queue in the `None` state. + drop(run1); + assert!(future::poll_once(run2.as_mut()).await.is_none()); + }); + } } From 2ce5537641ac5ec09705d4b147af52c11a3299bc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 23:40:50 -0700 Subject: [PATCH 45/68] Run Miri on bevy_tasks in CI --- .github/workflows/ci.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ad9a1ce76777..a16c1111ea73f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,13 @@ jobs: miri: runs-on: macos-latest timeout-minutes: 60 + env: + # -Zrandomize-layout makes sure we dont rely on the layout of anything that might change + RUSTFLAGS: -Zrandomize-layout + # https://github.com/rust-lang/miri#miri--z-flags-and-environment-variables + # -Zmiri-disable-isolation is needed because our executor uses `fastrand` which accesses system time. + # -Zmiri-ignore-leaks is necessary because a bunch of tests don't join all threads before finishing. + MIRIFLAGS: -Zmiri-ignore-leaks -Zmiri-disable-isolation steps: - uses: actions/checkout@v4 - uses: actions/cache/restore@v4 @@ -113,15 +120,10 @@ jobs: components: miri - name: CI job # To run the tests one item at a time for troubleshooting, use + # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_tasks --lib -- --exact # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact + run: cargo miri test -p bevy_tasks --features bevy_executor --features multi_threaded run: cargo miri test -p bevy_ecs --features bevy_utils/debug - env: - # -Zrandomize-layout makes sure we dont rely on the layout of anything that might change - RUSTFLAGS: -Zrandomize-layout - # https://github.com/rust-lang/miri#miri--z-flags-and-environment-variables - # -Zmiri-disable-isolation is needed because our executor uses `fastrand` which accesses system time. - # -Zmiri-ignore-leaks is necessary because a bunch of tests don't join all threads before finishing. - MIRIFLAGS: -Zmiri-ignore-leaks -Zmiri-disable-isolation check-compiles: runs-on: ubuntu-latest From 8af9886f9fa2d7917929add2dc45816fddc10928 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 11:06:18 -0700 Subject: [PATCH 46/68] Make stealing a non-blocking operation --- crates/bevy_tasks/src/bevy_executor.rs | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 34ebfcebe5f0c..fe07ae862190f 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -47,7 +47,7 @@ std::thread_local! { static LOCAL_QUEUE: UnsafeCell = const { UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), - local_active:Slab::new(), + local_active: Slab::new(), }) }; } @@ -788,26 +788,26 @@ impl Runner<'_> { } // Try stealing from other runners. - let stealer_queues = self.state.stealer_queues.read().unwrap(); - - // Pick a random starting point in the iterator list and rotate the list. - let n = stealer_queues.len(); - let start = rng.usize(..n); - let iter = stealer_queues - .iter() - .chain(stealer_queues.iter()) - .skip(start) - .take(n); - - // Remove this runner's local queue. - let iter = - iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); - - // Try stealing from each local queue in the list. - for local in iter { - steal(*local, &self.local_state.stealable_queue); - if let Some(r) = self.local_state.stealable_queue.pop() { - return Some(r); + if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { + // Pick a random starting point in the iterator list and rotate the list. + let n = stealer_queues.len(); + let start = rng.usize(..n); + let iter = stealer_queues + .iter() + .chain(stealer_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { + return Some(r); + } } } From 74d301c84d67ba85004b18a51fb3b1b3a746e2bf Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:20:43 -0700 Subject: [PATCH 47/68] Cache-pad the thread locals and disable multithreaded polling when the feature is not enabled --- crates/bevy_tasks/Cargo.toml | 2 + crates/bevy_tasks/src/bevy_executor.rs | 91 ++++++++++++++------------ 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 5b12d0c817372..62c4d78e3f84e 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -21,6 +21,7 @@ bevy_executor = [ "dep:fastrand", "dep:slab", "dep:thread_local", + "dep:crossbeam-utils", "dep:pin-project-lite", "async-task/std", ] @@ -55,6 +56,7 @@ atomic-waker = { version = "1", default-features = false } crossbeam-queue = { version = "0.3", default-features = false, features = [ "alloc", ] } +crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index fe07ae862190f..345874c21feb5 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -24,6 +24,7 @@ use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; +use crossbeam_utils::CachePadded; // ThreadLocalState *must* stay `Sync` due to a currently existing soundness hole. // See: https://github.com/Amanieu/thread_local-rs/issues/75 @@ -44,11 +45,11 @@ pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { } std::thread_local! { - static LOCAL_QUEUE: UnsafeCell = const { - UnsafeCell::new(LocalQueue { + static LOCAL_QUEUE: CachePadded> = const { + CachePadded::new(UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), local_active: Slab::new(), - }) + })) }; } @@ -765,7 +766,7 @@ impl Runner<'_> { } /// Waits for the next runnable task to run. - fn runnable(&mut self, rng: &mut fastrand::Rng) -> impl Future { + fn runnable(&mut self, _rng: &mut fastrand::Rng) -> impl Future { // SAFETY: The provided search function does not access LOCAL_QUEUE in any way, and thus cannot // alias. let runnable = unsafe { @@ -776,49 +777,53 @@ impl Runner<'_> { return Some(r); } - // Try the local queue. - if let Some(r) = self.local_state.stealable_queue.pop() { - return Some(r); - } - - // Try stealing from the global queue. - if let Some(r) = self.state.queue.pop() { - steal(&self.state.queue, &self.local_state.stealable_queue); - return Some(r); - } - - // Try stealing from other runners. - if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { - // Pick a random starting point in the iterator list and rotate the list. - let n = stealer_queues.len(); - let start = rng.usize(..n); - let iter = stealer_queues - .iter() - .chain(stealer_queues.iter()) - .skip(start) - .take(n); - - // Remove this runner's local queue. - let iter = - iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); - - // Try stealing from each local queue in the list. - for local in iter { - steal(*local, &self.local_state.stealable_queue); + crate::cfg::multi_threaded! { + if{ + // Try the local queue. if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } - } - } - if let Some(r) = self.local_state.thread_locked_queue.pop() { - // Do not steal from this queue. If other threads steal - // from this current thread, the task will be moved. - // - // Instead, flush all queued tasks into the local queue to - // minimize the effort required to scan for these tasks. - flush_to_local(&self.local_state.thread_locked_queue, tls); - return Some(r); + // Try stealing from the global queue. + if let Some(r) = self.state.queue.pop() { + steal(&self.state.queue, &self.local_state.stealable_queue); + return Some(r); + } + + // Try stealing from other runners. + if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { + // Pick a random starting point in the iterator list and rotate the list. + let n = stealer_queues.len(); + let start = _rng.usize(..n); + let iter = stealer_queues + .iter() + .chain(stealer_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { + return Some(r); + } + } + } + + if let Some(r) = self.local_state.thread_locked_queue.pop() { + // Do not steal from this queue. If other threads steal + // from this current thread, the task will be moved. + // + // Instead, flush all queued tasks into the local queue to + // minimize the effort required to scan for these tasks. + flush_to_local(&self.local_state.thread_locked_queue, tls); + return Some(r); + } + } else {} } None From 898166bb30c7c47d1d5f36ba833181f49c5c2cf9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:28:49 -0700 Subject: [PATCH 48/68] Fix Miri job to run them in sequence --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50fde2ced1ebb..bce2f347c7be8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,11 +118,13 @@ jobs: with: toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} components: miri - - name: CI job + - name: CI job (Tasks) # To run the tests one item at a time for troubleshooting, use # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_tasks --lib -- --exact - # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact run: cargo miri test -p bevy_tasks --features bevy_executor --features multi_threaded + - name: CI job (ECS) + # To run the tests one item at a time for troubleshooting, use + # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact run: cargo miri test -p bevy_ecs --features bevy_utils/debug check-compiles: From 2b967899f1a4a58b12687d8602dae59f6ee9265b Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:40:46 -0700 Subject: [PATCH 49/68] Fix up README and Clippy --- crates/bevy_tasks/README.md | 8 +++++--- crates/bevy_tasks/src/bevy_executor.rs | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/bevy_tasks/README.md b/crates/bevy_tasks/README.md index 04815df35e6ed..530489bfaa115 100644 --- a/crates/bevy_tasks/README.md +++ b/crates/bevy_tasks/README.md @@ -14,8 +14,8 @@ a single thread and having that thread await the completion of those tasks. This generating the tasks from a slice of data. This library is intended for games and makes no attempt to ensure fairness or ordering of spawned tasks. -It is based on [`async-executor`][async-executor], a lightweight executor that allows the end user to manage their own threads. -`async-executor` is based on async-task, a core piece of async-std. +It is based on a fork of [`async-executor`][async-executor], a lightweight executor that allows the end user to manage their own threads. +`async-executor` is based on [`async-task`][async-task], a core piece of [`smol`][smol]. ## Usage @@ -40,4 +40,6 @@ To enable `no_std` support in this crate, you will need to disable default featu [bevy]: https://bevy.org [rayon]: https://github.com/rayon-rs/rayon -[async-executor]: https://github.com/stjepang/async-executor +[async-executor]: https://github.com/smol-rs/async-executor +[smol]: https://github.com/smol-rs/smol +[async-task]: https://github.com/smol-rs/async-task diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 345874c21feb5..731b178195dd8 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1045,7 +1045,7 @@ mod test { use bevy_platform::sync::Mutex; use core::task::{Poll, Waker}; use async_task::Task; - use std::time::Duration; + use core::time::Duration; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -1086,7 +1086,7 @@ mod test { future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1108,7 +1108,7 @@ mod test { let task: Task<&str> = ex.spawn(async { &*s }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1130,7 +1130,7 @@ mod test { }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1153,7 +1153,7 @@ mod test { }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1173,12 +1173,13 @@ mod test { future::block_on(async move { let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; + #[expect(print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; let _ = stopper.recv().or(timeout).await; - }) + }); }); stop_timeout }; @@ -1209,7 +1210,7 @@ mod test { #[test] fn yield_now() { - do_run(|ex| async move { ex.spawn(future::yield_now()).await }) + do_run(|ex| async move { ex.spawn(future::yield_now()).await }); } #[test] @@ -1217,7 +1218,7 @@ mod test { do_run(|ex| async move { ex.spawn(async_io::Timer::after(Duration::from_millis(5))) .await; - }) + }); } #[test] @@ -1228,7 +1229,7 @@ mod test { // Running the executor should not panic. future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); From f5737f74b8be8d234cf5c8b543f8b4f09604da87 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:52:12 -0700 Subject: [PATCH 50/68] It's clippy --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 731b178195dd8..71b5b2acaea7a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1173,7 +1173,7 @@ mod test { future::block_on(async move { let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; - #[expect(print_stderr, reason = "Explicitly used to warn about timed out tests")] + #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; From 84edd4f024da022fbb0801de93f915402c9ec29f Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 14 Aug 2025 20:23:09 -0700 Subject: [PATCH 51/68] Address UB in spawn_scoped_local from OOMs --- crates/bevy_tasks/src/bevy_executor.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 71b5b2acaea7a..7500cc8cec2c5 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -13,6 +13,7 @@ use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; use core::cell::UnsafeCell; +use core::mem; use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; @@ -259,6 +260,8 @@ impl<'a> Executor<'a> { try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); + let builder = Builder::new().propagate_panic(true); + // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. This AsyncCallOnDrop will be // invoked after the surrounding scope has exited in either a @@ -267,6 +270,14 @@ impl<'a> Executor<'a> { try_with_local_queue(|tls| drop(tls.local_active.try_remove(index))).ok(); }); + // This is a critical section which will result in UB by aliasing active + // if the AsyncCallOnDrop is called while still in this function. + // + // To avoid this, this guard will abort the process if it does + // panic. Rust's drop order will ensure that this will run before + // executor, and thus before the above AsyncCallOnDrop is dropped. + let _panic_guard = AbortOnPanic; + // Create the task and register it in the set of active tasks. // // SAFETY: @@ -280,11 +291,12 @@ impl<'a> Executor<'a> { // must not leave the current thread of execution, and it does not // all of them are bound vy use of thread-local storage. // - `self.schedule_local()` is `'static`, as checked below. - let (runnable, task) = Builder::new() - .propagate_panic(true) + let (runnable, task) = builder .spawn_unchecked(|()| future, self.schedule_local()); entry.insert(runnable.waker()); + mem::forget(_panic_guard); + (runnable, task) }).unwrap() }; @@ -999,6 +1011,15 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re .finish() } +struct AbortOnPanic; + +impl Drop for AbortOnPanic { + fn drop(&mut self) { + // Panicking while unwinding will force an abort. + panic!("Aborting due to allocator error"); + } +} + /// Runs a closure when dropped. struct CallOnDrop(F); From dc7ce63e3da308bea81c4a443f8dff93c02d3e9a Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 14 Aug 2025 20:23:58 -0700 Subject: [PATCH 52/68] Move expect to outer block --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 7500cc8cec2c5..39b7a4a9dda28 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1192,9 +1192,9 @@ mod test { let (stop_timeout, stopper) = async_channel::bounded::<()>(1); std::thread::spawn(move || { future::block_on(async move { + #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; - #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; From 00b62c27bb25604e6b133a039d3575be3b783296 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 18 Aug 2025 17:06:23 -0700 Subject: [PATCH 53/68] Fix lint --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 39b7a4a9dda28..6ce0900e7b434 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -430,7 +430,7 @@ impl<'a> Executor<'a> { // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; let clone = arc.clone(); - core::mem::forget(arc); + mem::forget(arc); clone } } From 5d122a01a97384040e45ab0bafaa60541cb91f03 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 18 Aug 2025 22:25:31 -0700 Subject: [PATCH 54/68] Go back to using ConcurrentQueue --- crates/bevy_tasks/Cargo.toml | 5 +- crates/bevy_tasks/src/bevy_executor.rs | 79 +++++++++----------------- crates/bevy_tasks/src/edge_executor.rs | 6 +- 3 files changed, 31 insertions(+), 59 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index b44a608c39650..aa2b47a3e4bb2 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -25,6 +25,7 @@ bevy_executor = [ "dep:pin-project-lite", "futures-lite", "async-task/std", + "concurrent-queue/std", ] # Provide an implementation of `block_on` from `futures-lite`. @@ -54,9 +55,7 @@ fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } -crossbeam-queue = { version = "0.3", default-features = false, features = [ - "alloc", -] } +concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 6ce0900e7b434..0c8a669563a55 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -21,7 +21,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use crossbeam_queue::{ArrayQueue, SegQueue}; +use concurrent_queue::{ConcurrentQueue, PushError}; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -83,16 +83,16 @@ impl Drop for LocalQueue { struct ThreadLocalState { executor: AtomicPtr, - stealable_queue: ArrayQueue, - thread_locked_queue: SegQueue, + stealable_queue: ConcurrentQueue, + thread_locked_queue: ConcurrentQueue, } impl Default for ThreadLocalState { fn default() -> Self { Self { executor: AtomicPtr::new(core::ptr::null_mut()), - stealable_queue: ArrayQueue::new(512), - thread_locked_queue: SegQueue::new(), + stealable_queue: ConcurrentQueue::bounded(512), + thread_locked_queue: ConcurrentQueue::unbounded(), } } } @@ -104,7 +104,7 @@ impl Default for ThreadLocalState { #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, - target_queue: &'static SegQueue, + target_queue: &'static ConcurrentQueue, state: Arc, _marker: PhantomData<&'a ()>, } @@ -360,7 +360,7 @@ impl<'a> Executor<'a> { state.notify_specific_thread(std::thread::current().id(), true); return; } - Err(r) => r, + Err(r) => r.into_inner(), } } else { runnable @@ -452,17 +452,17 @@ impl Drop for Executor<'_> { } drop(active); - while state.queue.pop().is_some() {} + while state.queue.pop().is_ok() {} } } /// The state of a executor. struct State { /// The global queue. - queue: SegQueue, + queue: ConcurrentQueue, /// Local queues created by runners. - stealer_queues: RwLock>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -478,7 +478,7 @@ impl State { /// Creates state for a new executor. const fn new() -> State { State { - queue: SegQueue::new(), + queue: ConcurrentQueue::unbounded(), stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { @@ -792,12 +792,12 @@ impl Runner<'_> { crate::cfg::multi_threaded! { if{ // Try the local queue. - if let Some(r) = self.local_state.stealable_queue.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. - if let Some(r) = self.state.queue.pop() { + if let Ok(r) = self.state.queue.pop() { steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } @@ -820,13 +820,13 @@ impl Runner<'_> { // Try stealing from each local queue in the list. for local in iter { steal(*local, &self.local_state.stealable_queue); - if let Some(r) = self.local_state.stealable_queue.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } } } - if let Some(r) = self.local_state.thread_locked_queue.pop() { + if let Ok(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. // @@ -870,65 +870,38 @@ impl Drop for Runner<'_> { } // Re-schedule remaining tasks in the local queue. - while let Some(r) = self.local_state.stealable_queue.pop() { + while let Ok(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } } -trait WorkQueue { - fn stealable_count(&self) -> usize; - fn queue_pop(&self) -> Option; -} - -impl WorkQueue for ArrayQueue { - #[inline] - fn stealable_count(&self) -> usize { - self.len().div_ceil(2) - } - - #[inline] - fn queue_pop(&self) -> Option { - self.pop() - } -} - -impl WorkQueue for SegQueue { - #[inline] - fn stealable_count(&self) -> usize { - self.len() - } - - #[inline] - fn queue_pop(&self) -> Option { - self.pop() - } -} - /// Steals some items from one queue into another. -fn steal>(src: &Q, dest: &ArrayQueue) { +fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Half of `src`'s length rounded up. - let mut count = src.stealable_count(); + let mut count = src.len(); if count > 0 { - // Don't steal more than fits into the queue. - count = count.min(dest.capacity() - dest.len()); + if let Some(capacity) = dest.capacity() { + // Don't steal more than fits into the queue. + count = count.min(capacity- dest.len()); + } // Steal tasks. for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; + let Ok(val) = src.pop() else { break }; assert!(dest.push(val).is_ok()); } } } -fn flush_to_local(src: &SegQueue, dst: &mut LocalQueue) { +fn flush_to_local(src: &ConcurrentQueue, dst: &mut LocalQueue) { let count = src.len(); if count > 0 { // Steal tasks. for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; + let Ok(val) = src.pop() else { break }; dst.local_queue.push_front(val); } } @@ -975,7 +948,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index dc17077aef66d..be0589c7beff4 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -209,7 +209,7 @@ impl<'a, const C: usize> Executor<'a, C> { target_has_atomic = "ptr" ))] { - runnable = self.state().queue.pop(); + runnable = self.state().queue.pop().ok(); } #[cfg(not(all( @@ -314,7 +314,7 @@ struct State { target_has_atomic = "64", target_has_atomic = "ptr" ))] - queue: crossbeam_queue::ArrayQueue, + queue: concurrent_queue::ConcurrentQueue, #[cfg(not(all( target_has_atomic = "8", target_has_atomic = "16", @@ -336,7 +336,7 @@ impl State { target_has_atomic = "64", target_has_atomic = "ptr" ))] - queue: crossbeam_queue::ArrayQueue::new(C), + queue: concurrent_queue::ConcurrentQueue::bounded(C), #[cfg(not(all( target_has_atomic = "8", target_has_atomic = "16", From 73798b1908efe2a564afa7a60e0d3c6b34be548b Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 00:53:15 -0700 Subject: [PATCH 55/68] Don't panic --- crates/bevy_tasks/src/bevy_executor.rs | 24 +++++++++-------- crates/bevy_tasks/src/task_pool.rs | 36 +++++++++++++------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 0c8a669563a55..750575c71c879 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -21,7 +21,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use concurrent_queue::{ConcurrentQueue, PushError}; +use concurrent_queue::ConcurrentQueue; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -156,7 +156,8 @@ impl<'a> ThreadSpawner<'a> { // Instead of directly scheduling this task, it's put into the onto the // thread locked queue to be moved to the target thread, where it will // either be run immediately or flushed into the thread's local queue. - self.target_queue.push(runnable); + let result = self.target_queue.push(runnable); + debug_assert!(result.is_ok()); task } @@ -369,7 +370,8 @@ impl<'a> Executor<'a> { runnable }; // Otherwise push onto the global queue instead. - state.queue.push(runnable); + let result = state.queue.push(runnable); + debug_assert!(result.is_ok()); state.notify(); } } @@ -503,7 +505,7 @@ impl State { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_ok() { - let waker = self.sleepers.lock().unwrap().notify(); + let waker = self.sleepers.lock().unwrap_or_else(PoisonError::into_inner).notify(); if let Some(w) = waker { w.wake(); } @@ -513,7 +515,7 @@ impl State { /// Notifies a sleeping ticker. #[inline] fn notify_specific_thread(&self, thread_id: ThreadId, allow_stealing: bool) { - let mut sleepers = self.sleepers.lock().unwrap(); + let mut sleepers = self.sleepers.lock().unwrap_or_else(PoisonError::into_inner); let mut waker = sleepers.notify_specific_thread(thread_id); if waker.is_none() && allow_stealing @@ -649,7 +651,7 @@ impl Ticker<'_> { /// /// Returns `false` if the ticker was already sleeping and unnotified. fn sleep(&mut self, waker: &Waker) -> bool { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); match self.sleeping { // Move to sleeping state. @@ -675,7 +677,7 @@ impl Ticker<'_> { /// Moves the ticker into woken state. fn wake(&mut self) { if self.sleeping != 0 { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); sleepers.remove(self.sleeping); self.state @@ -716,7 +718,7 @@ impl Ticker<'_> { } } } - }).unwrap() + }).unwrap_or(Poll::Pending) } }) } @@ -726,7 +728,7 @@ impl Drop for Ticker<'_> { fn drop(&mut self) { // If this ticker is in sleeping state, it must be removed from the sleepers list. if self.sleeping != 0 { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); let notified = sleepers.remove(self.sleeping); self.state @@ -772,7 +774,7 @@ impl Runner<'_> { state .stealer_queues .write() - .unwrap() + .unwrap_or_else(PoisonError::into_inner) .push(&local_state.stealable_queue); runner } @@ -790,7 +792,7 @@ impl Runner<'_> { } crate::cfg::multi_threaded! { - if{ + if { // Try the local queue. if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 086e05d4e0614..270089b59f34f 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -5,7 +5,7 @@ use std::thread::{self, JoinHandle}; use crate::bevy_executor::Executor; use async_task::FallibleTask; use bevy_platform::sync::Arc; -use crossbeam_queue::SegQueue; +use concurrent_queue::ConcurrentQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; @@ -347,11 +347,11 @@ impl TaskPool { let executor: &Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env Executor = unsafe { mem::transmute(executor) }; - let spawned: SegQueue>>> = - SegQueue::new(); + let spawned: ConcurrentQueue>>> = + ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env SegQueue>>> = + let spawned: &'env ConcurrentQueue>>> = unsafe { mem::transmute(&spawned) }; let scope = Scope { @@ -374,14 +374,11 @@ impl TaskPool { } else { block_on(self.executor.run(async move { let mut results = Vec::with_capacity(spawned.len()); - while let Some(task) = spawned.pop() { - if let Some(res) = task.await { - match res { - Ok(res) => results.push(res), - Err(payload) => std::panic::resume_unwind(payload), - } - } else { - panic!("Failed to catch panic!"); + while let Ok(task) = spawned.pop() { + match task.await { + Some(Ok(res)) => results.push(res), + Some(Err(payload)) => std::panic::resume_unwind(payload), + None => panic!("Failed to catch panic!"), } } results @@ -455,7 +452,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope SegQueue>>>, + spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, @@ -475,7 +472,8 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .executor .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); - self.spawned.push(task); + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } #[expect( @@ -496,7 +494,8 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) .fallible() }; - self.spawned.push(task); + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } #[expect( @@ -519,8 +518,9 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .fallible() }; // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task); + // close and use an unbounded queue, so pushing should always succeed. + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } } @@ -530,7 +530,7 @@ where { fn drop(&mut self) { block_on(async { - while let Some(task) = self.spawned.pop() { + while let Ok(task) = self.spawned.pop() { task.cancel().await; } }); From 909f2e53324e0d95e4931bb1b585c497a16e8086 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 00:56:10 -0700 Subject: [PATCH 56/68] Formatting --- crates/bevy_tasks/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index aa2b47a3e4bb2..0eb81732728c0 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -55,7 +55,7 @@ fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } -concurrent-queue = { version = "2.5", default-features = false } +concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] From 96dd3d5d89ca1eee349dbb997cb8eeab638ee4a7 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 20:03:32 -0700 Subject: [PATCH 57/68] is_multiple_of lint --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 750575c71c879..ec9ee402994ba 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -847,7 +847,7 @@ impl Runner<'_> { // Bump the tick counter. self.ticks = self.ticks.wrapping_add(1); - if self.ticks % 64 == 0 { + if self.ticks.is_multiple_of(64) { // Steal tasks from the global queue to ensure fair task scheduling. steal(&self.state.queue, &self.local_state.stealable_queue); } From b10d226f7dc07f7b4f5e7b6f1835f1cf1914ecc1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 24 Aug 2025 00:08:18 -0700 Subject: [PATCH 58/68] Minimize cfg blocks --- crates/bevy_tasks/src/edge_executor.rs | 9 +++++++++ .../bevy_tasks/src/single_threaded_task_pool.rs | 15 +++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index be0589c7beff4..7abf27ed02f48 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -105,6 +105,15 @@ impl<'a, const C: usize> Executor<'a, C> { unsafe { self.spawn_unchecked(fut) } } + pub unsafe fn spawn_local_scoped(&self, fut: F) -> Task + where + F: Future + 'a, + F::Output: 'a, + { + // SAFETY: Original implementation missing safety documentation + unsafe { self.spawn_unchecked(fut) } + } + /// Attempts to run a task if at least one is scheduled. /// /// Running a scheduled task means simply polling its future once. diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 62f9f75e1fa23..1b652b23a8000 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -278,16 +278,11 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { }; let mut tasks = self.tasks_ref.borrow_mut(); - crate::cfg::bevy_executor! { - if { - #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] - // SAFETY: The surrounding scope will not terminate until all local tasks are done - // ensuring that the borrowed variables do not outlive the detached task. - tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); - } else { - tasks.push(self.executor_ref.spawn_local(f)); - } - } + + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] + // SAFETY: The surrounding scope will not terminate until all local tasks are done + // ensuring that the borrowed variables do not outlive the detached task. + tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); } } From e8b9c1f13ca15c8946e628cd62403e5585e7f4b4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 24 Aug 2025 00:32:55 -0700 Subject: [PATCH 59/68] Try getting rid of the main thread local executor tick system --- crates/bevy_app/src/app.rs | 5 +---- crates/bevy_app/src/schedule_runner.rs | 5 +---- crates/bevy_app/src/task_pool_plugin.rs | 18 --------------- crates/bevy_tasks/src/lib.rs | 6 ----- .../src/single_threaded_task_pool.rs | 22 ++++++------------- crates/bevy_tasks/src/usages.rs | 18 --------------- crates/bevy_winit/src/state.rs | 7 +----- 7 files changed, 10 insertions(+), 71 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 26a89d9f3ea3e..d0d2da8328a27 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -1401,10 +1401,7 @@ impl Plugin for HokeyPokey { type RunnerFn = Box AppExit>; fn run_once(mut app: App) -> AppExit { - while app.plugins_state() == PluginsState::Adding { - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - bevy_tasks::tick_global_task_pools_on_main_thread(); - } + while app.plugins_state() == PluginsState::Adding {} app.finish(); app.cleanup(); diff --git a/crates/bevy_app/src/schedule_runner.rs b/crates/bevy_app/src/schedule_runner.rs index 594f849b2f905..c69c79c96875d 100644 --- a/crates/bevy_app/src/schedule_runner.rs +++ b/crates/bevy_app/src/schedule_runner.rs @@ -76,10 +76,7 @@ impl Plugin for ScheduleRunnerPlugin { app.set_runner(move |mut app: App| { let plugins_state = app.plugins_state(); if plugins_state != PluginsState::Cleaned { - while app.plugins_state() == PluginsState::Adding { - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - bevy_tasks::tick_global_task_pools_on_main_thread(); - } + while app.plugins_state() == PluginsState::Adding {} app.finish(); app.cleanup(); } diff --git a/crates/bevy_app/src/task_pool_plugin.rs b/crates/bevy_app/src/task_pool_plugin.rs index 8014790f07772..a68938910cd1d 100644 --- a/crates/bevy_app/src/task_pool_plugin.rs +++ b/crates/bevy_app/src/task_pool_plugin.rs @@ -6,21 +6,6 @@ use bevy_tasks::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, TaskPoolBuil use core::fmt::Debug; use log::trace; -cfg_if::cfg_if! { - if #[cfg(not(all(target_arch = "wasm32", feature = "web")))] { - use {crate::Last, bevy_tasks::tick_global_task_pools_on_main_thread}; - use bevy_ecs::system::NonSendMarker; - - /// A system used to check and advanced our task pools. - /// - /// Calls [`tick_global_task_pools_on_main_thread`], - /// and uses [`NonSendMarker`] to ensure that this system runs on the main thread - fn tick_global_task_pools(_main_thread_marker: NonSendMarker) { - tick_global_task_pools_on_main_thread(); - } - } -} - /// Setup of default task pools: [`AsyncComputeTaskPool`], [`ComputeTaskPool`], [`IoTaskPool`]. #[derive(Default)] pub struct TaskPoolPlugin { @@ -32,9 +17,6 @@ impl Plugin for TaskPoolPlugin { fn build(&self, _app: &mut App) { // Setup the default bevy task pools self.task_pool_options.create_default_pools(); - - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - _app.add_systems(Last, tick_global_task_pools); } } diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index d03665c94bc86..7715e71395fb8 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -95,12 +95,6 @@ pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; pub use futures_lite; pub use futures_lite::future::poll_once; -cfg::web! { - if {} else { - pub use usages::tick_global_task_pools_on_main_thread; - } -} - cfg::multi_threaded! { if { mod task_pool; diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 1b652b23a8000..d76898b507e99 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -186,7 +186,13 @@ impl TaskPool { _ => { let task = EXECUTOR.spawn_local(future); // Loop until all tasks are done - while Self::try_tick_local() {} + crate::cfg::bevy_executor! { + if { + while !Executor::try_tick_local() {} + } else { + while EXECUTOR.try_tick() {} + } + } Task::new(task) } @@ -203,20 +209,6 @@ impl TaskPool { { self.spawn(future) } - - crate::cfg::web! { - if {} else { - pub(crate) fn try_tick_local() -> bool { - crate::cfg::bevy_executor! { - if { - Executor::try_tick_local() - } else { - EXECUTOR.try_tick() - } - } - } - } - } } /// A `TaskPool` scope for running one or more non-`'static` futures. diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index e96cbdc6b80b4..b6da4ac7b860e 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -74,21 +74,3 @@ taskpool! { /// See [`TaskPool`] documentation for details on Bevy tasks. (IO_TASK_POOL, IoTaskPool) } - -crate::cfg::web! { - if {} else { - /// A function used by `bevy_app` to tick the global tasks pools on the main thread. - /// This will run a maximum of 100 local tasks per executor per call to this function. - /// - /// # Warning - /// - /// This function *must* be called on the main thread, or the task pools will not be updated appropriately. - pub fn tick_global_task_pools_on_main_thread() { - for _ in 0..100 { - if !TaskPool::try_tick_local() { - break; - } - } - } - } -} diff --git a/crates/bevy_winit/src/state.rs b/crates/bevy_winit/src/state.rs index 6c1ed3dd446f0..53332d9fc74b9 100644 --- a/crates/bevy_winit/src/state.rs +++ b/crates/bevy_winit/src/state.rs @@ -15,8 +15,6 @@ use bevy_input::{ use bevy_log::{trace, warn}; use bevy_math::{ivec2, DVec2, Vec2}; use bevy_platform::time::Instant; -#[cfg(not(target_arch = "wasm32"))] -use bevy_tasks::tick_global_task_pools_on_main_thread; use core::marker::PhantomData; #[cfg(target_arch = "wasm32")] use winit::platform::web::EventLoopExtWebSys; @@ -153,10 +151,7 @@ impl ApplicationHandler for WinitAppRunnerState { let _span = tracing::info_span!("winit event_handler").entered(); if self.app.plugins_state() != PluginsState::Cleaned { - if self.app.plugins_state() != PluginsState::Ready { - #[cfg(not(target_arch = "wasm32"))] - tick_global_task_pools_on_main_thread(); - } else { + if self.app.plugins_state() == PluginsState::Ready { self.app.finish(); self.app.cleanup(); } From 3354efa1fb2b99166dd0f55552bec3b55cac85f8 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 24 Aug 2025 20:07:12 -0700 Subject: [PATCH 60/68] Use a proper struct for Sleepers --- crates/bevy_tasks/src/bevy_executor.rs | 40 +++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index ec9ee402994ba..359da1a12324a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -538,6 +538,13 @@ impl fmt::Debug for State { } } +/// A sleeping ticker +struct Sleeper { + id: usize, + thread_id: ThreadId, + waker: Waker, +} + /// A list of sleeping tickers. struct Sleepers { /// Number of sleeping tickers (both notified and unnotified). @@ -546,7 +553,7 @@ struct Sleepers { /// IDs and wakers of sleeping unnotified tickers. /// /// A sleeping ticker is notified when its waker is missing from this list. - wakers: Vec<(usize, ThreadId, Waker)>, + wakers: Vec, /// Reclaimed IDs. free_ids: Vec, @@ -555,13 +562,13 @@ struct Sleepers { impl Sleepers { /// Inserts a new sleeping ticker. fn insert(&mut self, waker: &Waker) -> usize { - let id = match self.free_ids.pop() { - Some(id) => id, - None => self.count + 1, - }; + let id = self.free_ids.pop().unwrap_or_else(|| self.count + 1); self.count += 1; - self.wakers - .push((id, std::thread::current().id(), waker.clone())); + self.wakers.push(Sleeper { + id, + thread_id: std::thread::current().id(), + waker: waker.clone() + }); id } @@ -570,14 +577,15 @@ impl Sleepers { /// Returns `true` if the ticker was notified. fn update(&mut self, id: usize, waker: &Waker) -> bool { for item in &mut self.wakers { - if item.0 == id { - item.2.clone_from(waker); + if item.id == id { + item.waker.clone_from(waker); return false; } } self.wakers - .push((id, std::thread::current().id(), waker.clone())); + .push(Sleeper { id, thread_id: std::thread::current().id(), + waker: waker.clone() }); true } @@ -589,7 +597,7 @@ impl Sleepers { self.free_ids.push(id); for i in (0..self.wakers.len()).rev() { - if self.wakers[i].0 == id { + if self.wakers[i].id == id { self.wakers.remove(i); return false; } @@ -607,7 +615,7 @@ impl Sleepers { /// If a ticker was notified already or there are no tickers, `None` will be returned. fn notify(&mut self) -> Option { if self.wakers.len() == self.count { - self.wakers.pop().map(|item| item.2) + self.wakers.pop().map(|item| item.waker) } else { None } @@ -617,10 +625,10 @@ impl Sleepers { /// /// If a ticker was notified already or there are no tickers, `None` will be returned. fn notify_specific_thread(&mut self, thread_id: ThreadId) -> Option { - for i in (0..self.wakers.len()).rev() { - if self.wakers[i].1 == thread_id { - let (_, _, waker) = self.wakers.remove(i); - return Some(waker); + for i in 0..self.wakers.len() { + if self.wakers[i].thread_id == thread_id { + let sleeper = self.wakers.remove(i); + return Some(sleeper.waker); } } None From c6fecbd16f659644c09d1564f34c88c7aa44025c Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 24 Aug 2025 21:28:18 -0700 Subject: [PATCH 61/68] Update module docs --- crates/bevy_tasks/src/bevy_executor.rs | 6 ++++++ crates/bevy_tasks/src/edge_executor.rs | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 359da1a12324a..309192848cc8c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1,3 +1,9 @@ +//! Fork of `async_executor`. +//! +//! It has been vendored along with its tests to update several outdated dependencies. +//! +//! [`async_executor`]: https://github.com/smol-rs/async-executor + #![expect( unsafe_code, reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 7abf27ed02f48..efe3d9930c3c5 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -1,8 +1,7 @@ -//! Alternative to `async_executor` based on [`edge_executor`] by Ivan Markov. +//! Alternative to `bevy_executor` based on [`edge_executor`] by Ivan Markov. //! //! It has been vendored along with its tests to update several outdated dependencies. //! -//! [`async_executor`]: https://github.com/smol-rs/async-executor //! [`edge_executor`]: https://github.com/ivmarkov/edge-executor #![expect(unsafe_code, reason = "original implementation relies on unsafe")] From 5df82e1737d09e72fe57ea6a785099626c338720 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 24 Aug 2025 22:31:17 -0700 Subject: [PATCH 62/68] Remove unused code --- crates/bevy_tasks/src/bevy_executor.rs | 8 +++++--- crates/bevy_tasks/src/task_pool.rs | 4 ---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 309192848cc8c..cd67f4e44e8e2 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -589,9 +589,11 @@ impl Sleepers { } } - self.wakers - .push(Sleeper { id, thread_id: std::thread::current().id(), - waker: waker.clone() }); + self.wakers.push(Sleeper { + id, + thread_id: std::thread::current().id(), + waker: waker.clone() + }); true } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 270089b59f34f..d2985fb485641 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -418,10 +418,6 @@ impl TaskPool { { Task::new(self.executor.spawn_local(future)) } - - pub(crate) fn try_tick_local() -> bool { - Executor::try_tick_local() - } } impl Default for TaskPool { From ee8f53113d035a55e610a818523eae99b545fd59 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 25 Aug 2025 10:18:33 -0700 Subject: [PATCH 63/68] Cleanup using utilities we already have --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index d76898b507e99..14571aefb6df7 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,5 +1,6 @@ use alloc::{string::String, vec::Vec, fmt}; -use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem, task::{Poll, Context, Waker}, pin::Pin}; +use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; +use crate::futures::now_or_never; use crate::{block_on, Task}; @@ -155,14 +156,11 @@ impl TaskPool { } })); - let mut context = Context::from_waker(Waker::noop()); tasks .take() .into_iter() - .map(|mut task| match Pin::new(&mut task).poll(&mut context) { - Poll::Ready(result) => result, - Poll::Pending => unreachable!(), - }) + .map(now_or_never) + .map(Option::unwrap) .collect() } From 20fe27b93054ce437233b0278992edff00a0cbcc Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 27 Aug 2025 19:52:55 -0700 Subject: [PATCH 64/68] Avoid the extra Waker clone when initially scheduling tasks --- crates/bevy_tasks/src/bevy_executor.rs | 85 ++++++++++++++++---------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index cd67f4e44e8e2..763cabb064f6f 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -212,13 +212,14 @@ impl<'a> Executor<'a> { /// Spawns a task onto the executor. pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state().active(); + let state = self.state(); + let mut active = state.active(); // Remove the task from the set of active tasks when the future finishes. let entry = active.vacant_entry(); let index = entry.key(); - let state = self.state_as_arc(); - let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + let state_arc = self.state_as_arc(); + let future = AsyncCallOnDrop::new(future, move || drop(state_arc.active().try_remove(index))); // Create the task and register it in the set of active tasks. // @@ -241,7 +242,9 @@ impl<'a> Executor<'a> { }; entry.insert(runnable.waker()); - runnable.schedule(); + // Runnable::schedule has a extra extraneous Waker clone/drop if the schedule captures + // variables, so directly schedule here instead. + Self::schedule_runnable(&state, runnable); task } @@ -263,7 +266,7 @@ impl<'a> Executor<'a> { // // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. - let (runnable, task) = unsafe { + unsafe { try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); @@ -304,12 +307,13 @@ impl<'a> Executor<'a> { mem::forget(_panic_guard); - (runnable, task) - }).unwrap() - }; + // Runnable::schedule has a extra extraneous Waker clone/drop if the schedule captures + // variables, so directly schedule here instead. + Self::schedule_runnable_local(&self.state(), tls, runnable); - runnable.schedule(); - task + task + }).unwrap() + } } pub fn current_thread_spawner(&self) -> ThreadSpawner<'a> { @@ -358,27 +362,7 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); move |runnable| { - // Attempt to push onto the local queue first in dedicated executor threads, - // because we know that this thread is awake and always processing new tasks. - let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), Arc::as_ptr(&state)) { - match local_state.stealable_queue.push(runnable) { - Ok(()) => { - state.notify_specific_thread(std::thread::current().id(), true); - return; - } - Err(r) => r.into_inner(), - } - } else { - runnable - } - } else { - runnable - }; - // Otherwise push onto the global queue instead. - let result = state.queue.push(runnable); - debug_assert!(result.is_ok()); - state.notify(); + Self::schedule_runnable(&state, runnable); } } @@ -389,12 +373,47 @@ impl<'a> Executor<'a> { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { - state.notify_specific_thread(std::thread::current().id(), false); + unsafe { + // If this returns Err, the thread's destructor has been called and thus it's meaningless + // to push onto the queue. + let _ = try_with_local_queue(|tls| { + Self::schedule_runnable_local(&state, tls, runnable); + }); } } } + #[inline] + fn schedule_runnable(state: &State, runnable: Runnable) { + // Attempt to push onto the local queue first in dedicated executor threads, + // because we know that this thread is awake and always processing new tasks. + let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), state) { + match local_state.stealable_queue.push(runnable) { + Ok(()) => { + state.notify_specific_thread(std::thread::current().id(), true); + return; + } + Err(r) => r.into_inner(), + } + } else { + runnable + } + } else { + runnable + }; + // Otherwise push onto the global queue instead. + let result = state.queue.push(runnable); + debug_assert!(result.is_ok()); + state.notify(); + } + + #[inline] + fn schedule_runnable_local(state: &State, tls: &mut LocalQueue, runnable: Runnable) { + tls.local_queue.push_back(runnable); + state.notify_specific_thread(std::thread::current().id(), false); + } + /// Returns a pointer to the inner state. #[inline] fn state_ptr(&self) -> *const State { From ea262ef2c7027190e86333e7d8cbff1bef5861e4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 27 Aug 2025 20:01:37 -0700 Subject: [PATCH 65/68] Clippy, formatting, and panic documentation --- crates/bevy_tasks/src/bevy_executor.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 763cabb064f6f..8cd6c98b89651 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -242,9 +242,9 @@ impl<'a> Executor<'a> { }; entry.insert(runnable.waker()); - // Runnable::schedule has a extra extraneous Waker clone/drop if the schedule captures + // `Runnable::schedule ` has a extra extraneous Waker clone/drop if the schedule captures // variables, so directly schedule here instead. - Self::schedule_runnable(&state, runnable); + Self::schedule_runnable(state, runnable); task } @@ -255,6 +255,10 @@ impl<'a> Executor<'a> { } /// Spawns a non-'static and non-Send task onto the executor. + /// + /// # Panics + /// - This function will panic if any of the internal allocations causes an OOM (out of memory) error. + /// - This function will panic if the current thread's destructor has already been called. /// /// # Safety /// The caller must ensure that the returned Task does not outlive 'a. @@ -309,7 +313,7 @@ impl<'a> Executor<'a> { // Runnable::schedule has a extra extraneous Waker clone/drop if the schedule captures // variables, so directly schedule here instead. - Self::schedule_runnable_local(&self.state(), tls, runnable); + Self::schedule_runnable_local(self.state(), tls, runnable); task }).unwrap() From f305276364619c8456afcfcdf4297ba72a0240d1 Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 31 Aug 2025 22:00:11 -0700 Subject: [PATCH 66/68] Rename ThreadSpawner -> LocalTaskSpawner --- crates/bevy_ecs/src/schedule/executor/mod.rs | 2 +- .../src/schedule/executor/multi_threaded.rs | 14 +++++----- crates/bevy_render/src/pipelined_rendering.rs | 8 +++--- crates/bevy_tasks/src/bevy_executor.rs | 8 +++--- crates/bevy_tasks/src/lib.rs | 4 +-- .../src/single_threaded_task_pool.rs | 10 +++---- crates/bevy_tasks/src/task_pool.rs | 26 +++++++++---------- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/crates/bevy_ecs/src/schedule/executor/mod.rs b/crates/bevy_ecs/src/schedule/executor/mod.rs index 7219bbb3ab801..1949701f15df8 100644 --- a/crates/bevy_ecs/src/schedule/executor/mod.rs +++ b/crates/bevy_ecs/src/schedule/executor/mod.rs @@ -11,7 +11,7 @@ use core::any::TypeId; pub use self::{simple::SimpleExecutor, single_threaded::SingleThreadedExecutor}; #[cfg(feature = "std")] -pub use self::multi_threaded::{MainThreadSpawner, MultiThreadedExecutor}; +pub use self::multi_threaded::{MainThreadTaskSpawner, MultiThreadedExecutor}; use fixedbitset::FixedBitSet; diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index b16e6e22a7b53..3dd1d20109559 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, vec::Vec}; use bevy_platform::cell::SyncUnsafeCell; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadSpawner}; +use bevy_tasks::{ComputeTaskPool, LocalTaskSpawner, Scope, TaskPool}; use concurrent_queue::ConcurrentQueue; use core::{any::Any, panic::AssertUnwindSafe}; use fixedbitset::FixedBitSet; @@ -269,7 +269,7 @@ impl SystemExecutor for MultiThreadedExecutor { } let thread_executor = world - .get_resource::() + .get_resource::() .map(|e| e.0.clone()); let environment = &Environment::new(self, schedule, world); @@ -861,20 +861,20 @@ unsafe fn evaluate_and_fold_conditions( .fold(true, |acc, res| acc && res) } -/// New-typed [`ThreadSpawner`] [`Resource`] that is used to run systems on the main thread +/// New-typed [`LocalTaskSpawner`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadSpawner(pub ThreadSpawner<'static>); +pub struct MainThreadTaskSpawner(pub LocalTaskSpawner<'static>); -impl Default for MainThreadSpawner { +impl Default for MainThreadTaskSpawner { fn default() -> Self { Self::new() } } -impl MainThreadSpawner { +impl MainThreadTaskSpawner { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadSpawner(ComputeTaskPool::get().current_thread_spawner()) + MainThreadTaskSpawner(ComputeTaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 61f9a446a2150..695d0ace21a01 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -3,7 +3,7 @@ use async_channel::{Receiver, Sender}; use bevy_app::{App, AppExit, AppLabel, Plugin, SubApp}; use bevy_ecs::{ resource::Resource, - schedule::MainThreadSpawner, + schedule::MainThreadTaskSpawner, world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; @@ -114,7 +114,7 @@ impl Plugin for PipelinedRenderingPlugin { if app.get_sub_app(RenderApp).is_none() { return; } - app.insert_resource(MainThreadSpawner::new()); + app.insert_resource(MainThreadTaskSpawner::new()); let mut sub_app = SubApp::new(); sub_app.set_extract(renderer_extract); @@ -136,7 +136,7 @@ impl Plugin for PipelinedRenderingPlugin { .expect("Unable to get RenderApp. Another plugin may have removed the RenderApp before PipelinedRenderingPlugin"); // clone main thread executor to render world - let executor = app.world().get_resource::().unwrap(); + let executor = app.world().get_resource::().unwrap(); render_app.world_mut().insert_resource(executor.clone()); render_to_app_sender.send_blocking(render_app).unwrap(); @@ -181,7 +181,7 @@ impl Plugin for PipelinedRenderingPlugin { // This function waits for the rendering world to be received, // runs extract, and then sends the rendering world back to the render thread. fn renderer_extract(app_world: &mut World, _world: &mut World) { - app_world.resource_scope(|world, main_thread_executor: Mut| { + app_world.resource_scope(|world, main_thread_executor: Mut| { world.resource_scope(|world, mut render_channels: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 8cd6c98b89651..0806a0d53ccb4 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -108,14 +108,14 @@ impl Default for ThreadLocalState { /// /// [`TaskPool::current_thread_spawner`]: crate::TaskPool::current_thread_spawner #[derive(Clone, Debug)] -pub struct ThreadSpawner<'a> { +pub struct LocalTaskSpawner<'a> { thread_id: ThreadId, target_queue: &'static ConcurrentQueue, state: Arc, _marker: PhantomData<&'a ()>, } -impl<'a> ThreadSpawner<'a> { +impl<'a> LocalTaskSpawner<'a> { /// Spawns a task onto the specific target thread. pub fn spawn( &self, @@ -320,8 +320,8 @@ impl<'a> Executor<'a> { } } - pub fn current_thread_spawner(&self) -> ThreadSpawner<'a> { - ThreadSpawner { + pub fn current_thread_spawner(&self) -> LocalTaskSpawner<'a> { + LocalTaskSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, state: self.state_as_arc(), diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 7715e71395fb8..4a2d4d1b6f477 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -99,11 +99,11 @@ cfg::multi_threaded! { if { mod task_pool; - pub use task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; + pub use task_pool::{Scope, TaskPool, TaskPoolBuilder, LocalTaskSpawner}; } else { mod single_threaded_task_pool; - pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; + pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, LocalTaskSpawner}; } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 14571aefb6df7..044a9fdd8f10f 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -22,9 +22,9 @@ pub struct TaskPoolBuilder {} /// task pool. In the case of the multithreaded task pool this struct is used to spawn /// tasks on a specific thread. But the wasm task pool just calls /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread -/// and so the [`ThreadSpawner`] does nothing. +/// and so the [`LocalTaskSpawner`] does nothing. #[derive(Clone)] -pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); +pub struct LocalTaskSpawner<'a>(PhantomData<&'a ()>); impl TaskPoolBuilder { /// Creates a new `TaskPoolBuilder` instance @@ -70,8 +70,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { - ThreadSpawner(PhantomData) + pub fn current_thread_spawner(&self) -> LocalTaskSpawner<'static> { + LocalTaskSpawner(PhantomData) } /// Create a `TaskPool` with the default configuration. @@ -109,7 +109,7 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] pub fn scope_with_executor<'env, F, T>( &self, - _thread_executor: Option, + _thread_executor: Option, f: F, ) -> Vec where diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index d2985fb485641..159a28366716e 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -10,7 +10,7 @@ use futures_lite::FutureExt; use crate::{block_on, Task}; -pub use crate::bevy_executor::ThreadSpawner; +pub use crate::bevy_executor::LocalTaskSpawner; struct CallOnDrop(Option>); @@ -138,9 +138,9 @@ pub struct TaskPool { } impl TaskPool { - /// Creates a [`ThreadSpawner`] for this current thread of execution. + /// Creates a [`LocalTaskSpawner`] for this current thread of execution. /// Can be used to spawn new tasks to execute exclusively on this thread. - pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + pub fn current_thread_spawner(&self) -> LocalTaskSpawner<'static> { self.executor.current_thread_spawner() } @@ -302,14 +302,14 @@ impl TaskPool { self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } - /// This allows passing an external [`ThreadSpawner`] to spawn tasks to. When you pass an external spawner - /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadSpawner`] originated from. - /// If [`None`] is passed the scope will use a [`ThreadSpawner`] that is ticked on the current thread. + /// This allows passing an external [`LocalTaskSpawner`] to spawn tasks to. When you pass an external spawner + /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`LocalTaskSpawner`] originated from. + /// If [`None`] is passed the scope will use a [`LocalTaskSpawner`] that is ticked on the current thread. /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, - external_spawner: Option>, + external_spawner: Option>, f: F, ) -> Vec where @@ -329,8 +329,8 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] fn scope_with_executor_inner<'env, F, T>( &self, - external_spawner: ThreadSpawner<'env>, - scope_spawner: ThreadSpawner<'env>, + external_spawner: LocalTaskSpawner<'env>, + scope_spawner: LocalTaskSpawner<'env>, f: F, ) -> Vec where @@ -446,8 +446,8 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope Executor<'scope>, - external_spawner: ThreadSpawner<'scope>, - scope_spawner: ThreadSpawner<'scope>, + external_spawner: LocalTaskSpawner<'scope>, + scope_spawner: LocalTaskSpawner<'scope>, spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -474,7 +474,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requires 'static Futures" + reason = "LocalTaskSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of @@ -496,7 +496,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requires 'static Futures" + reason = "LocalTaskSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread of the external thread executor. /// This is typically the main thread. The scope *must* outlive From 5373e7c9fd3b2f59291d748021106894ece7bc91 Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 31 Aug 2025 22:41:30 -0700 Subject: [PATCH 67/68] Fix macro --- crates/bevy_tasks/src/bevy_executor.rs | 80 +++++++++++++------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 0806a0d53ccb4..0be228ec2e09c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -831,52 +831,50 @@ impl Runner<'_> { } crate::cfg::multi_threaded! { - if { - // Try the local queue. - if let Ok(r) = self.local_state.stealable_queue.pop() { - return Some(r); - } + // Try the local queue. + if let Ok(r) = self.local_state.stealable_queue.pop() { + return Some(r); + } - // Try stealing from the global queue. - if let Ok(r) = self.state.queue.pop() { - steal(&self.state.queue, &self.local_state.stealable_queue); - return Some(r); - } + // Try stealing from the global queue. + if let Ok(r) = self.state.queue.pop() { + steal(&self.state.queue, &self.local_state.stealable_queue); + return Some(r); + } - // Try stealing from other runners. - if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { - // Pick a random starting point in the iterator list and rotate the list. - let n = stealer_queues.len(); - let start = _rng.usize(..n); - let iter = stealer_queues - .iter() - .chain(stealer_queues.iter()) - .skip(start) - .take(n); - - // Remove this runner's local queue. - let iter = - iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); - - // Try stealing from each local queue in the list. - for local in iter { - steal(*local, &self.local_state.stealable_queue); - if let Ok(r) = self.local_state.stealable_queue.pop() { - return Some(r); - } + // Try stealing from other runners. + if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { + // Pick a random starting point in the iterator list and rotate the list. + let n = stealer_queues.len(); + let start = _rng.usize(..n); + let iter = stealer_queues + .iter() + .chain(stealer_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(*local, &self.local_state.stealable_queue); + if let Ok(r) = self.local_state.stealable_queue.pop() { + return Some(r); } } + } - if let Ok(r) = self.local_state.thread_locked_queue.pop() { - // Do not steal from this queue. If other threads steal - // from this current thread, the task will be moved. - // - // Instead, flush all queued tasks into the local queue to - // minimize the effort required to scan for these tasks. - flush_to_local(&self.local_state.thread_locked_queue, tls); - return Some(r); - } - } else {} + if let Ok(r) = self.local_state.thread_locked_queue.pop() { + // Do not steal from this queue. If other threads steal + // from this current thread, the task will be moved. + // + // Instead, flush all queued tasks into the local queue to + // minimize the effort required to scan for these tasks. + flush_to_local(&self.local_state.thread_locked_queue, tls); + return Some(r); + } } None From 9c35b818424227ef04a67b3773b6e7edcf091c77 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 3 Sep 2025 22:55:38 -0700 Subject: [PATCH 68/68] Spin loop hints --- crates/bevy_app/src/app.rs | 4 +++- crates/bevy_app/src/schedule_runner.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index d0d2da8328a27..1d08fedcca76e 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -1401,7 +1401,9 @@ impl Plugin for HokeyPokey { type RunnerFn = Box AppExit>; fn run_once(mut app: App) -> AppExit { - while app.plugins_state() == PluginsState::Adding {} + while app.plugins_state() == PluginsState::Adding { + core::hint::spin_loop(); + } app.finish(); app.cleanup(); diff --git a/crates/bevy_app/src/schedule_runner.rs b/crates/bevy_app/src/schedule_runner.rs index c69c79c96875d..7e987cfe58be1 100644 --- a/crates/bevy_app/src/schedule_runner.rs +++ b/crates/bevy_app/src/schedule_runner.rs @@ -76,7 +76,9 @@ impl Plugin for ScheduleRunnerPlugin { app.set_runner(move |mut app: App| { let plugins_state = app.plugins_state(); if plugins_state != PluginsState::Cleaned { - while app.plugins_state() == PluginsState::Adding {} + while app.plugins_state() == PluginsState::Adding { + core::hint::spin_loop(); + } app.finish(); app.cleanup(); }