From 36916903e615a78114fc552bdfe52cb80d5aa51c Mon Sep 17 00:00:00 2001 From: James Liu Date: Fri, 25 Jul 2025 17:50:29 -0700 Subject: [PATCH 01/68] Inline async_executor --- Cargo.toml | 4 - crates/bevy_a11y/Cargo.toml | 2 +- crates/bevy_asset/Cargo.toml | 4 +- crates/bevy_ecs/Cargo.toml | 8 +- crates/bevy_input/Cargo.toml | 2 +- crates/bevy_input_focus/Cargo.toml | 2 +- crates/bevy_internal/Cargo.toml | 9 - crates/bevy_tasks/Cargo.toml | 24 +- crates/bevy_tasks/src/async_executor.rs | 1218 +++++++++++++++++++++++ crates/bevy_tasks/src/executor.rs | 6 +- crates/bevy_tasks/src/lib.rs | 4 +- crates/bevy_transform/Cargo.toml | 8 +- 12 files changed, 1244 insertions(+), 47 deletions(-) create mode 100644 crates/bevy_tasks/src/async_executor.rs diff --git a/Cargo.toml b/Cargo.toml index 4b4f00b090393..6bcb821a23f20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,7 +125,6 @@ unused_qualifications = "warn" [features] default = [ "std", - "async_executor", "android-game-activity", "android_shared_stdcxx", "animation", @@ -554,9 +553,6 @@ custom_cursor = ["bevy_internal/custom_cursor"] # Experimental support for nodes that are ignored for UI layouting ghost_nodes = ["bevy_internal/ghost_nodes"] -# Uses `async-executor` as a task execution backend. -async_executor = ["std", "bevy_internal/async_executor"] - # Allows access to the `std` crate. std = ["bevy_internal/std"] diff --git a/crates/bevy_a11y/Cargo.toml b/crates/bevy_a11y/Cargo.toml index 262b8e5b823fe..8c1ed0438e66f 100644 --- a/crates/bevy_a11y/Cargo.toml +++ b/crates/bevy_a11y/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy", "accessibility", "a11y"] [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor"] +default = ["std", "bevy_reflect"] # Functionality diff --git a/crates/bevy_asset/Cargo.toml b/crates/bevy_asset/Cargo.toml index edf8986130a00..dc66b3e08a4fc 100644 --- a/crates/bevy_asset/Cargo.toml +++ b/crates/bevy_asset/Cargo.toml @@ -27,9 +27,7 @@ bevy_ecs = { path = "../bevy_ecs", version = "0.17.0-dev", default-features = fa bevy_reflect = { path = "../bevy_reflect", version = "0.17.0-dev", default-features = false, features = [ "uuid", ] } -bevy_tasks = { path = "../bevy_tasks", version = "0.17.0-dev", default-features = false, features = [ - "async_executor", -] } +bevy_tasks = { path = "../bevy_tasks", version = "0.17.0-dev", default-features = false } bevy_utils = { path = "../bevy_utils", version = "0.17.0-dev", default-features = false } bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ "std", diff --git a/crates/bevy_ecs/Cargo.toml b/crates/bevy_ecs/Cargo.toml index f0f9b782afff2..2c291677d5daf 100644 --- a/crates/bevy_ecs/Cargo.toml +++ b/crates/bevy_ecs/Cargo.toml @@ -11,7 +11,7 @@ categories = ["game-engines", "data-structures"] rust-version = "1.86.0" [features] -default = ["std", "bevy_reflect", "async_executor", "backtrace"] +default = ["std", "bevy_reflect", "backtrace"] # Functionality @@ -48,12 +48,6 @@ bevy_debug_stepping = [] ## This will often provide more detailed error messages. track_location = [] -# Executor Backend - -## Uses `async-executor` as a task execution backend. -## This backend is incompatible with `no_std` targets. -async_executor = ["std", "bevy_tasks/async_executor"] - # Platform Compatibility ## Allows access to the `std` crate. Enabling this feature will prevent compilation diff --git a/crates/bevy_input/Cargo.toml b/crates/bevy_input/Cargo.toml index c32a87a52d9dc..2fe037a58fba4 100644 --- a/crates/bevy_input/Cargo.toml +++ b/crates/bevy_input/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy"] [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor", "smol_str"] +default = ["std", "bevy_reflect", "smol_str"] # Functionality diff --git a/crates/bevy_input_focus/Cargo.toml b/crates/bevy_input_focus/Cargo.toml index 60b824258df31..2cd9d18771b8c 100644 --- a/crates/bevy_input_focus/Cargo.toml +++ b/crates/bevy_input_focus/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["bevy"] rust-version = "1.85.0" [features] -default = ["std", "bevy_reflect", "bevy_ecs/async_executor"] +default = ["std", "bevy_reflect"] # Functionality diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index 81fcb61133085..683de2fbd4137 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -356,15 +356,6 @@ libm = [ "bevy_window?/libm", ] -# Uses `async-executor` as a task execution backend. -# This backend is incompatible with `no_std` targets. -async_executor = [ - "std", - "bevy_tasks/async_executor", - "bevy_ecs/async_executor", - "bevy_transform/async_executor", -] - # Enables use of browser APIs. # Note this is currently only applicable on `wasm32` architectures. web = [ diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index e28d7fc88d319..1c43f917a6357 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" keywords = ["bevy"] [features] -default = ["std", "async_executor"] +default = ["std"] # Functionality @@ -19,19 +19,23 @@ multi_threaded = [ "std", "dep:async-channel", "dep:concurrent-queue", - "async_executor", + "dep:fastrand", ] -## Uses `async-executor` as a task execution backend. -## This backend is incompatible with `no_std` targets. -async_executor = ["std", "dep:async-executor"] - # Platform Compatibility ## Allows access to the `std` crate. Enabling this feature will prevent compilation ## on `no_std` targets, but provides access to certain additional features on ## supported platforms. -std = ["futures-lite/std", "async-task/std", "bevy_platform/std"] +std = [ + "futures-lite/std", + "async-task/std", + "bevy_platform/std", + "fastrand/std", + "dep:slab", + "dep:concurrent-queue", + "dep:pin-project-lite", +] ## `critical-section` provides the building blocks for synchronization primitives ## on all platforms, including `no_std`. @@ -42,7 +46,6 @@ critical-section = ["bevy_platform/critical-section"] web = [ "bevy_platform/web", "dep:wasm-bindgen-futures", - "dep:pin-project", "dep:futures-channel", ] @@ -60,7 +63,9 @@ derive_more = { version = "2", default-features = false, features = [ "deref_mut", ] } cfg-if = "1.0.0" -async-executor = { version = "1.11", optional = true } +slab = { version = "0.4", optional = true } +pin-project-lite = { version = "0.2", optional = true } +fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } concurrent-queue = { version = "2.0.0", optional = true } @@ -71,7 +76,6 @@ crossbeam-queue = { version = "0.3", default-features = false, features = [ [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = { version = "0.4", optional = true } -pin-project = { version = "1", optional = true } futures-channel = { version = "0.3", optional = true } [target.'cfg(not(all(target_has_atomic = "8", target_has_atomic = "16", target_has_atomic = "32", target_has_atomic = "64", target_has_atomic = "ptr")))'.dependencies] diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs new file mode 100644 index 0000000000000..1aad2a6c133ad --- /dev/null +++ b/crates/bevy_tasks/src/async_executor.rs @@ -0,0 +1,1218 @@ +use std::fmt; +use std::marker::PhantomData; +use std::panic::{RefUnwindSafe, UnwindSafe}; +use std::pin::Pin; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; +use std::task::{Context, Poll, Waker}; + +use async_task::{Builder, Runnable}; +use concurrent_queue::ConcurrentQueue; +use futures_lite::{future, prelude::*}; +use pin_project_lite::pin_project; +use bevy_platform::prelude::Vec; +use slab::Slab; + +/// An async executor. +/// +/// # Examples +/// +/// A multi-threaded executor: +/// +/// ``` +/// use async_channel::unbounded; +/// use async_executor::Executor; +/// use easy_parallel::Parallel; +/// use futures_lite::future; +/// +/// let ex = Executor::new(); +/// let (signal, shutdown) = unbounded::<()>(); +/// +/// Parallel::new() +/// // Run four executor threads. +/// .each(0..4, |_| future::block_on(ex.run(shutdown.recv()))) +/// // Run the main future on the current thread. +/// .finish(|| future::block_on(async { +/// println!("Hello world!"); +/// drop(signal); +/// })); +/// ``` +pub struct Executor<'a> { + /// The executor state. + state: AtomicPtr, + + /// Makes the `'a` lifetime invariant. + _marker: PhantomData>, +} + +#[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" +)] +// SAFETY: Executor stores no thread local state that can be accessed via other thread. +unsafe impl Send for Executor<'_> {} +#[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" +)] +// SAFETY: Executor internally synchronizes all of it's operations internally. +unsafe impl Sync for Executor<'_> {} + +impl UnwindSafe for Executor<'_> {} +impl RefUnwindSafe for Executor<'_> {} + +impl fmt::Debug for Executor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_executor(self, "Executor", f) + } +} + +impl<'a> Executor<'a> { + /// Creates a new executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// ``` + pub const fn new() -> Executor<'a> { + Executor { + state: AtomicPtr::new(std::ptr::null_mut()), + _marker: PhantomData, + } + } + + /// Returns `true` if there are no unfinished tasks. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// assert!(ex.is_empty()); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(!ex.is_empty()); + /// + /// assert!(ex.try_tick()); + /// assert!(ex.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.state().active().is_empty() + } + + /// Spawns a task onto the executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// ``` + pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + let mut active = self.state().active(); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: `T` and the future are `Send`. + unsafe { self.spawn_inner(future, &mut active) } + } + + /// Spawns many tasks onto the executor. + /// + /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and + /// spawns all of the tasks in one go. With large amounts of tasks this can improve + /// contention. + /// + /// For very large numbers of tasks the lock is occasionally dropped and re-acquired to + /// prevent runner thread starvation. It is assumed that the iterator provided does not + /// block; blocking iterators can lock up the internal mutex and therefore the entire + /// executor. + /// + /// ## Example + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::{stream, prelude::*}; + /// use std::future::ready; + /// + /// # futures_lite::future::block_on(async { + /// let mut ex = Executor::new(); + /// + /// let futures = [ + /// ready(1), + /// ready(2), + /// ready(3) + /// ]; + /// + /// // Spawn all of the futures onto the executor at once. + /// let mut tasks = vec![]; + /// ex.spawn_many(futures, &mut tasks); + /// + /// // Await all of them. + /// let results = ex.run(async move { + /// stream::iter(tasks).then(|x| x).collect::>().await + /// }).await; + /// assert_eq!(results, [1, 2, 3]); + /// # }); + /// ``` + /// + /// [`spawn`]: Executor::spawn + pub fn spawn_many + Send + 'a>( + &self, + futures: impl IntoIterator, + handles: &mut impl Extend>, + ) { + let mut active = Some(self.state().active()); + + // Convert the futures into tasks. + let tasks = futures.into_iter().enumerate().map(move |(i, future)| { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: `T` and the future are `Send`. + let task = unsafe { self.spawn_inner(future, active.as_mut().unwrap()) }; + + // Yield the lock every once in a while to ease contention. + if i.wrapping_sub(1) % 500 == 0 { + drop(active.take()); + active = Some(self.state().active()); + } + + task + }); + + // Push the tasks to the user's collection. + handles.extend(tasks); + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + /// Spawn a future while holding the inner lock. + /// + /// # Safety + /// + /// If this is an `Executor`, `F` and `T` must be `Send`. + unsafe fn spawn_inner( + &self, + future: impl Future + 'a, + active: &mut Slab, + ) -> Task { + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state_as_arc(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()); + entry.insert(runnable.waker()); + + runnable.schedule(); + task + } + + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// + /// let ex = Executor::new(); + /// assert!(!ex.try_tick()); // no tasks to run + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(ex.try_tick()); // a task was found + /// ``` + pub fn try_tick(&self) -> bool { + self.state().try_tick() + } + + /// Runs a single task. + /// + /// Running a task means simply polling its future once. + /// + /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::future; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// future::block_on(ex.tick()); // runs the task + /// ``` + pub async fn tick(&self) { + self.state().tick().await; + } + + /// Runs the executor until the given future completes. + /// + /// # Examples + /// + /// ``` + /// use async_executor::Executor; + /// use futures_lite::future; + /// + /// let ex = Executor::new(); + /// + /// let task = ex.spawn(async { 1 + 2 }); + /// let res = future::block_on(ex.run(async { task.await * 2 })); + /// + /// assert_eq!(res, 6); + /// ``` + pub async fn run(&self, future: impl Future) -> T { + self.state().run(future).await + } + + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { + let state = self.state_as_arc(); + + // TODO: If possible, push into the current local queue and notify the ticker. + move |runnable| { + state.queue.push(runnable).unwrap(); + state.notify(); + } + } + + /// Returns a pointer to the inner state. + #[inline] + fn state_ptr(&self) -> *const State { + #[cold] + fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { + let state = Arc::new(State::new()); + // TODO: Switch this to use cast_mut once the MSRV can be bumped past 1.65 + let ptr = Arc::into_raw(state) as *mut State; + if let Err(actual) = atomic_ptr.compare_exchange( + std::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This was just created from Arc::into_raw. + drop(unsafe { Arc::from_raw(ptr) }); + actual + } else { + ptr + } + } + + let mut ptr = self.state.load(Ordering::Acquire); + if ptr.is_null() { + ptr = alloc_state(&self.state); + } + ptr + } + + /// Returns a reference to the inner state. + #[inline] + fn state(&self) -> &State { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: So long as an Executor lives, it's state pointer will always be valid + // when accessed through state_ptr. + unsafe { &*self.state_ptr() } + } + + // Clones the inner state Arc + #[inline] + fn state_as_arc(&self) -> Arc { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: So long as an Executor lives, it's state pointer will always be a valid + // Arc when accessed through state_ptr. + let arc = unsafe { Arc::from_raw(self.state_ptr()) }; + let clone = arc.clone(); + std::mem::forget(arc); + clone + } +} + +impl Drop for Executor<'_> { + fn drop(&mut self) { + let ptr = *self.state.get_mut(); + if ptr.is_null() { + return; + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: As ptr is not null, it was allocated via Arc::new and converted + // via Arc::into_raw in state_ptr. + let state = unsafe { Arc::from_raw(ptr) }; + + let mut active = state.active(); + for w in active.drain() { + w.wake(); + } + drop(active); + + while state.queue.pop().is_ok() {} + } +} + +impl<'a> Default for Executor<'a> { + fn default() -> Executor<'a> { + Executor::new() + } +} + +/// A thread-local executor. +/// +/// The executor can only be run on the thread that created it. +/// +/// # Examples +/// +/// ``` +/// use async_executor::LocalExecutor; +/// use futures_lite::future; +/// +/// let local_ex = LocalExecutor::new(); +/// +/// future::block_on(local_ex.run(async { +/// println!("Hello world!"); +/// })); +/// ``` +pub struct LocalExecutor<'a> { + /// The inner executor. + inner: Executor<'a>, + + /// Makes the type `!Send` and `!Sync`. + _marker: PhantomData>, +} + +impl UnwindSafe for LocalExecutor<'_> {} +impl RefUnwindSafe for LocalExecutor<'_> {} + +impl fmt::Debug for LocalExecutor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_executor(&self.inner, "LocalExecutor", f) + } +} + +impl<'a> LocalExecutor<'a> { + /// Creates a single-threaded executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// ``` + pub const fn new() -> LocalExecutor<'a> { + LocalExecutor { + inner: Executor::new(), + _marker: PhantomData, + } + } + + /// Returns `true` if there are no unfinished tasks. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// assert!(local_ex.is_empty()); + /// + /// let task = local_ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(!local_ex.is_empty()); + /// + /// assert!(local_ex.try_tick()); + /// assert!(local_ex.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.inner().is_empty() + } + + /// Spawns a task onto the executor. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let local_ex = LocalExecutor::new(); + /// + /// let task = local_ex.spawn(async { + /// println!("Hello world"); + /// }); + /// ``` + pub fn spawn(&self, future: impl Future + 'a) -> Task { + let mut active = self.inner().state().active(); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This executor is not thread safe, so the future and its result + // cannot be sent to another thread. + unsafe { self.inner().spawn_inner(future, &mut active) } + } + + /// Spawns many tasks onto the executor. + /// + /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and + /// spawns all of the tasks in one go. With large amounts of tasks this can improve + /// contention. + /// + /// It is assumed that the iterator provided does not block; blocking iterators can lock up + /// the internal mutex and therefore the entire executor. Unlike [`Executor::spawn`], the + /// mutex is not released, as there are no other threads that can poll this executor. + /// + /// ## Example + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::{stream, prelude::*}; + /// use std::future::ready; + /// + /// # futures_lite::future::block_on(async { + /// let mut ex = LocalExecutor::new(); + /// + /// let futures = [ + /// ready(1), + /// ready(2), + /// ready(3) + /// ]; + /// + /// // Spawn all of the futures onto the executor at once. + /// let mut tasks = vec![]; + /// ex.spawn_many(futures, &mut tasks); + /// + /// // Await all of them. + /// let results = ex.run(async move { + /// stream::iter(tasks).then(|x| x).collect::>().await + /// }).await; + /// assert_eq!(results, [1, 2, 3]); + /// # }); + /// ``` + /// + /// [`spawn`]: LocalExecutor::spawn + /// [`Executor::spawn_many`]: Executor::spawn_many + pub fn spawn_many + 'a>( + &self, + futures: impl IntoIterator, + handles: &mut impl Extend>, + ) { + let mut active = self.inner().state().active(); + + // Convert all of the futures to tasks. + let tasks = futures.into_iter().map(|future| { + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: This executor is not thread safe, so the future and its result + // cannot be sent to another thread. + unsafe { self.inner().spawn_inner(future, &mut active) } + + // As only one thread can spawn or poll tasks at a time, there is no need + // to release lock contention here. + }); + + // Push them to the user's collection. + handles.extend(tasks); + } + + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// + /// let ex = LocalExecutor::new(); + /// assert!(!ex.try_tick()); // no tasks to run + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// assert!(ex.try_tick()); // a task was found + /// ``` + pub fn try_tick(&self) -> bool { + self.inner().try_tick() + } + + /// Runs a single task. + /// + /// Running a task means simply polling its future once. + /// + /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::future; + /// + /// let ex = LocalExecutor::new(); + /// + /// let task = ex.spawn(async { + /// println!("Hello world"); + /// }); + /// future::block_on(ex.tick()); // runs the task + /// ``` + pub async fn tick(&self) { + self.inner().tick().await + } + + /// Runs the executor until the given future completes. + /// + /// # Examples + /// + /// ``` + /// use async_executor::LocalExecutor; + /// use futures_lite::future; + /// + /// let local_ex = LocalExecutor::new(); + /// + /// let task = local_ex.spawn(async { 1 + 2 }); + /// let res = future::block_on(local_ex.run(async { task.await * 2 })); + /// + /// assert_eq!(res, 6); + /// ``` + pub async fn run(&self, future: impl Future) -> T { + self.inner().run(future).await + } + + /// Returns a reference to the inner executor. + fn inner(&self) -> &Executor<'a> { + &self.inner + } +} + +impl<'a> Default for LocalExecutor<'a> { + fn default() -> LocalExecutor<'a> { + LocalExecutor::new() + } +} + +/// The state of a executor. +struct State { + /// The global queue. + queue: ConcurrentQueue, + + /// Local queues created by runners. + local_queues: RwLock>>>, + + /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. + notified: AtomicBool, + + /// A list of sleeping tickers. + sleepers: Mutex, + + /// Currently active tasks. + active: Mutex>, +} + +impl State { + /// Creates state for a new executor. + const fn new() -> State { + State { + queue: ConcurrentQueue::unbounded(), + local_queues: RwLock::new(Vec::new()), + notified: AtomicBool::new(true), + sleepers: Mutex::new(Sleepers { + count: 0, + wakers: Vec::new(), + free_ids: Vec::new(), + }), + active: Mutex::new(Slab::new()), + } + } + + /// Returns a reference to currently active tasks. + fn active(&self) -> MutexGuard<'_, Slab> { + self.active.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Notifies a sleeping ticker. + #[inline] + fn notify(&self) { + if self + .notified + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + let waker = self.sleepers.lock().unwrap().notify(); + if let Some(w) = waker { + w.wake(); + } + } + } + + pub(crate) fn try_tick(&self) -> bool { + match self.queue.pop() { + Err(_) => false, + Ok(runnable) => { + // Notify another ticker now to pick up where this ticker left off, just in case + // running the task takes a long time. + self.notify(); + + // Run the task. + runnable.run(); + true + } + } + } + + pub(crate) async fn tick(&self) { + let runnable = Ticker::new(self).runnable().await; + runnable.run(); + } + + pub async fn run(&self, future: impl Future) -> T { + let mut runner = Runner::new(self); + let mut rng = fastrand::Rng::new(); + + // A future that runs tasks forever. + let run_forever = async { + loop { + for _ in 0..200 { + let runnable = runner.runnable(&mut rng).await; + runnable.run(); + } + future::yield_now().await; + } + }; + + // Run `future` and `run_forever` concurrently until `future` completes. + future.or(run_forever).await + } +} + +/// A list of sleeping tickers. +struct Sleepers { + /// Number of sleeping tickers (both notified and unnotified). + count: usize, + + /// IDs and wakers of sleeping unnotified tickers. + /// + /// A sleeping ticker is notified when its waker is missing from this list. + wakers: Vec<(usize, Waker)>, + + /// Reclaimed IDs. + free_ids: Vec, +} + +impl Sleepers { + /// Inserts a new sleeping ticker. + fn insert(&mut self, waker: &Waker) -> usize { + let id = match self.free_ids.pop() { + Some(id) => id, + None => self.count + 1, + }; + self.count += 1; + self.wakers.push((id, waker.clone())); + id + } + + /// Re-inserts a sleeping ticker's waker if it was notified. + /// + /// Returns `true` if the ticker was notified. + fn update(&mut self, id: usize, waker: &Waker) -> bool { + for item in &mut self.wakers { + if item.0 == id { + item.1.clone_from(waker); + return false; + } + } + + self.wakers.push((id, waker.clone())); + true + } + + /// Removes a previously inserted sleeping ticker. + /// + /// Returns `true` if the ticker was notified. + fn remove(&mut self, id: usize) -> bool { + self.count -= 1; + self.free_ids.push(id); + + for i in (0..self.wakers.len()).rev() { + if self.wakers[i].0 == id { + self.wakers.remove(i); + return false; + } + } + true + } + + /// Returns `true` if a sleeping ticker is notified or no tickers are sleeping. + fn is_notified(&self) -> bool { + self.count == 0 || self.count > self.wakers.len() + } + + /// Returns notification waker for a sleeping ticker. + /// + /// If a ticker was notified already or there are no tickers, `None` will be returned. + fn notify(&mut self) -> Option { + if self.wakers.len() == self.count { + self.wakers.pop().map(|item| item.1) + } else { + None + } + } +} + +/// Runs task one by one. +struct Ticker<'a> { + /// The executor state. + state: &'a State, + + /// Set to a non-zero sleeper ID when in sleeping state. + /// + /// States a ticker can be in: + /// 1) Woken. + /// 2a) Sleeping and unnotified. + /// 2b) Sleeping and notified. + sleeping: usize, +} + +impl Ticker<'_> { + /// Creates a ticker. + fn new(state: &State) -> Ticker<'_> { + Ticker { state, sleeping: 0 } + } + + /// Moves the ticker into sleeping and unnotified state. + /// + /// Returns `false` if the ticker was already sleeping and unnotified. + fn sleep(&mut self, waker: &Waker) -> bool { + let mut sleepers = self.state.sleepers.lock().unwrap(); + + match self.sleeping { + // Move to sleeping state. + 0 => { + self.sleeping = sleepers.insert(waker); + } + + // Already sleeping, check if notified. + id => { + if !sleepers.update(id, waker) { + return false; + } + } + } + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + + true + } + + /// Moves the ticker into woken state. + fn wake(&mut self) { + if self.sleeping != 0 { + let mut sleepers = self.state.sleepers.lock().unwrap(); + sleepers.remove(self.sleeping); + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + } + self.sleeping = 0; + } + + /// Waits for the next runnable task to run. + async fn runnable(&mut self) -> Runnable { + self.runnable_with(|| self.state.queue.pop().ok()).await + } + + /// Waits for the next runnable task to run, given a function that searches for a task. + async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { + future::poll_fn(|cx| { + loop { + match search() { + None => { + // Move to sleeping and unnotified state. + if !self.sleep(cx.waker()) { + // If already sleeping and unnotified, return. + return Poll::Pending; + } + } + Some(r) => { + // Wake up. + self.wake(); + + // Notify another ticker now to pick up where this ticker left off, just in + // case running the task takes a long time. + self.state.notify(); + + return Poll::Ready(r); + } + } + } + }) + .await + } +} + +impl Drop for Ticker<'_> { + fn drop(&mut self) { + // If this ticker is in sleeping state, it must be removed from the sleepers list. + if self.sleeping != 0 { + let mut sleepers = self.state.sleepers.lock().unwrap(); + let notified = sleepers.remove(self.sleeping); + + self.state + .notified + .store(sleepers.is_notified(), Ordering::Release); + + // If this ticker was notified, then notify another ticker. + if notified { + drop(sleepers); + self.state.notify(); + } + } + } +} + +/// A worker in a work-stealing executor. +/// +/// This is just a ticker that also has an associated local queue for improved cache locality. +struct Runner<'a> { + /// The executor state. + state: &'a State, + + /// Inner ticker. + ticker: Ticker<'a>, + + /// The local queue. + local: Arc>, + + /// Bumped every time a runnable task is found. + ticks: usize, +} + +impl Runner<'_> { + /// Creates a runner and registers it in the executor state. + fn new(state: &State) -> Runner<'_> { + let runner = Runner { + state, + ticker: Ticker::new(state), + local: Arc::new(ConcurrentQueue::bounded(512)), + ticks: 0, + }; + state + .local_queues + .write() + .unwrap() + .push(runner.local.clone()); + runner + } + + /// Waits for the next runnable task to run. + async fn runnable(&mut self, rng: &mut fastrand::Rng) -> Runnable { + let runnable = self + .ticker + .runnable_with(|| { + // Try the local queue. + if let Ok(r) = self.local.pop() { + return Some(r); + } + + // Try stealing from the global queue. + if let Ok(r) = self.state.queue.pop() { + steal(&self.state.queue, &self.local); + return Some(r); + } + + // Try stealing from other runners. + let local_queues = self.state.local_queues.read().unwrap(); + + // Pick a random starting point in the iterator list and rotate the list. + let n = local_queues.len(); + let start = rng.usize(..n); + let iter = local_queues + .iter() + .chain(local_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = iter.filter(|local| !Arc::ptr_eq(local, &self.local)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(local, &self.local); + if let Ok(r) = self.local.pop() { + return Some(r); + } + } + + None + }) + .await; + + // Bump the tick counter. + self.ticks = self.ticks.wrapping_add(1); + + if self.ticks % 64 == 0 { + // Steal tasks from the global queue to ensure fair task scheduling. + steal(&self.state.queue, &self.local); + } + + runnable + } +} + +impl Drop for Runner<'_> { + fn drop(&mut self) { + // Remove the local queue. + self.state + .local_queues + .write() + .unwrap() + .retain(|local| !Arc::ptr_eq(local, &self.local)); + + // Re-schedule remaining tasks in the local queue. + while let Ok(r) = self.local.pop() { + r.schedule(); + } + } +} + +/// Steals some items from one queue into another. +fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { + // Half of `src`'s length rounded up. + let mut count = (src.len() + 1) / 2; + + if count > 0 { + // Don't steal more than fits into the queue. + if let Some(cap) = dest.capacity() { + count = count.min(cap - dest.len()); + } + + // Steal tasks. + for _ in 0..count { + if let Ok(t) = src.pop() { + assert!(dest.push(t).is_ok()); + } else { + break; + } + } + } +} + +/// Debug implementation for `Executor` and `LocalExecutor`. +fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Get a reference to the state. + let ptr = executor.state.load(Ordering::Acquire); + if ptr.is_null() { + // The executor has not been initialized. + struct Uninitialized; + + impl fmt::Debug for Uninitialized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + + return f.debug_tuple(name).field(&Uninitialized).finish(); + } + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // SAFETY: If the state pointer is not null, it must have been + // allocated properly by Arc::new and converted via Arc::into_raw + // in state_ptr. + let state = unsafe { &*ptr }; + + debug_state(state, name, f) +} + +/// Debug implementation for `Executor` and `LocalExecutor`. +fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// Debug wrapper for the number of active tasks. + struct ActiveTasks<'a>(&'a Mutex>); + + impl fmt::Debug for ActiveTasks<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_lock() { + Ok(lock) => fmt::Debug::fmt(&lock.len(), f), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(err)) => fmt::Debug::fmt(&err.into_inner().len(), f), + } + } + } + + /// Debug wrapper for the local runners. + struct LocalRunners<'a>(&'a RwLock>>>); + + impl fmt::Debug for LocalRunners<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_read() { + Ok(lock) => f + .debug_list() + .entries(lock.iter().map(|queue| queue.len())) + .finish(), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(_)) => f.write_str(""), + } + } + } + + /// Debug wrapper for the sleepers. + struct SleepCount<'a>(&'a Mutex); + + impl fmt::Debug for SleepCount<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.try_lock() { + Ok(lock) => fmt::Debug::fmt(&lock.count, f), + Err(TryLockError::WouldBlock) => f.write_str(""), + Err(TryLockError::Poisoned(_)) => f.write_str(""), + } + } + } + + f.debug_struct(name) + .field("active", &ActiveTasks(&state.active)) + .field("global_tasks", &state.queue.len()) + .field("local_runners", &LocalRunners(&state.local_queues)) + .field("sleepers", &SleepCount(&state.sleepers)) + .finish() +} + +/// Runs a closure when dropped. +struct CallOnDrop(F); + +impl Drop for CallOnDrop { + fn drop(&mut self) { + (self.0)(); + } +} + +pin_project! { + /// A wrapper around a future, running a closure when dropped. + struct AsyncCallOnDrop { + #[pin] + future: Fut, + cleanup: CallOnDrop, + } +} + +impl AsyncCallOnDrop { + fn new(future: Fut, cleanup: Cleanup) -> Self { + Self { + future, + cleanup: CallOnDrop(cleanup), + } + } +} + +impl Future for AsyncCallOnDrop { + type Output = Fut::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.project().future.poll(cx) + } +} + +fn _ensure_send_and_sync() { + use futures_lite::future::pending; + + fn is_send(_: T) {} + fn is_sync(_: T) {} + fn is_static(_: T) {} + + is_send::>(Executor::new()); + is_sync::>(Executor::new()); + + let ex = Executor::new(); + is_send(ex.run(pending::<()>())); + is_sync(ex.run(pending::<()>())); + is_send(ex.tick()); + is_sync(ex.tick()); + is_send(ex.schedule()); + is_sync(ex.schedule()); + is_static(ex.schedule()); + + /// ```compile_fail + /// use async_executor::LocalExecutor; + /// use futures_lite::future::pending; + /// + /// fn is_send(_: T) {} + /// fn is_sync(_: T) {} + /// + /// is_send::>(LocalExecutor::new()); + /// is_sync::>(LocalExecutor::new()); + /// + /// let ex = LocalExecutor::new(); + /// is_send(ex.run(pending::<()>())); + /// is_sync(ex.run(pending::<()>())); + /// is_send(ex.tick()); + /// is_sync(ex.tick()); + /// ``` + fn _negative_test() {} +} diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 01bbe4a669258..deb8f2fca3cf7 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -15,9 +15,9 @@ use core::{ use derive_more::{Deref, DerefMut}; cfg_if::cfg_if! { - if #[cfg(feature = "async_executor")] { - type ExecutorInner<'a> = async_executor::Executor<'a>; - type LocalExecutorInner<'a> = async_executor::LocalExecutor<'a>; + if #[cfg(feature = "std")] { + type ExecutorInner<'a> = crate::async_executor::Executor<'a>; + type LocalExecutorInner<'a> = crate::async_executor::LocalExecutor<'a>; } else { type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; type LocalExecutorInner<'a> = crate::edge_executor::LocalExecutor<'a, 64>; diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 66899ef36f095..4f23e7954bf36 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -42,7 +42,9 @@ pub type BoxedFuture<'a, T> = core::pin::Pin Date: Sat, 26 Jul 2025 15:18:43 -0700 Subject: [PATCH 02/68] Merge LocalExecutor into the Executor implementation --- crates/bevy_tasks/Cargo.toml | 2 + crates/bevy_tasks/src/async_executor.rs | 608 +++++------------- crates/bevy_tasks/src/executor.rs | 32 +- .../src/single_threaded_task_pool.rs | 67 +- crates/bevy_tasks/src/task_pool.rs | 55 +- crates/bevy_tasks/src/usages.rs | 26 +- 6 files changed, 200 insertions(+), 590 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 1c43f917a6357..a27379e05b08c 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -33,6 +33,7 @@ std = [ "bevy_platform/std", "fastrand/std", "dep:slab", + "dep:thread_local", "dep:concurrent-queue", "dep:pin-project-lite", ] @@ -65,6 +66,7 @@ derive_more = { version = "2", default-features = false, features = [ cfg-if = "1.0.0" slab = { version = "0.4", optional = true } pin-project-lite = { version = "0.2", optional = true } +thread_local = { version = "1.1", optional = true } fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 1aad2a6c133ad..8f862f8f0e662 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -1,43 +1,30 @@ +use std::cell::RefCell; +use std::collections::VecDeque; use std::fmt; use std::marker::PhantomData; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; -use std::rc::Rc; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; -use async_task::{Builder, Runnable}; +use async_task::{Builder, Runnable, Task}; +use bevy_platform::prelude::Vec; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; -use bevy_platform::prelude::Vec; use slab::Slab; +use thread_local::ThreadLocal; + +static LOCAL_QUEUE: ThreadLocal> = ThreadLocal::new(); + +#[derive(Default)] +struct LocalQueue { + queue: VecDeque, + active: Slab, +} /// An async executor. -/// -/// # Examples -/// -/// A multi-threaded executor: -/// -/// ``` -/// use async_channel::unbounded; -/// use async_executor::Executor; -/// use easy_parallel::Parallel; -/// use futures_lite::future; -/// -/// let ex = Executor::new(); -/// let (signal, shutdown) = unbounded::<()>(); -/// -/// Parallel::new() -/// // Run four executor threads. -/// .each(0..4, |_| future::block_on(ex.run(shutdown.recv()))) -/// // Run the main future on the current thread. -/// .finish(|| future::block_on(async { -/// println!("Hello world!"); -/// drop(signal); -/// })); -/// ``` pub struct Executor<'a> { /// The executor state. state: AtomicPtr, @@ -70,14 +57,6 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// ``` pub const fn new() -> Executor<'a> { Executor { state: AtomicPtr::new(std::ptr::null_mut()), @@ -85,141 +64,68 @@ impl<'a> Executor<'a> { } } - /// Returns `true` if there are no unfinished tasks. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// assert!(ex.is_empty()); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(!ex.is_empty()); - /// - /// assert!(ex.try_tick()); - /// assert!(ex.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.state().active().is_empty() - } - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::Executor; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { let mut active = self.state().active(); + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state_as_arc(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + #[expect( unsafe_code, reason = "unsized coercion is an unstable feature for non-std types" )] - // SAFETY: `T` and the future are `Send`. - unsafe { self.spawn_inner(future, &mut active) } - } - - /// Spawns many tasks onto the executor. - /// - /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and - /// spawns all of the tasks in one go. With large amounts of tasks this can improve - /// contention. - /// - /// For very large numbers of tasks the lock is occasionally dropped and re-acquired to - /// prevent runner thread starvation. It is assumed that the iterator provided does not - /// block; blocking iterators can lock up the internal mutex and therefore the entire - /// executor. - /// - /// ## Example - /// - /// ``` - /// use async_executor::Executor; - /// use futures_lite::{stream, prelude::*}; - /// use std::future::ready; - /// - /// # futures_lite::future::block_on(async { - /// let mut ex = Executor::new(); - /// - /// let futures = [ - /// ready(1), - /// ready(2), - /// ready(3) - /// ]; - /// - /// // Spawn all of the futures onto the executor at once. - /// let mut tasks = vec![]; - /// ex.spawn_many(futures, &mut tasks); - /// - /// // Await all of them. - /// let results = ex.run(async move { - /// stream::iter(tasks).then(|x| x).collect::>().await - /// }).await; - /// assert_eq!(results, [1, 2, 3]); - /// # }); - /// ``` - /// - /// [`spawn`]: Executor::spawn - pub fn spawn_many + Send + 'a>( - &self, - futures: impl IntoIterator, - handles: &mut impl Extend>, - ) { - let mut active = Some(self.state().active()); - - // Convert the futures into tasks. - let tasks = futures.into_iter().enumerate().map(move |(i, future)| { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: `T` and the future are `Send`. - let task = unsafe { self.spawn_inner(future, active.as_mut().unwrap()) }; - - // Yield the lock every once in a while to ease contention. - if i.wrapping_sub(1) % 500 == 0 { - drop(active.take()); - active = Some(self.state().active()); - } - - task - }); + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()) + }; + entry.insert(runnable.waker()); - // Push the tasks to the user's collection. - handles.extend(tasks); + runnable.schedule(); + task } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - /// Spawn a future while holding the inner lock. - /// - /// # Safety - /// - /// If this is an `Executor`, `F` and `T` must be `Send`. - unsafe fn spawn_inner( - &self, - future: impl Future + 'a, - active: &mut Slab, - ) -> Task { + /// Spawns a non-Send task onto the executor. + pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let entry = active.vacant_entry(); + let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + let mut local_state = local_queue.borrow_mut(); + let entry = local_state.active.vacant_entry(); let index = entry.key(); - let state = self.state_as_arc(); - let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + let future = AsyncCallOnDrop::new(future, move || { + drop(local_queue.borrow_mut().active.try_remove(index)) + }); + #[expect( + unsafe_code, + reason = "Builder::spawn_local requires a 'static lifetime" + )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -242,11 +148,15 @@ impl<'a> Executor<'a> { // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. // Therefore we do not need to worry about what is done with the // `Waker`. - let (runnable, task) = Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule()); + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule_local()) + }; entry.insert(runnable.waker()); + drop(local_state); + runnable.schedule(); task } @@ -258,7 +168,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// /// let ex = Executor::new(); /// assert!(!ex.try_tick()); // no tasks to run @@ -272,6 +182,16 @@ impl<'a> Executor<'a> { self.state().try_tick() } + pub fn try_tick_local() -> bool { + if let Some(runnable) = LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back() { + // Run the task. + runnable.run(); + true + } else { + false + } + } + /// Runs a single task. /// /// Running a task means simply polling its future once. @@ -281,7 +201,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// use futures_lite::future; /// /// let ex = Executor::new(); @@ -300,7 +220,7 @@ impl<'a> Executor<'a> { /// # Examples /// /// ``` - /// use async_executor::Executor; + /// use crate::async_executor::Executor; /// use futures_lite::future; /// /// let ex = Executor::new(); @@ -325,6 +245,16 @@ impl<'a> Executor<'a> { } } + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule_local(&self) -> impl Fn(Runnable) + 'static { + let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + // TODO: If possible, push into the current local queue and notify the ticker. + move |runnable| { + local_queue.borrow_mut().queue.push_back(runnable); + // state.notify(); + } + } + /// Returns a pointer to the inner state. #[inline] fn state_ptr(&self) -> *const State { @@ -367,7 +297,9 @@ impl<'a> Executor<'a> { )] // SAFETY: So long as an Executor lives, it's state pointer will always be valid // when accessed through state_ptr. - unsafe { &*self.state_ptr() } + unsafe { + &*self.state_ptr() + } } // Clones the inner state Arc @@ -417,243 +349,6 @@ impl<'a> Default for Executor<'a> { } } -/// A thread-local executor. -/// -/// The executor can only be run on the thread that created it. -/// -/// # Examples -/// -/// ``` -/// use async_executor::LocalExecutor; -/// use futures_lite::future; -/// -/// let local_ex = LocalExecutor::new(); -/// -/// future::block_on(local_ex.run(async { -/// println!("Hello world!"); -/// })); -/// ``` -pub struct LocalExecutor<'a> { - /// The inner executor. - inner: Executor<'a>, - - /// Makes the type `!Send` and `!Sync`. - _marker: PhantomData>, -} - -impl UnwindSafe for LocalExecutor<'_> {} -impl RefUnwindSafe for LocalExecutor<'_> {} - -impl fmt::Debug for LocalExecutor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - debug_executor(&self.inner, "LocalExecutor", f) - } -} - -impl<'a> LocalExecutor<'a> { - /// Creates a single-threaded executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// ``` - pub const fn new() -> LocalExecutor<'a> { - LocalExecutor { - inner: Executor::new(), - _marker: PhantomData, - } - } - - /// Returns `true` if there are no unfinished tasks. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// assert!(local_ex.is_empty()); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(!local_ex.is_empty()); - /// - /// assert!(local_ex.try_tick()); - /// assert!(local_ex.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.inner().is_empty() - } - - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let local_ex = LocalExecutor::new(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` - pub fn spawn(&self, future: impl Future + 'a) -> Task { - let mut active = self.inner().state().active(); - - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: This executor is not thread safe, so the future and its result - // cannot be sent to another thread. - unsafe { self.inner().spawn_inner(future, &mut active) } - } - - /// Spawns many tasks onto the executor. - /// - /// As opposed to the [`spawn`] method, this locks the executor's inner task lock once and - /// spawns all of the tasks in one go. With large amounts of tasks this can improve - /// contention. - /// - /// It is assumed that the iterator provided does not block; blocking iterators can lock up - /// the internal mutex and therefore the entire executor. Unlike [`Executor::spawn`], the - /// mutex is not released, as there are no other threads that can poll this executor. - /// - /// ## Example - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::{stream, prelude::*}; - /// use std::future::ready; - /// - /// # futures_lite::future::block_on(async { - /// let mut ex = LocalExecutor::new(); - /// - /// let futures = [ - /// ready(1), - /// ready(2), - /// ready(3) - /// ]; - /// - /// // Spawn all of the futures onto the executor at once. - /// let mut tasks = vec![]; - /// ex.spawn_many(futures, &mut tasks); - /// - /// // Await all of them. - /// let results = ex.run(async move { - /// stream::iter(tasks).then(|x| x).collect::>().await - /// }).await; - /// assert_eq!(results, [1, 2, 3]); - /// # }); - /// ``` - /// - /// [`spawn`]: LocalExecutor::spawn - /// [`Executor::spawn_many`]: Executor::spawn_many - pub fn spawn_many + 'a>( - &self, - futures: impl IntoIterator, - handles: &mut impl Extend>, - ) { - let mut active = self.inner().state().active(); - - // Convert all of the futures to tasks. - let tasks = futures.into_iter().map(|future| { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] - // SAFETY: This executor is not thread safe, so the future and its result - // cannot be sent to another thread. - unsafe { self.inner().spawn_inner(future, &mut active) } - - // As only one thread can spawn or poll tasks at a time, there is no need - // to release lock contention here. - }); - - // Push them to the user's collection. - handles.extend(tasks); - } - - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// - /// let ex = LocalExecutor::new(); - /// assert!(!ex.try_tick()); // no tasks to run - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.inner().try_tick() - } - - /// Runs a single task. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::future; - /// - /// let ex = LocalExecutor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// future::block_on(ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.inner().tick().await - } - - /// Runs the executor until the given future completes. - /// - /// # Examples - /// - /// ``` - /// use async_executor::LocalExecutor; - /// use futures_lite::future; - /// - /// let local_ex = LocalExecutor::new(); - /// - /// let task = local_ex.spawn(async { 1 + 2 }); - /// let res = future::block_on(local_ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` - pub async fn run(&self, future: impl Future) -> T { - self.inner().run(future).await - } - - /// Returns a reference to the inner executor. - fn inner(&self) -> &Executor<'a> { - &self.inner - } -} - -impl<'a> Default for LocalExecutor<'a> { - fn default() -> LocalExecutor<'a> { - LocalExecutor::new() - } -} - /// The state of a executor. struct State { /// The global queue. @@ -709,17 +404,21 @@ impl State { } pub(crate) fn try_tick(&self) -> bool { - match self.queue.pop() { - Err(_) => false, - Ok(runnable) => { - // Notify another ticker now to pick up where this ticker left off, just in case - // running the task takes a long time. - self.notify(); - - // Run the task. - runnable.run(); - true - } + let runnable = self + .queue + .pop() + .ok() + .or_else(|| LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back()); + if let Some(runnable) = runnable { + // Notify another ticker now to pick up where this ticker left off, just in case + // running the task takes a long time. + self.notify(); + + // Run the task. + runnable.run(); + true + } else { + false } } @@ -884,7 +583,15 @@ impl Ticker<'_> { /// Waits for the next runnable task to run. async fn runnable(&mut self) -> Runnable { - self.runnable_with(|| self.state.queue.pop().ok()).await + self.runnable_with(|| { + LOCAL_QUEUE + .get_or_default() + .borrow_mut() + .queue + .pop_back() + .or_else(|| self.state.queue.pop().ok()) + }) + .await } /// Waits for the next runnable task to run, given a function that searches for a task. @@ -951,6 +658,9 @@ struct Runner<'a> { /// Bumped every time a runnable task is found. ticks: usize, + + // The thread local state of the executor for the current thread. + local_state: &'static RefCell, } impl Runner<'_> { @@ -961,6 +671,7 @@ impl Runner<'_> { ticker: Ticker::new(state), local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, + local_state: LOCAL_QUEUE.get_or_default(), }; state .local_queues @@ -975,6 +686,10 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { + if let Some(r) = self.local_state.borrow_mut().queue.pop_back() { + return Some(r); + } + // Try the local queue. if let Ok(r) = self.local.pop() { return Some(r); @@ -1179,40 +894,39 @@ impl Future for AsyncCallOnDrop { } } -fn _ensure_send_and_sync() { - use futures_lite::future::pending; - - fn is_send(_: T) {} - fn is_sync(_: T) {} - fn is_static(_: T) {} - - is_send::>(Executor::new()); - is_sync::>(Executor::new()); - - let ex = Executor::new(); - is_send(ex.run(pending::<()>())); - is_sync(ex.run(pending::<()>())); - is_send(ex.tick()); - is_sync(ex.tick()); - is_send(ex.schedule()); - is_sync(ex.schedule()); - is_static(ex.schedule()); - - /// ```compile_fail - /// use async_executor::LocalExecutor; - /// use futures_lite::future::pending; - /// - /// fn is_send(_: T) {} - /// fn is_sync(_: T) {} - /// - /// is_send::>(LocalExecutor::new()); - /// is_sync::>(LocalExecutor::new()); - /// - /// let ex = LocalExecutor::new(); - /// is_send(ex.run(pending::<()>())); - /// is_sync(ex.run(pending::<()>())); - /// is_send(ex.tick()); - /// is_sync(ex.tick()); - /// ``` - fn _negative_test() {} +#[cfg(test)] +mod test { + fn _ensure_send_and_sync() { + fn is_send(_: T) {} + fn is_sync(_: T) {} + fn is_static(_: T) {} + + is_send::>(Executor::new()); + is_sync::>(Executor::new()); + + let ex = Executor::new(); + is_send(ex.tick()); + is_sync(ex.tick()); + is_send(ex.schedule()); + is_sync(ex.schedule()); + is_static(ex.schedule()); + + /// ```compile_fail + /// use crate::async_executor::LocalExecutor; + /// use futures_lite::future::pending; + /// + /// fn is_send(_: T) {} + /// fn is_sync(_: T) {} + /// + /// is_send::>(LocalExecutor::new()); + /// is_sync::>(LocalExecutor::new()); + /// + /// let ex = LocalExecutor::new(); + /// is_send(ex.run(pending::<()>())); + /// is_sync(ex.run(pending::<()>())); + /// is_send(ex.tick()); + /// is_sync(ex.tick()); + /// ``` + fn _negative_test() {} + } } diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index deb8f2fca3cf7..2c7e142a0a623 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -17,10 +17,8 @@ use derive_more::{Deref, DerefMut}; cfg_if::cfg_if! { if #[cfg(feature = "std")] { type ExecutorInner<'a> = crate::async_executor::Executor<'a>; - type LocalExecutorInner<'a> = crate::async_executor::LocalExecutor<'a>; } else { type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; - type LocalExecutorInner<'a> = crate::edge_executor::LocalExecutor<'a, 64>; } } @@ -36,16 +34,6 @@ pub use async_task::FallibleTask; #[derive(Deref, DerefMut, Default)] pub struct Executor<'a>(ExecutorInner<'a>); -/// Wrapper around a single-threaded async executor. -/// Spawning wont generally require tasks to be `Send` and `Sync`, at the cost of -/// this executor itself not being `Send` or `Sync`. This makes it unsuitable for -/// global statics. -/// -/// If need to store an executor in a global static, or send across threads, -/// consider using [`Executor`] instead. -#[derive(Deref, DerefMut, Default)] -pub struct LocalExecutor<'a>(LocalExecutorInner<'a>); - impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] @@ -53,14 +41,10 @@ impl Executor<'_> { pub const fn new() -> Self { Self(ExecutorInner::new()) } -} -impl LocalExecutor<'_> { - /// Construct a new [`LocalExecutor`] - #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] - #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { - Self(LocalExecutorInner::new()) + #[inline] + pub fn try_tick_local() -> bool { + ExecutorInner::try_tick_local() } } @@ -68,18 +52,8 @@ impl UnwindSafe for Executor<'_> {} impl RefUnwindSafe for Executor<'_> {} -impl UnwindSafe for LocalExecutor<'_> {} - -impl RefUnwindSafe for LocalExecutor<'_> {} - impl fmt::Debug for Executor<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Executor").finish() } } - -impl fmt::Debug for LocalExecutor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("LocalExecutor").finish() - } -} diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 0f9488bcd0f33..33dd8f3c0acd5 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -4,25 +4,12 @@ use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; -#[cfg(feature = "std")] -use std::thread_local; - #[cfg(not(feature = "std"))] use bevy_platform::sync::{Mutex, PoisonError}; -#[cfg(feature = "std")] -use crate::executor::LocalExecutor; - -#[cfg(not(feature = "std"))] -use crate::executor::Executor as LocalExecutor; +use crate::executor::Executor; -#[cfg(feature = "std")] -thread_local! { - static LOCAL_EXECUTOR: LocalExecutor<'static> = const { LocalExecutor::new() }; -} - -#[cfg(not(feature = "std"))] -static LOCAL_EXECUTOR: LocalExecutor<'static> = const { LocalExecutor::new() }; +static EXECUTOR: Executor = Executor::new(); #[cfg(feature = "std")] type ScopeResult = alloc::rc::Rc>>; @@ -147,9 +134,9 @@ impl TaskPool { // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor = &LocalExecutor::new(); + let executor = &Executor::new(); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor: &'env LocalExecutor<'env> = unsafe { mem::transmute(executor) }; + let executor: &'env Executor<'env> = unsafe { mem::transmute(executor) }; let results: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -203,21 +190,17 @@ impl TaskPool { if #[cfg(all(target_arch = "wasm32", feature = "web"))] { Task::wrap_future(future) } else if #[cfg(feature = "std")] { - LOCAL_EXECUTOR.with(|executor| { - let task = executor.spawn(future); - // Loop until all tasks are done - while executor.try_tick() {} + let task = EXECUTOR.spawn_local(future); + // Loop until all tasks are done + while EXECUTOR.try_tick() {} - Task::new(task) - }) + Task::new(task) } else { - { - let task = LOCAL_EXECUTOR.spawn(future); - // Loop until all tasks are done - while LOCAL_EXECUTOR.try_tick() {} + EXECUTOR.spawn_local(future); + // Loop until all tasks are done + while EXECUTOR.try_tick() {} - Task::new(task) - } + Task::new(task) } } } @@ -232,28 +215,6 @@ impl TaskPool { { self.spawn(future) } - - /// Runs a function with the local executor. Typically used to tick - /// the local executor on the main thread as it needs to share time with - /// other things. - /// - /// ``` - /// use bevy_tasks::TaskPool; - /// - /// TaskPool::new().with_local_executor(|local_executor| { - /// local_executor.try_tick(); - /// }); - /// ``` - pub fn with_local_executor(&self, f: F) -> R - where - F: FnOnce(&LocalExecutor) -> R, - { - #[cfg(feature = "std")] - return LOCAL_EXECUTOR.with(f); - - #[cfg(not(feature = "std"))] - return f(&LOCAL_EXECUTOR); - } } /// A `TaskPool` scope for running one or more non-`'static` futures. @@ -261,7 +222,7 @@ impl TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor: &'scope LocalExecutor<'scope>, + executor: &'scope Executor<'scope>, // Vector to gather results of all futures spawned during scope run results: &'env RefCell>>, @@ -313,7 +274,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; - self.executor.spawn(f).detach(); + self.executor.spawn_local(f).detach(); } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 25255a1e5d1d3..45c098adbcd1f 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -143,7 +143,6 @@ pub struct TaskPool { impl TaskPool { thread_local! { - static LOCAL_EXECUTOR: crate::executor::LocalExecutor<'static> = const { crate::executor::LocalExecutor::new() }; static THREAD_EXECUTOR: Arc> = Arc::new(ThreadExecutor::new()); } @@ -187,28 +186,20 @@ impl TaskPool { thread_builder .spawn(move || { - TaskPool::LOCAL_EXECUTOR.with(|local_executor| { - if let Some(on_thread_spawn) = on_thread_spawn { - on_thread_spawn(); - drop(on_thread_spawn); - } - let _destructor = CallOnDrop(on_thread_destroy); - loop { - let res = std::panic::catch_unwind(|| { - let tick_forever = async move { - loop { - local_executor.tick().await; - } - }; - block_on(ex.run(tick_forever.or(shutdown_rx.recv()))) - }); - if let Ok(value) = res { - // Use unwrap_err because we expect a Closed error - value.unwrap_err(); - break; - } + if let Some(on_thread_spawn) = on_thread_spawn { + on_thread_spawn(); + drop(on_thread_spawn); + } + let _destructor = CallOnDrop(on_thread_destroy); + loop { + let res = + std::panic::catch_unwind(|| block_on(ex.run(shutdown_rx.recv()))); + if let Ok(value) = res { + // Use unwrap_err because we expect a Closed error + value.unwrap_err(); + break; } - }); + } }) .expect("Failed to spawn thread.") }) @@ -578,25 +569,7 @@ impl TaskPool { where T: 'static, { - Task::new(TaskPool::LOCAL_EXECUTOR.with(|executor| executor.spawn(future))) - } - - /// Runs a function with the local executor. Typically used to tick - /// the local executor on the main thread as it needs to share time with - /// other things. - /// - /// ``` - /// use bevy_tasks::TaskPool; - /// - /// TaskPool::new().with_local_executor(|local_executor| { - /// local_executor.try_tick(); - /// }); - /// ``` - pub fn with_local_executor(&self, f: F) -> R - where - F: FnOnce(&crate::executor::LocalExecutor) -> R, - { - Self::LOCAL_EXECUTOR.with(f) + Task::new(self.executor.spawn_local(future)) } } diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 8b08d5941c6bd..eb8429fe0948f 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,6 +1,7 @@ use super::TaskPool; use bevy_platform::sync::OnceLock; use core::ops::Deref; +use crate::executor::Executor; macro_rules! taskpool { ($(#[$attr:meta])* ($static:ident, $type:ident)) => { @@ -83,24 +84,9 @@ taskpool! { /// This function *must* be called on the main thread, or the task pools will not be updated appropriately. #[cfg(not(all(target_arch = "wasm32", feature = "web")))] pub fn tick_global_task_pools_on_main_thread() { - COMPUTE_TASK_POOL - .get() - .unwrap() - .with_local_executor(|compute_local_executor| { - ASYNC_COMPUTE_TASK_POOL - .get() - .unwrap() - .with_local_executor(|async_local_executor| { - IO_TASK_POOL - .get() - .unwrap() - .with_local_executor(|io_local_executor| { - for _ in 0..100 { - compute_local_executor.try_tick(); - async_local_executor.try_tick(); - io_local_executor.try_tick(); - } - }); - }); - }); + for _ in 0..100 { + if !Executor::try_tick_local() { + break; + } + } } From 2b8ed66ceee2c38b3be662ba9c227fa565feed1d Mon Sep 17 00:00:00 2001 From: James Liu Date: Sat, 26 Jul 2025 20:12:54 -0700 Subject: [PATCH 03/68] Merge ThreadEecutors into the new forked async_executor --- .../src/schedule/executor/multi_threaded.rs | 9 +- crates/bevy_render/src/pipelined_rendering.rs | 2 +- crates/bevy_tasks/src/async_executor.rs | 270 ++++++++++-------- crates/bevy_tasks/src/lib.rs | 5 +- crates/bevy_tasks/src/task_pool.rs | 234 +++------------ crates/bevy_tasks/src/thread_executor.rs | 133 --------- crates/bevy_tasks/src/usages.rs | 2 +- 7 files changed, 209 insertions(+), 446 deletions(-) delete mode 100644 crates/bevy_tasks/src/thread_executor.rs diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index bd99344f498fa..5637a12ab9fa2 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -1,7 +1,6 @@ use alloc::{boxed::Box, vec::Vec}; use bevy_platform::cell::SyncUnsafeCell; -use bevy_platform::sync::Arc; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadExecutor}; +use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadSpawner}; use concurrent_queue::ConcurrentQueue; use core::{any::Any, panic::AssertUnwindSafe}; use fixedbitset::FixedBitSet; @@ -272,12 +271,10 @@ impl SystemExecutor for MultiThreadedExecutor { let thread_executor = world .get_resource::() .map(|e| e.0.clone()); - let thread_executor = thread_executor.as_deref(); let environment = &Environment::new(self, schedule, world); ComputeTaskPool::get_or_init(TaskPool::default).scope_with_executor( - false, thread_executor, |scope| { let context = Context { @@ -866,7 +863,7 @@ unsafe fn evaluate_and_fold_conditions( /// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadExecutor(pub Arc>); +pub struct MainThreadExecutor(pub ThreadSpawner<'static>); impl Default for MainThreadExecutor { fn default() -> Self { @@ -877,7 +874,7 @@ impl Default for MainThreadExecutor { impl MainThreadExecutor { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadExecutor(TaskPool::get_thread_executor()) + MainThreadExecutor(ComputeTaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 00dfc4ba0e19c..9769b051bd341 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -186,7 +186,7 @@ fn renderer_extract(app_world: &mut World, _world: &mut World) { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. if let Some(mut render_app) = ComputeTaskPool::get() - .scope_with_executor(true, Some(&*main_thread_executor.0), |s| { + .scope_with_executor(Some(main_thread_executor.0.clone()), |s| { s.spawn(async { render_channels.recv().await }); }) .pop() diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 8f862f8f0e662..c577fa60f26cc 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -7,6 +7,7 @@ use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; +use std::thread::{Thread, ThreadId}; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; @@ -16,12 +17,93 @@ use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; -static LOCAL_QUEUE: ThreadLocal> = ThreadLocal::new(); +static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); -#[derive(Default)] -struct LocalQueue { - queue: VecDeque, - active: Slab, +struct ThreadLocalState { + thread_id: ThreadId, + thread_locked_queue: ConcurrentQueue, + local_queue: RefCell>, + local_active: RefCell>, +} + +impl Default for ThreadLocalState { + fn default() -> Self { + Self { + thread_id: std::thread::current().id(), + thread_locked_queue: ConcurrentQueue::unbounded(), + local_queue: RefCell::new(VecDeque::new()), + local_active: RefCell::new(Slab::new()), + } + } +} + +#[derive(Clone, Debug)] +pub struct ThreadSpawner<'a> { + thread_id: ThreadId, + target_queue: &'static ConcurrentQueue, + state: Arc, + _marker: PhantomData<&'a ()>, +} + +impl<'a> ThreadSpawner<'a> { + /// Spawns a task onto the executor. + pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + let mut active = self.state.active(); + + // Remove the task from the set of active tasks when the future finishes. + let entry = active.vacant_entry(); + let index = entry.key(); + let state = self.state.clone(); + let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + + #[expect( + unsafe_code, + reason = "unsized coercion is an unstable feature for non-std types" + )] + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule()) + }; + entry.insert(runnable.waker()); + + runnable.schedule(); + task + } + + /// Returns a function that schedules a runnable task when it gets woken up. + fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { + let thread_id = self.thread_id; + let queue: &'static ConcurrentQueue = self.target_queue; + let state = self.state.clone(); + + move |runnable| { + queue.push(runnable).unwrap(); + state.notify_specific_thread(thread_id); + } + } } /// An async executor. @@ -114,12 +196,12 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); - let mut local_state = local_queue.borrow_mut(); - let entry = local_state.active.vacant_entry(); + let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); + let mut local_active = local_state.local_active.borrow_mut(); + let entry = local_active.vacant_entry(); let index = entry.key(); let future = AsyncCallOnDrop::new(future, move || { - drop(local_queue.borrow_mut().active.try_remove(index)) + drop(local_state.local_active.borrow_mut().try_remove(index)) }); #[expect( @@ -155,35 +237,28 @@ impl<'a> Executor<'a> { }; entry.insert(runnable.waker()); - drop(local_state); + drop(local_active); runnable.schedule(); task } - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// - /// let ex = Executor::new(); - /// assert!(!ex.try_tick()); // no tasks to run - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.state().try_tick() + pub fn current_thread_spawner(&self) -> ThreadSpawner<'a> { + ThreadSpawner { + thread_id: std::thread::current().id(), + target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, + state: self.state_as_arc(), + _marker: PhantomData, + } } pub fn try_tick_local() -> bool { - if let Some(runnable) = LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back() { + if let Some(runnable) = THREAD_LOCAL_STATE + .get_or_default() + .local_queue + .borrow_mut() + .pop_back() + { // Run the task. runnable.run(); true @@ -192,44 +267,7 @@ impl<'a> Executor<'a> { } } - /// Runs a single task. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// use futures_lite::future; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { - /// println!("Hello world"); - /// }); - /// future::block_on(ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.state().tick().await; - } - /// Runs the executor until the given future completes. - /// - /// # Examples - /// - /// ``` - /// use crate::async_executor::Executor; - /// use futures_lite::future; - /// - /// let ex = Executor::new(); - /// - /// let task = ex.spawn(async { 1 + 2 }); - /// let res = future::block_on(ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await } @@ -247,11 +285,12 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let local_queue: &'static RefCell = LOCAL_QUEUE.get_or_default(); + let state = self.state_as_arc(); + let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { - local_queue.borrow_mut().queue.push_back(runnable); - // state.notify(); + local_state.local_queue.borrow_mut().push_back(runnable); + state.notify_specific_thread(local_state.thread_id); } } @@ -403,30 +442,19 @@ impl State { } } - pub(crate) fn try_tick(&self) -> bool { - let runnable = self - .queue - .pop() - .ok() - .or_else(|| LOCAL_QUEUE.get_or_default().borrow_mut().queue.pop_back()); - if let Some(runnable) = runnable { - // Notify another ticker now to pick up where this ticker left off, just in case - // running the task takes a long time. - self.notify(); - - // Run the task. - runnable.run(); - true - } else { - false + /// Notifies a sleeping ticker. + #[inline] + fn notify_specific_thread(&self, thread_id: ThreadId) { + let waker = self + .sleepers + .lock() + .unwrap() + .notify_specific_thread(thread_id); + if let Some(w) = waker { + w.wake(); } } - pub(crate) async fn tick(&self) { - let runnable = Ticker::new(self).runnable().await; - runnable.run(); - } - pub async fn run(&self, future: impl Future) -> T { let mut runner = Runner::new(self); let mut rng = fastrand::Rng::new(); @@ -447,6 +475,12 @@ impl State { } } +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_state(self, "State", f) + } +} + /// A list of sleeping tickers. struct Sleepers { /// Number of sleeping tickers (both notified and unnotified). @@ -455,7 +489,7 @@ struct Sleepers { /// IDs and wakers of sleeping unnotified tickers. /// /// A sleeping ticker is notified when its waker is missing from this list. - wakers: Vec<(usize, Waker)>, + wakers: Vec<(usize, ThreadId, Waker)>, /// Reclaimed IDs. free_ids: Vec, @@ -469,7 +503,8 @@ impl Sleepers { None => self.count + 1, }; self.count += 1; - self.wakers.push((id, waker.clone())); + self.wakers + .push((id, std::thread::current().id(), waker.clone())); id } @@ -479,12 +514,13 @@ impl Sleepers { fn update(&mut self, id: usize, waker: &Waker) -> bool { for item in &mut self.wakers { if item.0 == id { - item.1.clone_from(waker); + item.2.clone_from(waker); return false; } } - self.wakers.push((id, waker.clone())); + self.wakers + .push((id, std::thread::current().id(), waker.clone())); true } @@ -514,11 +550,24 @@ impl Sleepers { /// If a ticker was notified already or there are no tickers, `None` will be returned. fn notify(&mut self) -> Option { if self.wakers.len() == self.count { - self.wakers.pop().map(|item| item.1) + self.wakers.pop().map(|item| item.2) } else { None } } + + /// Returns notification waker for a sleeping ticker. + /// + /// If a ticker was notified already or there are no tickers, `None` will be returned. + fn notify_specific_thread(&mut self, thread_id: ThreadId) -> Option { + for i in (0..self.wakers.len()).rev() { + if self.wakers[i].1 == thread_id { + let (_, _, waker) = self.wakers.remove(i); + return Some(waker); + } + } + None + } } /// Runs task one by one. @@ -581,19 +630,6 @@ impl Ticker<'_> { self.sleeping = 0; } - /// Waits for the next runnable task to run. - async fn runnable(&mut self) -> Runnable { - self.runnable_with(|| { - LOCAL_QUEUE - .get_or_default() - .borrow_mut() - .queue - .pop_back() - .or_else(|| self.state.queue.pop().ok()) - }) - .await - } - /// Waits for the next runnable task to run, given a function that searches for a task. async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { future::poll_fn(|cx| { @@ -660,7 +696,7 @@ struct Runner<'a> { ticks: usize, // The thread local state of the executor for the current thread. - local_state: &'static RefCell, + local_state: &'static ThreadLocalState, } impl Runner<'_> { @@ -671,7 +707,7 @@ impl Runner<'_> { ticker: Ticker::new(state), local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, - local_state: LOCAL_QUEUE.get_or_default(), + local_state: THREAD_LOCAL_STATE.get_or_default(), }; state .local_queues @@ -686,7 +722,7 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - if let Some(r) = self.local_state.borrow_mut().queue.pop_back() { + if let Some(r) = self.local_state.local_queue.borrow_mut().pop_back() { return Some(r); } @@ -724,6 +760,12 @@ impl Runner<'_> { } } + if let Ok(r) = self.local_state.thread_locked_queue.pop() { + // Do not steal from this queue. If other threads steal + // from this current thread, the task will be moved. + return Some(r); + } + None }) .await; @@ -896,6 +938,8 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { + use super::Executor; + fn _ensure_send_and_sync() { fn is_send(_: T) {} fn is_sync(_: T) {} @@ -910,6 +954,8 @@ mod test { is_send(ex.schedule()); is_sync(ex.schedule()); is_static(ex.schedule()); + is_send(ex.current_thread_spawner()); + is_sync(ex.current_thread_spawner()); /// ```compile_fail /// use crate::async_executor::LocalExecutor; diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 4f23e7954bf36..2e04705b13fb0 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -60,14 +60,13 @@ pub use task::Task; cfg_if::cfg_if! { if #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { mod task_pool; - mod thread_executor; pub use task_pool::{Scope, TaskPool, TaskPoolBuilder}; - pub use thread_executor::{ThreadExecutor, ThreadExecutorTicker}; + pub use async_executor::ThreadSpawner; } else if #[cfg(any(target_arch = "wasm32", not(feature = "multi_threaded")))] { mod single_threaded_task_pool; - pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadExecutor}; + pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 45c098adbcd1f..bf9da3d2ed454 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -1,20 +1,14 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; -use std::{ - thread::{self, JoinHandle}, - thread_local, -}; +use std::thread::{self, JoinHandle}; +use crate::async_executor::ThreadSpawner; use crate::executor::FallibleTask; use bevy_platform::sync::Arc; use concurrent_queue::ConcurrentQueue; use futures_lite::FutureExt; -use crate::{ - block_on, - thread_executor::{ThreadExecutor, ThreadExecutorTicker}, - Task, -}; +use crate::{block_on, Task}; struct CallOnDrop(Option>); @@ -142,13 +136,9 @@ pub struct TaskPool { } impl TaskPool { - thread_local! { - static THREAD_EXECUTOR: Arc> = Arc::new(ThreadExecutor::new()); - } - - /// Each thread should only create one `ThreadExecutor`, otherwise, there are good chances they will deadlock - pub fn get_thread_executor() -> Arc> { - Self::THREAD_EXECUTOR.with(Clone::clone) + /// Each thread will only have one `ThreadExecutor`, otherwise, there are good chances they will deadlock + pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + self.executor.current_thread_spawner() } /// Create a `TaskPool` with the default configuration. @@ -303,57 +293,47 @@ impl TaskPool { F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - Self::THREAD_EXECUTOR.with(|scope_executor| { - self.scope_with_executor_inner(true, scope_executor, scope_executor, f) - }) + let scope_spawner = self.current_thread_spawner(); + self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } /// This allows passing an external executor to spawn tasks on. When you pass an external executor /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. /// - /// When `tick_task_pool_executor` is set to `true`, the multithreaded task stealing executor is ticked on the scope - /// thread. Disabling this can be useful when finishing the scope is latency sensitive. Pulling tasks from - /// global executor can run tasks unrelated to the scope and delay when the scope returns. - /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, - tick_task_pool_executor: bool, - external_executor: Option<&ThreadExecutor>, + external_spawner: Option>, f: F, ) -> Vec where F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - Self::THREAD_EXECUTOR.with(|scope_executor| { - // If an `external_executor` is passed, use that. Otherwise, get the executor stored - // in the `THREAD_EXECUTOR` thread local. - if let Some(external_executor) = external_executor { - self.scope_with_executor_inner( - tick_task_pool_executor, - external_executor, - scope_executor, - f, - ) - } else { - self.scope_with_executor_inner( - tick_task_pool_executor, - scope_executor, - scope_executor, - f, - ) - } - }) + let scope_spawner = self.executor.current_thread_spawner(); + // If an `external_executor` is passed, use that. Otherwise, get the executor stored + // in the `THREAD_EXECUTOR` thread local. + if let Some(external_spawner) = external_spawner { + self.scope_with_executor_inner( + external_spawner, + scope_spawner, + f, + ) + } else { + self.scope_with_executor_inner( + scope_spawner.clone(), + scope_spawner, + f, + ) + } } #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] fn scope_with_executor_inner<'env, F, T>( &self, - tick_task_pool_executor: bool, - external_executor: &ThreadExecutor, - scope_executor: &ThreadExecutor, + external_spawner: ThreadSpawner<'env>, + scope_spawner: ThreadSpawner<'env>, f: F, ) -> Vec where @@ -370,11 +350,6 @@ impl TaskPool { let executor: &crate::executor::Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let external_executor: &'env ThreadExecutor<'env> = - unsafe { mem::transmute(external_executor) }; - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let scope_executor: &'env ThreadExecutor<'env> = unsafe { mem::transmute(scope_executor) }; let spawned: ConcurrentQueue>>> = ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function @@ -385,8 +360,8 @@ impl TaskPool { let scope = Scope { executor, - external_executor, - scope_executor, + external_spawner, + scope_spawner, spawned, scope: PhantomData, env: PhantomData, @@ -401,144 +376,23 @@ impl TaskPool { if spawned.is_empty() { Vec::new() } else { - block_on(async move { - let get_results = async { - let mut results = Vec::with_capacity(spawned.len()); - while let Ok(task) = spawned.pop() { - if let Some(res) = task.await { - match res { - Ok(res) => results.push(res), - Err(payload) => std::panic::resume_unwind(payload), - } - } else { - panic!("Failed to catch panic!"); + block_on(self.executor.run(async move { + let mut results = Vec::with_capacity(spawned.len()); + while let Ok(task) = spawned.pop() { + if let Some(res) = task.await { + match res { + Ok(res) => results.push(res), + Err(payload) => std::panic::resume_unwind(payload), } + } else { + panic!("Failed to catch panic!"); } - results - }; - - let tick_task_pool_executor = tick_task_pool_executor || self.threads.is_empty(); - - // we get this from a thread local so we should always be on the scope executors thread. - // note: it is possible `scope_executor` and `external_executor` is the same executor, - // in that case, we should only tick one of them, otherwise, it may cause deadlock. - let scope_ticker = scope_executor.ticker().unwrap(); - let external_ticker = if !external_executor.is_same(scope_executor) { - external_executor.ticker() - } else { - None - }; - - match (external_ticker, tick_task_pool_executor) { - (Some(external_ticker), true) => { - Self::execute_global_external_scope( - executor, - external_ticker, - scope_ticker, - get_results, - ) - .await - } - (Some(external_ticker), false) => { - Self::execute_external_scope(external_ticker, scope_ticker, get_results) - .await - } - // either external_executor is none or it is same as scope_executor - (None, true) => { - Self::execute_global_scope(executor, scope_ticker, get_results).await - } - (None, false) => Self::execute_scope(scope_ticker, get_results).await, } - }) + results + })) } } - #[inline] - async fn execute_global_external_scope<'scope, 'ticker, T>( - executor: &'scope crate::executor::Executor<'scope>, - external_ticker: ThreadExecutorTicker<'scope, 'ticker>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - // we restart the executors if a task errors. if a scoped - // task errors it will panic the scope on the call to get_results - let execute_forever = async move { - loop { - let tick_forever = async { - loop { - external_ticker.tick().or(scope_ticker.tick()).await; - } - }; - // we don't care if it errors. If a scoped task errors it will propagate - // to get_results - let _result = AssertUnwindSafe(executor.run(tick_forever)) - .catch_unwind() - .await - .is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_external_scope<'scope, 'ticker, T>( - external_ticker: ThreadExecutorTicker<'scope, 'ticker>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - external_ticker.tick().or(scope_ticker.tick()).await; - } - }; - let _result = AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_global_scope<'scope, 'ticker, T>( - executor: &'scope crate::executor::Executor<'scope>, - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - scope_ticker.tick().await; - } - }; - let _result = AssertUnwindSafe(executor.run(tick_forever)) - .catch_unwind() - .await - .is_ok(); - } - }; - get_results.or(execute_forever).await - } - - #[inline] - async fn execute_scope<'scope, 'ticker, T>( - scope_ticker: ThreadExecutorTicker<'scope, 'ticker>, - get_results: impl Future>, - ) -> Vec { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - scope_ticker.tick().await; - } - }; - let _result = AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); - } - }; - get_results.or(execute_forever).await - } - /// Spawns a static future onto the thread pool. The returned [`Task`] is a /// future that can be polled for the result. It can also be canceled and /// "detached", allowing the task to continue running even if dropped. In @@ -599,8 +453,8 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope crate::executor::Executor<'scope>, - external_executor: &'scope ThreadExecutor<'scope>, - scope_executor: &'scope ThreadExecutor<'scope>, + external_spawner: ThreadSpawner<'scope>, + scope_spawner: ThreadSpawner<'scope>, spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -634,7 +488,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { let task = self - .scope_executor + .scope_spawner .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); // ConcurrentQueue only errors when closed or full, but we never @@ -651,7 +505,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_external + 'scope + Send>(&self, f: Fut) { let task = self - .external_executor + .external_spawner .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); // ConcurrentQueue only errors when closed or full, but we never diff --git a/crates/bevy_tasks/src/thread_executor.rs b/crates/bevy_tasks/src/thread_executor.rs deleted file mode 100644 index 86d2ab280d87c..0000000000000 --- a/crates/bevy_tasks/src/thread_executor.rs +++ /dev/null @@ -1,133 +0,0 @@ -use core::marker::PhantomData; -use std::thread::{self, ThreadId}; - -use crate::executor::Executor; -use async_task::Task; -use futures_lite::Future; - -/// An executor that can only be ticked on the thread it was instantiated on. But -/// can spawn `Send` tasks from other threads. -/// -/// # Example -/// ``` -/// # use std::sync::{Arc, atomic::{AtomicI32, Ordering}}; -/// use bevy_tasks::ThreadExecutor; -/// -/// let thread_executor = ThreadExecutor::new(); -/// let count = Arc::new(AtomicI32::new(0)); -/// -/// // create some owned values that can be moved into another thread -/// let count_clone = count.clone(); -/// -/// std::thread::scope(|scope| { -/// scope.spawn(|| { -/// // we cannot get the ticker from another thread -/// let not_thread_ticker = thread_executor.ticker(); -/// assert!(not_thread_ticker.is_none()); -/// -/// // but we can spawn tasks from another thread -/// thread_executor.spawn(async move { -/// count_clone.fetch_add(1, Ordering::Relaxed); -/// }).detach(); -/// }); -/// }); -/// -/// // the tasks do not make progress unless the executor is manually ticked -/// assert_eq!(count.load(Ordering::Relaxed), 0); -/// -/// // tick the ticker until task finishes -/// let thread_ticker = thread_executor.ticker().unwrap(); -/// thread_ticker.try_tick(); -/// assert_eq!(count.load(Ordering::Relaxed), 1); -/// ``` -#[derive(Debug)] -pub struct ThreadExecutor<'task> { - executor: Executor<'task>, - thread_id: ThreadId, -} - -impl<'task> Default for ThreadExecutor<'task> { - fn default() -> Self { - Self { - executor: Executor::new(), - thread_id: thread::current().id(), - } - } -} - -impl<'task> ThreadExecutor<'task> { - /// create a new [`ThreadExecutor`] - pub fn new() -> Self { - Self::default() - } - - /// Spawn a task on the thread executor - pub fn spawn( - &self, - future: impl Future + Send + 'task, - ) -> Task { - self.executor.spawn(future) - } - - /// Gets the [`ThreadExecutorTicker`] for this executor. - /// Use this to tick the executor. - /// It only returns the ticker if it's on the thread the executor was created on - /// and returns `None` otherwise. - pub fn ticker<'ticker>(&'ticker self) -> Option> { - if thread::current().id() == self.thread_id { - return Some(ThreadExecutorTicker { - executor: self, - _marker: PhantomData, - }); - } - None - } - - /// Returns true if `self` and `other`'s executor is same - pub fn is_same(&self, other: &Self) -> bool { - core::ptr::eq(self, other) - } -} - -/// Used to tick the [`ThreadExecutor`]. The executor does not -/// make progress unless it is manually ticked on the thread it was -/// created on. -#[derive(Debug)] -pub struct ThreadExecutorTicker<'task, 'ticker> { - executor: &'ticker ThreadExecutor<'task>, - // make type not send or sync - _marker: PhantomData<*const ()>, -} - -impl<'task, 'ticker> ThreadExecutorTicker<'task, 'ticker> { - /// Tick the thread executor. - pub async fn tick(&self) { - self.executor.executor.tick().await; - } - - /// Synchronously try to tick a task on the executor. - /// Returns false if does not find a task to tick. - pub fn try_tick(&self) -> bool { - self.executor.executor.try_tick() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::sync::Arc; - - #[test] - fn test_ticker() { - let executor = Arc::new(ThreadExecutor::new()); - let ticker = executor.ticker(); - assert!(ticker.is_some()); - - thread::scope(|s| { - s.spawn(|| { - let ticker = executor.ticker(); - assert!(ticker.is_none()); - }); - }); - } -} diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index eb8429fe0948f..8bd0bd3182cd2 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,7 +1,7 @@ use super::TaskPool; +use crate::executor::Executor; use bevy_platform::sync::OnceLock; use core::ops::Deref; -use crate::executor::Executor; macro_rules! taskpool { ($(#[$attr:meta])* ($static:ident, $type:ident)) => { From 1121542bdbbde3873731705b9e9aaf192590be4f Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 27 Jul 2025 09:10:47 -0700 Subject: [PATCH 04/68] Move stealer queues to TLS storage. Avoid extra Arc allocations --- crates/bevy_tasks/src/async_executor.rs | 244 ++++++++++++++---------- crates/bevy_tasks/src/task_pool.rs | 12 +- 2 files changed, 148 insertions(+), 108 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index c577fa60f26cc..f6f396b2200da 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,20 +19,32 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); +std::thread_local! { + static LOCAL_QUEUE: LocalQueue = const { + LocalQueue { + local_queue: RefCell::new(VecDeque::new()), + local_active: RefCell::new(Slab::new()), + } + }; +} + +struct LocalQueue { + local_queue: RefCell>, + local_active: RefCell>, +} + struct ThreadLocalState { thread_id: ThreadId, + stealable_queue: ConcurrentQueue, thread_locked_queue: ConcurrentQueue, - local_queue: RefCell>, - local_active: RefCell>, } impl Default for ThreadLocalState { fn default() -> Self { Self { thread_id: std::thread::current().id(), + stealable_queue: ConcurrentQueue::bounded(512), thread_locked_queue: ConcurrentQueue::unbounded(), - local_queue: RefCell::new(VecDeque::new()), - local_active: RefCell::new(Slab::new()), } } } @@ -101,7 +113,7 @@ impl<'a> ThreadSpawner<'a> { move |runnable| { queue.push(runnable).unwrap(); - state.notify_specific_thread(thread_id); + state.notify_specific_thread(thread_id, false); } } } @@ -196,49 +208,52 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); - let mut local_active = local_state.local_active.borrow_mut(); - let entry = local_active.vacant_entry(); - let index = entry.key(); - let future = AsyncCallOnDrop::new(future, move || { - drop(local_state.local_active.borrow_mut().try_remove(index)) + let (runnable, task) = LOCAL_QUEUE.with(|tls| { + let mut local_active = tls.local_active.borrow_mut(); + let entry = local_active.vacant_entry(); + let index = entry.key(); + let future = AsyncCallOnDrop::new(future, move || { + LOCAL_QUEUE.with(|tls| { + drop(tls.local_active.borrow_mut().try_remove(index)) + }); + }); + + #[expect( + unsafe_code, + reason = "Builder::spawn_local requires a 'static lifetime" + )] + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = unsafe { + Builder::new() + .propagate_panic(true) + .spawn_unchecked(|()| future, self.schedule_local()) + }; + entry.insert(runnable.waker()); + + (runnable, task) }); - #[expect( - unsafe_code, - reason = "Builder::spawn_local requires a 'static lifetime" - )] - // Create the task and register it in the set of active tasks. - // - // SAFETY: - // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. - let (runnable, task) = unsafe { - Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule_local()) - }; - entry.insert(runnable.waker()); - - drop(local_active); - runnable.schedule(); task } @@ -253,18 +268,19 @@ impl<'a> Executor<'a> { } pub fn try_tick_local() -> bool { - if let Some(runnable) = THREAD_LOCAL_STATE - .get_or_default() - .local_queue - .borrow_mut() - .pop_back() - { - // Run the task. - runnable.run(); - true - } else { - false - } + LOCAL_QUEUE.with(|tls| { + if let Some(runnable) = tls + .local_queue + .borrow_mut() + .pop_back() + { + // Run the task. + runnable.run(); + true + } else { + false + } + }) } /// Runs the executor until the given future completes. @@ -276,8 +292,21 @@ impl<'a> Executor<'a> { fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { let state = self.state_as_arc(); - // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { + // Attempt to push onto the local queue first, because we know that + // this thread is awake. + // let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + // match local_state.stealable_queue.push(runnable) { + // Ok(()) => { + // state.notify_specific_thread(local_state.thread_id, true); + // return; + // } + // Err(r) => r.into_inner(), + // } + // } else { + // runnable + // }; + // Otherwise push onto the global queue instead. state.queue.push(runnable).unwrap(); state.notify(); } @@ -289,8 +318,10 @@ impl<'a> Executor<'a> { let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { - local_state.local_queue.borrow_mut().push_back(runnable); - state.notify_specific_thread(local_state.thread_id); + LOCAL_QUEUE.with(|tls| { + tls.local_queue.borrow_mut().push_back(runnable); + }); + state.notify_specific_thread(local_state.thread_id, false); } } @@ -394,7 +425,7 @@ struct State { queue: ConcurrentQueue, /// Local queues created by runners. - local_queues: RwLock>>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -411,7 +442,7 @@ impl State { const fn new() -> State { State { queue: ConcurrentQueue::unbounded(), - local_queues: RwLock::new(Vec::new()), + stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { count: 0, @@ -444,12 +475,18 @@ impl State { /// Notifies a sleeping ticker. #[inline] - fn notify_specific_thread(&self, thread_id: ThreadId) { - let waker = self - .sleepers - .lock() - .unwrap() - .notify_specific_thread(thread_id); + fn notify_specific_thread(&self, thread_id: ThreadId, allow_stealing: bool) { + let mut sleepers = self.sleepers.lock().unwrap(); + let mut waker = sleepers.notify_specific_thread(thread_id); + if waker.is_none() + && allow_stealing + && self + .notified + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + waker = sleepers.notify(); + } if let Some(w) = waker { w.wake(); } @@ -689,9 +726,6 @@ struct Runner<'a> { /// Inner ticker. ticker: Ticker<'a>, - /// The local queue. - local: Arc>, - /// Bumped every time a runnable task is found. ticks: usize, @@ -702,18 +736,18 @@ struct Runner<'a> { impl Runner<'_> { /// Creates a runner and registers it in the executor state. fn new(state: &State) -> Runner<'_> { + let local_state = THREAD_LOCAL_STATE.get_or_default(); let runner = Runner { state, ticker: Ticker::new(state), - local: Arc::new(ConcurrentQueue::bounded(512)), ticks: 0, - local_state: THREAD_LOCAL_STATE.get_or_default(), + local_state, }; state - .local_queues + .stealer_queues .write() .unwrap() - .push(runner.local.clone()); + .push(&local_state.stealable_queue); runner } @@ -722,40 +756,46 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - if let Some(r) = self.local_state.local_queue.borrow_mut().pop_back() { - return Some(r); + { + let local_pop = LOCAL_QUEUE.with(|tls| { + tls.local_queue.borrow_mut().pop_back() + }); + if let Some(r) = local_pop { + return Some(r); + } } // Try the local queue. - if let Ok(r) = self.local.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. if let Ok(r) = self.state.queue.pop() { - steal(&self.state.queue, &self.local); + steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } // Try stealing from other runners. - let local_queues = self.state.local_queues.read().unwrap(); + let stealer_queues = self.state.stealer_queues.read().unwrap(); // Pick a random starting point in the iterator list and rotate the list. - let n = local_queues.len(); + let n = stealer_queues.len(); let start = rng.usize(..n); - let iter = local_queues + let iter = stealer_queues .iter() - .chain(local_queues.iter()) + .chain(stealer_queues.iter()) .skip(start) .take(n); // Remove this runner's local queue. - let iter = iter.filter(|local| !Arc::ptr_eq(local, &self.local)); + let iter = + iter.filter(|local| !std::ptr::eq(**local, &self.local_state.stealable_queue)); // Try stealing from each local queue in the list. for local in iter { - steal(local, &self.local); - if let Ok(r) = self.local.pop() { + steal(local, &self.local_state.stealable_queue); + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } } @@ -775,7 +815,7 @@ impl Runner<'_> { if self.ticks % 64 == 0 { // Steal tasks from the global queue to ensure fair task scheduling. - steal(&self.state.queue, &self.local); + steal(&self.state.queue, &self.local_state.stealable_queue); } runnable @@ -785,14 +825,20 @@ impl Runner<'_> { impl Drop for Runner<'_> { fn drop(&mut self) { // Remove the local queue. - self.state - .local_queues - .write() - .unwrap() - .retain(|local| !Arc::ptr_eq(local, &self.local)); + { + let mut stealer_queues = self.state.stealer_queues.write().unwrap(); + if let Some((idx, _)) = stealer_queues + .iter() + .enumerate() + .rev() + .find(|(_, local)| !std::ptr::eq(**local, &self.local_state.stealable_queue)) + { + stealer_queues.remove(idx); + } + } // Re-schedule remaining tasks in the local queue. - while let Ok(r) = self.local.pop() { + while let Ok(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } @@ -865,7 +911,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -896,7 +942,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re f.debug_struct(name) .field("active", &ActiveTasks(&state.active)) .field("global_tasks", &state.queue.len()) - .field("local_runners", &LocalRunners(&state.local_queues)) + .field("stealer_queues", &LocalRunners(&state.stealer_queues)) .field("sleepers", &SleepCount(&state.sleepers)) .finish() } @@ -939,6 +985,8 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { use super::Executor; + use super::ThreadLocalState; + use super::THREAD_LOCAL_STATE; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -949,13 +997,13 @@ mod test { is_sync::>(Executor::new()); let ex = Executor::new(); - is_send(ex.tick()); - is_sync(ex.tick()); is_send(ex.schedule()); is_sync(ex.schedule()); is_static(ex.schedule()); is_send(ex.current_thread_spawner()); is_sync(ex.current_thread_spawner()); + is_send(THREAD_LOCAL_STATE.get_or_default()); + is_sync(THREAD_LOCAL_STATE.get_or_default()); /// ```compile_fail /// use crate::async_executor::LocalExecutor; diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index bf9da3d2ed454..19a1e6887447d 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -315,17 +315,9 @@ impl TaskPool { // If an `external_executor` is passed, use that. Otherwise, get the executor stored // in the `THREAD_EXECUTOR` thread local. if let Some(external_spawner) = external_spawner { - self.scope_with_executor_inner( - external_spawner, - scope_spawner, - f, - ) + self.scope_with_executor_inner(external_spawner, scope_spawner, f) } else { - self.scope_with_executor_inner( - scope_spawner.clone(), - scope_spawner, - f, - ) + self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } } From 01ac223cf17293701efb4ec5653670f5cd05492d Mon Sep 17 00:00:00 2001 From: James Liu Date: Sun, 27 Jul 2025 19:21:05 -0700 Subject: [PATCH 05/68] Queue tasks directly back onto local queues to avoid global injector queue contention --- crates/bevy_tasks/src/async_executor.rs | 54 +++++++++++++------------ crates/bevy_tasks/src/task_pool.rs | 2 + 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f6f396b2200da..705cc9060655d 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,6 +19,11 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); +pub(crate) fn install_runtime_into_current_thread() { + let tls = THREAD_LOCAL_STATE.get_or_default(); + tls.executor_thread.store(true, Ordering::Relaxed); +} + std::thread_local! { static LOCAL_QUEUE: LocalQueue = const { LocalQueue { @@ -34,6 +39,7 @@ struct LocalQueue { } struct ThreadLocalState { + executor_thread: AtomicBool, thread_id: ThreadId, stealable_queue: ConcurrentQueue, thread_locked_queue: ConcurrentQueue, @@ -42,6 +48,7 @@ struct ThreadLocalState { impl Default for ThreadLocalState { fn default() -> Self { Self { + executor_thread: AtomicBool::new(false), thread_id: std::thread::current().id(), stealable_queue: ConcurrentQueue::bounded(512), thread_locked_queue: ConcurrentQueue::unbounded(), @@ -213,9 +220,7 @@ impl<'a> Executor<'a> { let entry = local_active.vacant_entry(); let index = entry.key(); let future = AsyncCallOnDrop::new(future, move || { - LOCAL_QUEUE.with(|tls| { - drop(tls.local_active.borrow_mut().try_remove(index)) - }); + LOCAL_QUEUE.with(|tls| drop(tls.local_active.borrow_mut().try_remove(index))); }); #[expect( @@ -269,11 +274,7 @@ impl<'a> Executor<'a> { pub fn try_tick_local() -> bool { LOCAL_QUEUE.with(|tls| { - if let Some(runnable) = tls - .local_queue - .borrow_mut() - .pop_back() - { + if let Some(runnable) = tls.local_queue.borrow_mut().pop_back() { // Run the task. runnable.run(); true @@ -293,19 +294,23 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); move |runnable| { - // Attempt to push onto the local queue first, because we know that - // this thread is awake. - // let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - // match local_state.stealable_queue.push(runnable) { - // Ok(()) => { - // state.notify_specific_thread(local_state.thread_id, true); - // return; - // } - // Err(r) => r.into_inner(), - // } - // } else { - // runnable - // }; + // Attempt to push onto the local queue first in dedicated executor threads, + // because we know that this thread is awake and always processing new tasks. + let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + if local_state.executor_thread.load(Ordering::Relaxed) { + match local_state.stealable_queue.push(runnable) { + Ok(()) => { + state.notify_specific_thread(local_state.thread_id, true); + return; + } + Err(r) => r.into_inner(), + } + } else { + runnable + } + } else { + runnable + }; // Otherwise push onto the global queue instead. state.queue.push(runnable).unwrap(); state.notify(); @@ -316,7 +321,6 @@ impl<'a> Executor<'a> { fn schedule_local(&self) -> impl Fn(Runnable) + 'static { let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); - // TODO: If possible, push into the current local queue and notify the ticker. move |runnable| { LOCAL_QUEUE.with(|tls| { tls.local_queue.borrow_mut().push_back(runnable); @@ -757,9 +761,7 @@ impl Runner<'_> { .ticker .runnable_with(|| { { - let local_pop = LOCAL_QUEUE.with(|tls| { - tls.local_queue.borrow_mut().pop_back() - }); + let local_pop = LOCAL_QUEUE.with(|tls| tls.local_queue.borrow_mut().pop_back()); if let Some(r) = local_pop { return Some(r); } @@ -831,7 +833,7 @@ impl Drop for Runner<'_> { .iter() .enumerate() .rev() - .find(|(_, local)| !std::ptr::eq(**local, &self.local_state.stealable_queue)) + .find(|(_, local)| std::ptr::eq(**local, &self.local_state.stealable_queue)) { stealer_queues.remove(idx); } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 19a1e6887447d..ac31c86359b35 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -176,6 +176,8 @@ impl TaskPool { thread_builder .spawn(move || { + crate::async_executor::install_runtime_into_current_thread(); + if let Some(on_thread_spawn) = on_thread_spawn { on_thread_spawn(); drop(on_thread_spawn); From 1dab08e37a1d1a260f1324621c6a69b37d4014f2 Mon Sep 17 00:00:00 2001 From: James Liu Date: Mon, 28 Jul 2025 13:58:54 -0700 Subject: [PATCH 06/68] Use UnsafeCell instead of RefCell --- crates/bevy_tasks/src/async_executor.rs | 223 ++++++++++++------------ 1 file changed, 112 insertions(+), 111 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 705cc9060655d..32645c190eb59 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -1,4 +1,8 @@ -use std::cell::RefCell; +#![expect( + unsafe_code, + reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" +)] + use std::collections::VecDeque; use std::fmt; use std::marker::PhantomData; @@ -7,7 +11,7 @@ use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; -use std::thread::{Thread, ThreadId}; +use std::thread::ThreadId; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; @@ -24,18 +28,54 @@ pub(crate) fn install_runtime_into_current_thread() { tls.executor_thread.store(true, Ordering::Relaxed); } -std::thread_local! { - static LOCAL_QUEUE: LocalQueue = const { - LocalQueue { - local_queue: RefCell::new(VecDeque::new()), - local_active: RefCell::new(Slab::new()), +// Do not access this directly, use `with_local_queue` instead. +cfg_if::cfg_if! { + if #[cfg(all(debug_assertions, not(miri)))] { + use std::cell::RefCell; + + std::thread_local! { + static LOCAL_QUEUE: RefCell = const { + RefCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; + } + } else { + use std::cell::UnsafeCell; + + std::thread_local! { + static LOCAL_QUEUE: UnsafeCell = const { + UnsafeCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; + } + } +} + +/// # Safety +/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +#[inline(always)] +unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { + LOCAL_QUEUE.with(|tls| { + cfg_if::cfg_if! { + if #[cfg(all(debug_assertions, not(miri)))] { + f(&mut *tls.borrow_mut()) + } else { + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. The caller guarantees that this function is not used with + // LOCAL_QUEUE in any way. + f(unsafe { &mut *tls.get() }) + } } - }; + }) } struct LocalQueue { - local_queue: RefCell>, - local_active: RefCell>, + local_queue: VecDeque, + local_active: Slab, } struct ThreadLocalState { @@ -75,10 +115,6 @@ impl<'a> ThreadSpawner<'a> { let state = self.state.clone(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -131,22 +167,9 @@ pub struct Executor<'a> { state: AtomicPtr, /// Makes the `'a` lifetime invariant. - _marker: PhantomData>, + _marker: PhantomData<&'a ()>, } -#[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" -)] -// SAFETY: Executor stores no thread local state that can be accessed via other thread. -unsafe impl Send for Executor<'_> {} -#[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" -)] -// SAFETY: Executor internally synchronizes all of it's operations internally. -unsafe impl Sync for Executor<'_> {} - impl UnwindSafe for Executor<'_> {} impl RefUnwindSafe for Executor<'_> {} @@ -175,10 +198,6 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // Create the task and register it in the set of active tasks. // // SAFETY: @@ -215,49 +234,51 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'a) -> Task { // Remove the task from the set of active tasks when the future finishes. - let (runnable, task) = LOCAL_QUEUE.with(|tls| { - let mut local_active = tls.local_active.borrow_mut(); - let entry = local_active.vacant_entry(); - let index = entry.key(); - let future = AsyncCallOnDrop::new(future, move || { - LOCAL_QUEUE.with(|tls| drop(tls.local_active.borrow_mut().try_remove(index))); - }); - - #[expect( - unsafe_code, - reason = "Builder::spawn_local requires a 'static lifetime" - )] - // Create the task and register it in the set of active tasks. - // - // SAFETY: - // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. - let (runnable, task) = unsafe { - Builder::new() + // + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let (runnable, task) = unsafe { + with_local_queue(|tls| { + let entry = tls.local_active.vacant_entry(); + let index = entry.key(); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. This AsyncCallOnDrop will be + // invoked after the surrounding scope has exited in either a + // `try_tick_local` or `run` call. + let future = AsyncCallOnDrop::new(future, move || { + with_local_queue(|tls| drop(tls.local_active.try_remove(index))); + }); + + // Create the task and register it in the set of active tasks. + // + // SAFETY: + // + // If `future` is not `Send`, this must be a `LocalExecutor` as per this + // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, + // `try_tick`, `tick` and `run` can only be called from the origin + // thread of the `LocalExecutor`. Similarly, `spawn` can only be called + // from the origin thread, ensuring that `future` and the executor share + // the same origin thread. The `Runnable` can be scheduled from other + // threads, but because of the above `Runnable` can only be called or + // dropped on the origin thread. + // + // `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // + // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. + let (runnable, task) = Builder::new() .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule_local()) - }; - entry.insert(runnable.waker()); + .spawn_unchecked(|()| future, self.schedule_local()); + entry.insert(runnable.waker()); - (runnable, task) - }); + (runnable, task) + }) + }; runnable.schedule(); task @@ -273,15 +294,13 @@ impl<'a> Executor<'a> { } pub fn try_tick_local() -> bool { - LOCAL_QUEUE.with(|tls| { - if let Some(runnable) = tls.local_queue.borrow_mut().pop_back() { - // Run the task. - runnable.run(); - true - } else { - false - } - }) + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) } + .map(|runnable| runnable.run()) + .is_some() } /// Runs the executor until the given future completes. @@ -322,9 +341,12 @@ impl<'a> Executor<'a> { let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { - LOCAL_QUEUE.with(|tls| { - tls.local_queue.borrow_mut().push_back(runnable); - }); + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { + with_local_queue(|tls| tls.local_queue.push_back(runnable)); + } state.notify_specific_thread(local_state.thread_id, false); } } @@ -335,18 +357,13 @@ impl<'a> Executor<'a> { #[cold] fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { let state = Arc::new(State::new()); - // TODO: Switch this to use cast_mut once the MSRV can be bumped past 1.65 - let ptr = Arc::into_raw(state) as *mut State; + let ptr = Arc::into_raw(state).cast_mut(); if let Err(actual) = atomic_ptr.compare_exchange( std::ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire, ) { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: This was just created from Arc::into_raw. drop(unsafe { Arc::from_raw(ptr) }); actual @@ -365,24 +382,14 @@ impl<'a> Executor<'a> { /// Returns a reference to the inner state. #[inline] fn state(&self) -> &State { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: So long as an Executor lives, it's state pointer will always be valid // when accessed through state_ptr. - unsafe { - &*self.state_ptr() - } + unsafe { &*self.state_ptr() } } // Clones the inner state Arc #[inline] fn state_as_arc(&self) -> Arc { - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: So long as an Executor lives, it's state pointer will always be a valid // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; @@ -399,10 +406,6 @@ impl Drop for Executor<'_> { return; } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: As ptr is not null, it was allocated via Arc::new and converted // via Arc::into_raw in state_ptr. let state = unsafe { Arc::from_raw(ptr) }; @@ -761,7 +764,9 @@ impl Runner<'_> { .ticker .runnable_with(|| { { - let local_pop = LOCAL_QUEUE.with(|tls| tls.local_queue.borrow_mut().pop_back()); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) }; if let Some(r) = local_pop { return Some(r); } @@ -885,10 +890,6 @@ fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_ return f.debug_tuple(name).field(&Uninitialized).finish(); } - #[expect( - unsafe_code, - reason = "unsized coercion is an unstable feature for non-std types" - )] // SAFETY: If the state pointer is not null, it must have been // allocated properly by Arc::new and converted via Arc::into_raw // in state_ptr. From 0b036fe4936d028d763454e39e31a191712959be Mon Sep 17 00:00:00 2001 From: James Liu Date: Mon, 28 Jul 2025 14:29:31 -0700 Subject: [PATCH 07/68] Optimize access of thread locked tasks --- crates/bevy_tasks/src/async_executor.rs | 65 +++++++++++++++++++------ 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 32645c190eb59..6271598e66765 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -144,18 +144,25 @@ impl<'a> ThreadSpawner<'a> { }; entry.insert(runnable.waker()); - runnable.schedule(); + // Instead of directly scheduling this task, it's put into the onto the + // thread locked queue to be moved to the target thread, where it will + // either be run immediately or flushed into the thread's local queue. + self.target_queue.push(runnable).unwrap(); task } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { let thread_id = self.thread_id; - let queue: &'static ConcurrentQueue = self.target_queue; let state = self.state.clone(); move |runnable| { - queue.push(runnable).unwrap(); + // SAFETY: This value is in thread local storage and thus can only be accesed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { + with_local_queue(|tls| tls.local_queue.push_back(runnable)); + } state.notify_specific_thread(thread_id, false); } } @@ -298,7 +305,7 @@ impl<'a> Executor<'a> { // from multiple locations simultaneously. As the Runnable is run after // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) } + unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } .map(|runnable| runnable.run()) .is_some() } @@ -763,13 +770,11 @@ impl Runner<'_> { let runnable = self .ticker .runnable_with(|| { - { - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. - let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_back()) }; - if let Some(r) = local_pop { - return Some(r); - } + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) }; + if let Some(r) = local_pop { + return Some(r); } // Try the local queue. @@ -810,6 +815,13 @@ impl Runner<'_> { if let Ok(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. + // + // Instead, flush all queued tasks into the local queue to + // minimize the effort required to scan for these tasks. + // + // SAFETY: This is not being used at the same time as any + // access to LOCAL_QUEUE. + unsafe { flush_to_local(&self.local_state.thread_locked_queue) }; return Some(r); } @@ -864,15 +876,38 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Steal tasks. for _ in 0..count { - if let Ok(t) = src.pop() { - assert!(dest.push(t).is_ok()); - } else { - break; + match src.pop() { + Ok(t) => assert!(dest.push(t).is_ok()), + Err(_) => break, } } } } +/// Flushes all of the items from a queue into the thread local queue. +/// +/// # Safety +/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +unsafe fn flush_to_local(src: &ConcurrentQueue) { + let count = src.len(); + + if count > 0 { + // SAFETY: Caller assures that LOCAL_QUEUE does not have any + // overlapping accesses. + unsafe { + with_local_queue(|tls| { + // Steal tasks. + for _ in 0..count { + match src.pop() { + Ok(t) => tls.local_queue.push_front(t), + Err(_) => break, + } + } + }) + } + } +} + /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Get a reference to the state. From 45b5e15a78f3b797e58ab68f7bcd156dfe1d9b67 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 29 Jul 2025 05:28:36 -0700 Subject: [PATCH 08/68] Clean up unnecessary unsafe use --- crates/bevy_tasks/src/async_executor.rs | 110 +++--------------------- crates/bevy_tasks/src/executor.rs | 4 +- 2 files changed, 16 insertions(+), 98 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 6271598e66765..446ce57306bcb 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -8,7 +8,7 @@ use std::fmt; use std::marker::PhantomData; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; use std::task::{Context, Poll, Waker}; use std::thread::ThreadId; @@ -171,7 +171,7 @@ impl<'a> ThreadSpawner<'a> { /// An async executor. pub struct Executor<'a> { /// The executor state. - state: AtomicPtr, + state: Arc, /// Makes the `'a` lifetime invariant. _marker: PhantomData<&'a ()>, @@ -188,21 +188,21 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - pub const fn new() -> Executor<'a> { + pub fn new() -> Executor<'a> { Executor { - state: AtomicPtr::new(std::ptr::null_mut()), + state: Arc::new(State::new()), _marker: PhantomData, } } /// Spawns a task onto the executor. pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state().active(); + let mut active = self.state.active(); // Remove the task from the set of active tasks when the future finishes. let entry = active.vacant_entry(); let index = entry.key(); - let state = self.state_as_arc(); + let state = self.state.clone(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); // Create the task and register it in the set of active tasks. @@ -295,7 +295,7 @@ impl<'a> Executor<'a> { ThreadSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, - state: self.state_as_arc(), + state: self.state.clone(), _marker: PhantomData, } } @@ -312,12 +312,12 @@ impl<'a> Executor<'a> { /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { - self.state().run(future).await + self.state.run(future).await } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { - let state = self.state_as_arc(); + let state = self.state.clone(); move |runnable| { // Attempt to push onto the local queue first in dedicated executor threads, @@ -345,7 +345,7 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let state = self.state_as_arc(); + let state = self.state.clone(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accesed @@ -357,79 +357,17 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, false); } } - - /// Returns a pointer to the inner state. - #[inline] - fn state_ptr(&self) -> *const State { - #[cold] - fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { - let state = Arc::new(State::new()); - let ptr = Arc::into_raw(state).cast_mut(); - if let Err(actual) = atomic_ptr.compare_exchange( - std::ptr::null_mut(), - ptr, - Ordering::AcqRel, - Ordering::Acquire, - ) { - // SAFETY: This was just created from Arc::into_raw. - drop(unsafe { Arc::from_raw(ptr) }); - actual - } else { - ptr - } - } - - let mut ptr = self.state.load(Ordering::Acquire); - if ptr.is_null() { - ptr = alloc_state(&self.state); - } - ptr - } - - /// Returns a reference to the inner state. - #[inline] - fn state(&self) -> &State { - // SAFETY: So long as an Executor lives, it's state pointer will always be valid - // when accessed through state_ptr. - unsafe { &*self.state_ptr() } - } - - // Clones the inner state Arc - #[inline] - fn state_as_arc(&self) -> Arc { - // SAFETY: So long as an Executor lives, it's state pointer will always be a valid - // Arc when accessed through state_ptr. - let arc = unsafe { Arc::from_raw(self.state_ptr()) }; - let clone = arc.clone(); - std::mem::forget(arc); - clone - } } impl Drop for Executor<'_> { fn drop(&mut self) { - let ptr = *self.state.get_mut(); - if ptr.is_null() { - return; - } - - // SAFETY: As ptr is not null, it was allocated via Arc::new and converted - // via Arc::into_raw in state_ptr. - let state = unsafe { Arc::from_raw(ptr) }; - - let mut active = state.active(); + let mut active = self.state.active(); for w in active.drain() { w.wake(); } drop(active); - while state.queue.pop().is_ok() {} - } -} - -impl<'a> Default for Executor<'a> { - fn default() -> Executor<'a> { - Executor::new() + while self.state.queue.pop().is_ok() {} } } @@ -885,7 +823,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { } /// Flushes all of the items from a queue into the thread local queue. -/// +/// /// # Safety /// This must not be accessed at the same time as LOCAL_QUEUE in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { @@ -910,27 +848,7 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Get a reference to the state. - let ptr = executor.state.load(Ordering::Acquire); - if ptr.is_null() { - // The executor has not been initialized. - struct Uninitialized; - - impl fmt::Debug for Uninitialized { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("") - } - } - - return f.debug_tuple(name).field(&Uninitialized).finish(); - } - - // SAFETY: If the state pointer is not null, it must have been - // allocated properly by Arc::new and converted via Arc::into_raw - // in state_ptr. - let state = unsafe { &*ptr }; - - debug_state(state, name, f) + debug_state(&executor.state, name, f) } /// Debug implementation for `Executor` and `LocalExecutor`. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 2c7e142a0a623..118fc1f6f2865 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -32,13 +32,13 @@ pub use async_task::FallibleTask; /// If you require an executor _without_ the `Send` and `Sync` requirements, consider /// using [`LocalExecutor`] instead. #[derive(Deref, DerefMut, Default)] -pub struct Executor<'a>(ExecutorInner<'a>); +pub(crate) struct Executor<'a>(ExecutorInner<'a>); impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { + pub fn new() -> Self { Self(ExecutorInner::new()) } From ab45573f7d44ebe91437ffcd8b7c1a5f659d7a27 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 29 Jul 2025 18:54:57 -0700 Subject: [PATCH 09/68] Address potential unsoundness with ThreadSpawner --- crates/bevy_tasks/src/async_executor.rs | 99 +++++++++++-------------- crates/bevy_tasks/src/executor.rs | 2 +- crates/bevy_tasks/src/task_pool.rs | 30 ++++++-- 3 files changed, 65 insertions(+), 66 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 446ce57306bcb..f76c3399aca57 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -105,8 +105,23 @@ pub struct ThreadSpawner<'a> { } impl<'a> ThreadSpawner<'a> { + /// Spawns a task onto the specific target thread. + pub fn spawn( + &self, + future: impl Future + Send + 'static, + ) -> Task { + // SAFETY: T and `future` are both 'static, so the Task is guaranteed to not outlive it. + unsafe { self.spawn_scoped(future) } + } + /// Spawns a task onto the executor. - pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + /// + /// # Safety + /// The caller must ensure that the returned Task does not outlive 'a. + pub unsafe fn spawn_scoped( + &self, + future: impl Future + Send + 'a, + ) -> Task { let mut active = self.state.active(); // Remove the task from the set of active tasks when the future finishes. @@ -119,24 +134,13 @@ impl<'a> ThreadSpawner<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is `Send`. Therefore we do not need to worry about what thread + // the produced `Runnable` is used and dropped from. + // - `future` is not `'static`, but the caller must make sure that the Task + // and thus the `Runnable` will not outlive `'a`. + // - `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. let (runnable, task) = unsafe { Builder::new() .propagate_panic(true) @@ -209,24 +213,16 @@ impl<'a> Executor<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is `Send`. Therefore we do not need to worry about what thread + // the produced `Runnable` is used and dropped from. + // - `future` is not `'static`, but we make sure that the `Runnable` does + // not outlive `'a`. When the executor is dropped, the `active` field is + // drained and all of the `Waker`s are woken. Then, the queue inside of + // the `Executor` is drained of all of its runnables. This ensures that + // runnables are dropped and this precondition is satisfied. + // - `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. + // Therefore we do not need to worry about what is done with the + // `Waker`. let (runnable, task) = unsafe { Builder::new() .propagate_panic(true) @@ -239,7 +235,7 @@ impl<'a> Executor<'a> { } /// Spawns a non-Send task onto the executor. - pub fn spawn_local(&self, future: impl Future + 'a) -> Task { + pub fn spawn_local(&self, future: impl Future + 'static) -> Task { // Remove the task from the set of active tasks when the future finishes. // // SAFETY: There are no instances where the value is accessed mutably @@ -260,24 +256,13 @@ impl<'a> Executor<'a> { // // SAFETY: // - // If `future` is not `Send`, this must be a `LocalExecutor` as per this - // function's unsafe precondition. Since `LocalExecutor` is `!Sync`, - // `try_tick`, `tick` and `run` can only be called from the origin - // thread of the `LocalExecutor`. Similarly, `spawn` can only be called - // from the origin thread, ensuring that `future` and the executor share - // the same origin thread. The `Runnable` can be scheduled from other - // threads, but because of the above `Runnable` can only be called or - // dropped on the origin thread. - // - // `future` is not `'static`, but we make sure that the `Runnable` does - // not outlive `'a`. When the executor is dropped, the `active` field is - // drained and all of the `Waker`s are woken. Then, the queue inside of - // the `Executor` is drained of all of its runnables. This ensures that - // runnables are dropped and this precondition is satisfied. - // - // `self.schedule()` is `Send`, `Sync` and `'static`, as checked below. - // Therefore we do not need to worry about what is done with the - // `Waker`. + // - `future` is not `Send`, but the produced `Runnable` does is bound + // to thread-local storage and thus cannot leave this thread of execution. + // - `future` is `'static`. + // - `self.schedule_local()` is not `Send` or `Sync` so all instances + // must not leave the current thread of execution, and it does not + // all of them are bound vy use of thread-local storage. + // - `self.schedule_local()` is `'static`, as checked below. let (runnable, task) = Builder::new() .propagate_panic(true) .spawn_unchecked(|()| future, self.schedule_local()); @@ -823,7 +808,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { } /// Flushes all of the items from a queue into the thread local queue. -/// +/// /// # Safety /// This must not be accessed at the same time as LOCAL_QUEUE in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 118fc1f6f2865..ff40adc1befd5 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -31,7 +31,7 @@ pub use async_task::FallibleTask; /// /// If you require an executor _without_ the `Send` and `Sync` requirements, consider /// using [`LocalExecutor`] instead. -#[derive(Deref, DerefMut, Default)] +#[derive(Deref, DerefMut)] pub(crate) struct Executor<'a>(ExecutorInner<'a>); impl Executor<'_> { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index ac31c86359b35..49b9d4ec28aa4 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -474,6 +474,10 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { self.spawned.push(task).unwrap(); } + #[expect( + unsafe_code, + reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + )] /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of /// [`TaskPool::scope`]'s return value. Users should generally prefer to use @@ -481,15 +485,22 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { - let task = self - .scope_spawner - .spawn(AssertUnwindSafe(f).catch_unwind()) - .fallible(); + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + let task = unsafe { + self.scope_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .fallible() + }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); } + #[expect( + unsafe_code, + reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + )] /// Spawns a scoped future onto the thread of the external thread executor. /// This is typically the main thread. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of @@ -498,10 +509,13 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn_on_external + 'scope + Send>(&self, f: Fut) { - let task = self - .external_spawner - .spawn(AssertUnwindSafe(f).catch_unwind()) - .fallible(); + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + let task = unsafe { + self.external_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .fallible() + }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); From 05f1f40c3fe74bd5798875bd80d84bbe6b2eb9a7 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:18:34 -0700 Subject: [PATCH 10/68] Fix some CI errors --- crates/bevy_tasks/src/async_executor.rs | 42 +++++++++++++------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f76c3399aca57..c6b0d1a0bc765 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -3,18 +3,18 @@ reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] -use std::collections::VecDeque; -use std::fmt; -use std::marker::PhantomData; -use std::panic::{RefUnwindSafe, UnwindSafe}; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard, RwLock, TryLockError}; -use std::task::{Context, Poll, Waker}; +use core::marker::PhantomData; +use core::panic::{RefUnwindSafe, UnwindSafe}; +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll, Waker}; use std::thread::ThreadId; +use alloc::collections::VecDeque; +use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; +use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; @@ -31,7 +31,7 @@ pub(crate) fn install_runtime_into_current_thread() { // Do not access this directly, use `with_local_queue` instead. cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { - use std::cell::RefCell; + use core::cell::RefCell; std::thread_local! { static LOCAL_QUEUE: RefCell = const { @@ -42,7 +42,7 @@ cfg_if::cfg_if! { }; } } else { - use std::cell::UnsafeCell; + use core::cell::UnsafeCell; std::thread_local! { static LOCAL_QUEUE: UnsafeCell = const { @@ -56,13 +56,13 @@ cfg_if::cfg_if! { } /// # Safety -/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { LOCAL_QUEUE.with(|tls| { cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { - f(&mut *tls.borrow_mut()) + f(&mut tls.borrow_mut()) } else { // SAFETY: This value is in thread local storage and thus can only be accesed // from one thread. The caller guarantees that this function is not used with @@ -96,6 +96,8 @@ impl Default for ThreadLocalState { } } +/// A task spawner for a specific thread. Must be created by calling [`TaskPool::current_thread_spawner`] +/// from the target thread. #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, @@ -291,7 +293,7 @@ impl<'a> Executor<'a> { // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } - .map(|runnable| runnable.run()) + .map(Runnable::run) .is_some() } @@ -392,7 +394,7 @@ impl State { /// Returns a reference to currently active tasks. fn active(&self) -> MutexGuard<'_, Slab> { - self.active.lock().unwrap_or_else(|e| e.into_inner()) + self.active.lock().unwrap_or_else(PoisonError::into_inner) } /// Notifies a sleeping ticker. @@ -725,7 +727,7 @@ impl Runner<'_> { // Remove this runner's local queue. let iter = - iter.filter(|local| !std::ptr::eq(**local, &self.local_state.stealable_queue)); + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); // Try stealing from each local queue in the list. for local in iter { @@ -773,7 +775,7 @@ impl Drop for Runner<'_> { .iter() .enumerate() .rev() - .find(|(_, local)| std::ptr::eq(**local, &self.local_state.stealable_queue)) + .find(|(_, local)| core::ptr::eq(**local, &self.local_state.stealable_queue)) { stealer_queues.remove(idx); } @@ -789,7 +791,7 @@ impl Drop for Runner<'_> { /// Steals some items from one queue into another. fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Half of `src`'s length rounded up. - let mut count = (src.len() + 1) / 2; + let mut count = src.len().div_ceil(2); if count > 0 { // Don't steal more than fits into the queue. @@ -810,12 +812,12 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { /// Flushes all of the items from a queue into the thread local queue. /// /// # Safety -/// This must not be accessed at the same time as LOCAL_QUEUE in any way. +/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. unsafe fn flush_to_local(src: &ConcurrentQueue) { let count = src.len(); if count > 0 { - // SAFETY: Caller assures that LOCAL_QUEUE does not have any + // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any // overlapping accesses. unsafe { with_local_queue(|tls| { @@ -826,7 +828,7 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { Err(_) => break, } } - }) + }); } } } From 0f226bfd6992232355844926ce5f91867970effb Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:20:38 -0700 Subject: [PATCH 11/68] Update docs --- docs/cargo_features.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/cargo_features.md b/docs/cargo_features.md index 6e409cf998bb7..e23ecf84eb969 100644 --- a/docs/cargo_features.md +++ b/docs/cargo_features.md @@ -14,7 +14,6 @@ The default feature set enables most of the expected features of a game engine, |android-game-activity|Android GameActivity support. Default, choose between this and `android-native-activity`.| |android_shared_stdcxx|Enable using a shared stdlib for cxx on Android| |animation|Enable animation support, and glTF animation loading| -|async_executor|Uses `async-executor` as a task execution backend.| |bevy_animation|Provides animation functionality| |bevy_anti_aliasing|Provides various anti aliasing solutions| |bevy_asset|Provides asset functionality| From 9c2847391368a0a5c3eb91fb625e380656ed9341 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:21:09 -0700 Subject: [PATCH 12/68] Format TOML files --- crates/bevy_tasks/Cargo.toml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index a27379e05b08c..54202a0db7e77 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -28,8 +28,8 @@ multi_threaded = [ ## on `no_std` targets, but provides access to certain additional features on ## supported platforms. std = [ - "futures-lite/std", - "async-task/std", + "futures-lite/std", + "async-task/std", "bevy_platform/std", "fastrand/std", "dep:slab", @@ -44,11 +44,7 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = [ - "bevy_platform/web", - "dep:wasm-bindgen-futures", - "dep:futures-channel", -] +web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel"] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ From 7f8c93204ca444c1193ce953dd6586e12d03aac4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 21:24:14 -0700 Subject: [PATCH 13/68] Make note on ThreadLocal soundness hole --- crates/bevy_tasks/src/async_executor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index c6b0d1a0bc765..cbf83e0daa77e 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -21,6 +21,8 @@ use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; +// ThreadLocalState *must* stay `Sync` due to a currently existing soundness hole. +// See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); pub(crate) fn install_runtime_into_current_thread() { From f84028e0c4efbe7e3358ee472f1dc96cd7b5370f Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 22:51:50 -0700 Subject: [PATCH 14/68] Allow clippy warning --- crates/bevy_ecs/src/schedule/executor/mod.rs | 2 +- .../bevy_ecs/src/schedule/executor/multi_threaded.rs | 12 ++++++------ crates/bevy_render/src/pipelined_rendering.rs | 8 ++++---- crates/bevy_tasks/src/async_executor.rs | 2 +- crates/bevy_tasks/src/executor.rs | 5 +---- crates/bevy_tasks/src/task_pool.rs | 9 +++++---- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/crates/bevy_ecs/src/schedule/executor/mod.rs b/crates/bevy_ecs/src/schedule/executor/mod.rs index 655a71caa70e6..3dce10640d333 100644 --- a/crates/bevy_ecs/src/schedule/executor/mod.rs +++ b/crates/bevy_ecs/src/schedule/executor/mod.rs @@ -11,7 +11,7 @@ use core::any::TypeId; pub use self::{simple::SimpleExecutor, single_threaded::SingleThreadedExecutor}; #[cfg(feature = "std")] -pub use self::multi_threaded::{MainThreadExecutor, MultiThreadedExecutor}; +pub use self::multi_threaded::{MainThreadSpawner, MultiThreadedExecutor}; use fixedbitset::FixedBitSet; diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index 5637a12ab9fa2..b16e6e22a7b53 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -269,7 +269,7 @@ impl SystemExecutor for MultiThreadedExecutor { } let thread_executor = world - .get_resource::() + .get_resource::() .map(|e| e.0.clone()); let environment = &Environment::new(self, schedule, world); @@ -861,20 +861,20 @@ unsafe fn evaluate_and_fold_conditions( .fold(true, |acc, res| acc && res) } -/// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread +/// New-typed [`ThreadSpawner`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadExecutor(pub ThreadSpawner<'static>); +pub struct MainThreadSpawner(pub ThreadSpawner<'static>); -impl Default for MainThreadExecutor { +impl Default for MainThreadSpawner { fn default() -> Self { Self::new() } } -impl MainThreadExecutor { +impl MainThreadSpawner { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadExecutor(ComputeTaskPool::get().current_thread_spawner()) + MainThreadSpawner(ComputeTaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 9769b051bd341..61f9a446a2150 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -3,7 +3,7 @@ use async_channel::{Receiver, Sender}; use bevy_app::{App, AppExit, AppLabel, Plugin, SubApp}; use bevy_ecs::{ resource::Resource, - schedule::MainThreadExecutor, + schedule::MainThreadSpawner, world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; @@ -114,7 +114,7 @@ impl Plugin for PipelinedRenderingPlugin { if app.get_sub_app(RenderApp).is_none() { return; } - app.insert_resource(MainThreadExecutor::new()); + app.insert_resource(MainThreadSpawner::new()); let mut sub_app = SubApp::new(); sub_app.set_extract(renderer_extract); @@ -136,7 +136,7 @@ impl Plugin for PipelinedRenderingPlugin { .expect("Unable to get RenderApp. Another plugin may have removed the RenderApp before PipelinedRenderingPlugin"); // clone main thread executor to render world - let executor = app.world().get_resource::().unwrap(); + let executor = app.world().get_resource::().unwrap(); render_app.world_mut().insert_resource(executor.clone()); render_to_app_sender.send_blocking(render_app).unwrap(); @@ -181,7 +181,7 @@ impl Plugin for PipelinedRenderingPlugin { // This function waits for the rendering world to be received, // runs extract, and then sends the rendering world back to the render thread. fn renderer_extract(app_world: &mut World, _world: &mut World) { - app_world.resource_scope(|world, main_thread_executor: Mut| { + app_world.resource_scope(|world, main_thread_executor: Mut| { world.resource_scope(|world, mut render_channels: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index cbf83e0daa77e..be64d4a851805 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -902,6 +902,7 @@ impl Drop for CallOnDrop { } pin_project! { + #[expect(clippy::unused_unit)] /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] @@ -930,7 +931,6 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { use super::Executor; - use super::ThreadLocalState; use super::THREAD_LOCAL_STATE; fn _ensure_send_and_sync() { diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index ff40adc1befd5..2ac1fe9bde025 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -26,11 +26,8 @@ cfg_if::cfg_if! { pub use async_task::FallibleTask; /// Wrapper around a multi-threading-aware async executor. -/// Spawning will generally require tasks to be `Send` and `Sync` to allow multiple +/// spawning will generally require tasks to be `send` and `sync` to allow multiple /// threads to send/receive/advance tasks. -/// -/// If you require an executor _without_ the `Send` and `Sync` requirements, consider -/// using [`LocalExecutor`] instead. #[derive(Deref, DerefMut)] pub(crate) struct Executor<'a>(ExecutorInner<'a>); diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 49b9d4ec28aa4..866f3deeb65e5 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -136,7 +136,8 @@ pub struct TaskPool { } impl TaskPool { - /// Each thread will only have one `ThreadExecutor`, otherwise, there are good chances they will deadlock + /// Creates a [`ThreadSpawner`] for this current thread of execution. + /// Can be used to spawn new tasks to execute exclusively on this thread. pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { self.executor.current_thread_spawner() } @@ -299,9 +300,9 @@ impl TaskPool { self.scope_with_executor_inner(scope_spawner.clone(), scope_spawner, f) } - /// This allows passing an external executor to spawn tasks on. When you pass an external executor - /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. - /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. + /// This allows passing an external [`ThreadSpawner`] to spawn tasks to. When you pass an external spawner + /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadSpawner`] originated from. + /// If [`None`] is passed the scope will use a [`ThreadSpawner`] that is ticked on the current thread. /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( From f44d302edbad638a584b55bae323ea6e9788a75d Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 29 Jul 2025 22:57:48 -0700 Subject: [PATCH 15/68] Fix typos --- crates/bevy_tasks/src/async_executor.rs | 6 +++--- crates/bevy_tasks/src/task_pool.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index be64d4a851805..d95e6569256d1 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -66,7 +66,7 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { if #[cfg(all(debug_assertions, not(miri)))] { f(&mut tls.borrow_mut()) } else { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. The caller guarantees that this function is not used with // LOCAL_QUEUE in any way. f(unsafe { &mut *tls.get() }) @@ -165,7 +165,7 @@ impl<'a> ThreadSpawner<'a> { let state = self.state.clone(); move |runnable| { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. unsafe { @@ -337,7 +337,7 @@ impl<'a> Executor<'a> { let state = self.state.clone(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { - // SAFETY: This value is in thread local storage and thus can only be accesed + // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. unsafe { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 866f3deeb65e5..d3d9801aaa711 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -477,7 +477,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + reason = "ThreadSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of @@ -500,7 +500,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { #[expect( unsafe_code, - reason = "ThreadSpawner::spawn otherwise requries 'static Futures" + reason = "ThreadSpawner::spawn otherwise requires 'static Futures" )] /// Spawns a scoped future onto the thread of the external thread executor. /// This is typically the main thread. The scope *must* outlive From 227258ebce857aaabd247f2d74df6da0d1885c59 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 30 Jul 2025 00:50:43 -0700 Subject: [PATCH 16/68] Try to fix CI outside of miri --- crates/bevy_tasks/src/async_executor.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index d95e6569256d1..ea79e448b4008 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -2,6 +2,10 @@ unsafe_code, reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] +#![expect( + clippy::unused_unit, + reason = "False positive detection on {Async}CallOnDrop" +)] use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; @@ -100,6 +104,8 @@ impl Default for ThreadLocalState { /// A task spawner for a specific thread. Must be created by calling [`TaskPool::current_thread_spawner`] /// from the target thread. +/// +/// [`TaskPool::current_thread_spawner`]: crate::TaskPool::current_thread_spawner #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, @@ -902,7 +908,6 @@ impl Drop for CallOnDrop { } pin_project! { - #[expect(clippy::unused_unit)] /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] From d88035d5f830e6f2263eb79239836e9c43464d10 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 16:32:18 -0700 Subject: [PATCH 17/68] Remove depnedency on concurrent-queue --- crates/bevy_tasks/Cargo.toml | 3 - crates/bevy_tasks/src/async_executor.rs | 91 ++++++++++++++++--------- crates/bevy_tasks/src/task_pool.rs | 27 +++----- 3 files changed, 68 insertions(+), 53 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 54202a0db7e77..407505e5855b0 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -18,7 +18,6 @@ default = ["std"] multi_threaded = [ "std", "dep:async-channel", - "dep:concurrent-queue", "dep:fastrand", ] @@ -34,7 +33,6 @@ std = [ "fastrand/std", "dep:slab", "dep:thread_local", - "dep:concurrent-queue", "dep:pin-project-lite", ] @@ -66,7 +64,6 @@ thread_local = { version = "1.1", optional = true } fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } -concurrent-queue = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } crossbeam-queue = { version = "0.3", default-features = false, features = [ "alloc", diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index ea79e448b4008..a413444ba1468 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -19,7 +19,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use concurrent_queue::ConcurrentQueue; +use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future, prelude::*}; use pin_project_lite::pin_project; use slab::Slab; @@ -87,8 +87,8 @@ struct LocalQueue { struct ThreadLocalState { executor_thread: AtomicBool, thread_id: ThreadId, - stealable_queue: ConcurrentQueue, - thread_locked_queue: ConcurrentQueue, + stealable_queue: ArrayQueue, + thread_locked_queue: SegQueue, } impl Default for ThreadLocalState { @@ -96,8 +96,8 @@ impl Default for ThreadLocalState { Self { executor_thread: AtomicBool::new(false), thread_id: std::thread::current().id(), - stealable_queue: ConcurrentQueue::bounded(512), - thread_locked_queue: ConcurrentQueue::unbounded(), + stealable_queue: ArrayQueue::new(512), + thread_locked_queue: SegQueue::new(), } } } @@ -109,7 +109,7 @@ impl Default for ThreadLocalState { #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, - target_queue: &'static ConcurrentQueue, + target_queue: &'static SegQueue, state: Arc, _marker: PhantomData<&'a ()>, } @@ -161,7 +161,7 @@ impl<'a> ThreadSpawner<'a> { // Instead of directly scheduling this task, it's put into the onto the // thread locked queue to be moved to the target thread, where it will // either be run immediately or flushed into the thread's local queue. - self.target_queue.push(runnable).unwrap(); + self.target_queue.push(runnable); task } @@ -324,7 +324,7 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, true); return; } - Err(r) => r.into_inner(), + Err(r) => r, } } else { runnable @@ -333,7 +333,7 @@ impl<'a> Executor<'a> { runnable }; // Otherwise push onto the global queue instead. - state.queue.push(runnable).unwrap(); + state.queue.push(runnable); state.notify(); } } @@ -362,17 +362,17 @@ impl Drop for Executor<'_> { } drop(active); - while self.state.queue.pop().is_ok() {} + while self.state.queue.pop().is_some() {} } } /// The state of a executor. struct State { /// The global queue. - queue: ConcurrentQueue, + queue: SegQueue, /// Local queues created by runners. - stealer_queues: RwLock>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -388,7 +388,7 @@ impl State { /// Creates state for a new executor. const fn new() -> State { State { - queue: ConcurrentQueue::unbounded(), + queue: SegQueue::new(), stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { @@ -711,12 +711,12 @@ impl Runner<'_> { } // Try the local queue. - if let Ok(r) = self.local_state.stealable_queue.pop() { + if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. - if let Ok(r) = self.state.queue.pop() { + if let Some(r) = self.state.queue.pop() { steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } @@ -739,13 +739,13 @@ impl Runner<'_> { // Try stealing from each local queue in the list. for local in iter { - steal(local, &self.local_state.stealable_queue); - if let Ok(r) = self.local_state.stealable_queue.pop() { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } } - if let Ok(r) = self.local_state.thread_locked_queue.pop() { + if let Some(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. // @@ -790,29 +790,54 @@ impl Drop for Runner<'_> { } // Re-schedule remaining tasks in the local queue. - while let Ok(r) = self.local_state.stealable_queue.pop() { + while let Some(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } } +trait WorkQueue { + fn stealable_count(&self) -> usize; + fn queue_pop(&self) -> Option; +} + +impl WorkQueue for ArrayQueue { + #[inline] + fn stealable_count(&self) -> usize { + self.len().div_ceil(2) + } + + #[inline] + fn queue_pop(&self) -> Option { + self.pop() + } +} + +impl WorkQueue for SegQueue { + #[inline] + fn stealable_count(&self) -> usize { + self.len() + } + + #[inline] + fn queue_pop(&self) -> Option { + self.pop() + } +} + /// Steals some items from one queue into another. -fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { +fn steal>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. - let mut count = src.len().div_ceil(2); + let mut count = src.stealable_count(); if count > 0 { // Don't steal more than fits into the queue. - if let Some(cap) = dest.capacity() { - count = count.min(cap - dest.len()); - } + count = count.min(dest.capacity() - dest.len()); // Steal tasks. for _ in 0..count { - match src.pop() { - Ok(t) => assert!(dest.push(t).is_ok()), - Err(_) => break, - } + let Some(val) = src.queue_pop() else { break }; + assert!(dest.push(val).is_ok()); } } } @@ -821,7 +846,7 @@ fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { /// /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. -unsafe fn flush_to_local(src: &ConcurrentQueue) { +unsafe fn flush_to_local(src: &SegQueue) { let count = src.len(); if count > 0 { @@ -831,10 +856,8 @@ unsafe fn flush_to_local(src: &ConcurrentQueue) { with_local_queue(|tls| { // Steal tasks. for _ in 0..count { - match src.pop() { - Ok(t) => tls.local_queue.push_front(t), - Err(_) => break, - } + let Some(val) = src.queue_pop() else { break }; + tls.local_queue.push_front(val); } }); } @@ -862,7 +885,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index d3d9801aaa711..972b554783b98 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -5,7 +5,7 @@ use std::thread::{self, JoinHandle}; use crate::async_executor::ThreadSpawner; use crate::executor::FallibleTask; use bevy_platform::sync::Arc; -use concurrent_queue::ConcurrentQueue; +use crossbeam_queue::SegQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; @@ -345,13 +345,12 @@ impl TaskPool { let executor: &crate::executor::Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - let spawned: ConcurrentQueue>>> = - ConcurrentQueue::unbounded(); + let spawned: SegQueue>>> = + SegQueue::new(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env ConcurrentQueue< - FallibleTask>>, - > = unsafe { mem::transmute(&spawned) }; + let spawned: &'env SegQueue>>> = + unsafe { mem::transmute(&spawned) }; let scope = Scope { executor, @@ -373,7 +372,7 @@ impl TaskPool { } else { block_on(self.executor.run(async move { let mut results = Vec::with_capacity(spawned.len()); - while let Ok(task) = spawned.pop() { + while let Some(task) = spawned.pop() { if let Some(res) = task.await { match res { Ok(res) => results.push(res), @@ -450,7 +449,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope crate::executor::Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope ConcurrentQueue>>>, + spawned: &'scope SegQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, @@ -470,9 +469,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .executor .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); - // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } #[expect( @@ -493,9 +490,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) .fallible() }; - // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } #[expect( @@ -519,7 +514,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { }; // ConcurrentQueue only errors when closed or full, but we never // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task).unwrap(); + self.spawned.push(task); } } @@ -529,7 +524,7 @@ where { fn drop(&mut self) { block_on(async { - while let Ok(task) = self.spawned.pop() { + while let Some(task) = self.spawned.pop() { task.cancel().await; } }); From 202873615b32082712e0a218431e167596ec066c Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 16:40:22 -0700 Subject: [PATCH 18/68] Revert "Clean up unnecessary unsafe use" This reverts commit 8f1eaa71bcd28e79676e5930f29a2afe7daa6b79. --- crates/bevy_tasks/src/async_executor.rs | 101 +++++++++++++++++++++--- crates/bevy_tasks/src/executor.rs | 2 +- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index a413444ba1468..916ac6878a129 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -10,9 +10,8 @@ use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; -use std::thread::ThreadId; use alloc::collections::VecDeque; use alloc::fmt; @@ -185,7 +184,7 @@ impl<'a> ThreadSpawner<'a> { /// An async executor. pub struct Executor<'a> { /// The executor state. - state: Arc, + state: AtomicPtr, /// Makes the `'a` lifetime invariant. _marker: PhantomData<&'a ()>, @@ -202,21 +201,21 @@ impl fmt::Debug for Executor<'_> { impl<'a> Executor<'a> { /// Creates a new executor. - pub fn new() -> Executor<'a> { + pub const fn new() -> Executor<'a> { Executor { - state: Arc::new(State::new()), + state: AtomicPtr::new(std::ptr::null_mut()), _marker: PhantomData, } } /// Spawns a task onto the executor. pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state.active(); + let mut active = self.state().active(); // Remove the task from the set of active tasks when the future finishes. let entry = active.vacant_entry(); let index = entry.key(); - let state = self.state.clone(); + let state = self.state_as_arc(); let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); // Create the task and register it in the set of active tasks. @@ -290,7 +289,7 @@ impl<'a> Executor<'a> { ThreadSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, - state: self.state.clone(), + state: self.state_as_arc(), _marker: PhantomData, } } @@ -307,12 +306,12 @@ impl<'a> Executor<'a> { /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { - self.state.run(future).await + self.state().run(future).await } /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { - let state = self.state.clone(); + let state = self.state_as_arc(); move |runnable| { // Attempt to push onto the local queue first in dedicated executor threads, @@ -340,7 +339,7 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let state = self.state.clone(); + let state = self.state_as_arc(); let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed @@ -352,11 +351,67 @@ impl<'a> Executor<'a> { state.notify_specific_thread(local_state.thread_id, false); } } + + /// Returns a pointer to the inner state. + #[inline] + fn state_ptr(&self) -> *const State { + #[cold] + fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { + let state = Arc::new(State::new()); + let ptr = Arc::into_raw(state).cast_mut(); + if let Err(actual) = atomic_ptr.compare_exchange( + std::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) { + // SAFETY: This was just created from Arc::into_raw. + drop(unsafe { Arc::from_raw(ptr) }); + actual + } else { + ptr + } + } + + let mut ptr = self.state.load(Ordering::Acquire); + if ptr.is_null() { + ptr = alloc_state(&self.state); + } + ptr + } + + /// Returns a reference to the inner state. + #[inline] + fn state(&self) -> &State { + // SAFETY: So long as an Executor lives, it's state pointer will always be valid + // when accessed through state_ptr. + unsafe { &*self.state_ptr() } + } + + // Clones the inner state Arc + #[inline] + fn state_as_arc(&self) -> Arc { + // SAFETY: So long as an Executor lives, it's state pointer will always be a valid + // Arc when accessed through state_ptr. + let arc = unsafe { Arc::from_raw(self.state_ptr()) }; + let clone = arc.clone(); + std::mem::forget(arc); + clone + } } impl Drop for Executor<'_> { fn drop(&mut self) { - let mut active = self.state.active(); + let ptr = *self.state.get_mut(); + if ptr.is_null() { + return; + } + + // SAFETY: As ptr is not null, it was allocated via Arc::new and converted + // via Arc::into_raw in state_ptr. + let state = unsafe { Arc::from_raw(ptr) }; + + let mut active = state.active(); for w in active.drain() { w.wake(); } @@ -866,7 +921,27 @@ unsafe fn flush_to_local(src: &SegQueue) { /// Debug implementation for `Executor` and `LocalExecutor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { - debug_state(&executor.state, name, f) + // Get a reference to the state. + let ptr = executor.state.load(Ordering::Acquire); + if ptr.is_null() { + // The executor has not been initialized. + struct Uninitialized; + + impl fmt::Debug for Uninitialized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + + return f.debug_tuple(name).field(&Uninitialized).finish(); + } + + // SAFETY: If the state pointer is not null, it must have been + // allocated properly by Arc::new and converted via Arc::into_raw + // in state_ptr. + let state = unsafe { &*ptr }; + + debug_state(state, name, f) } /// Debug implementation for `Executor` and `LocalExecutor`. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index 2ac1fe9bde025..d8f6f862ed66d 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -35,7 +35,7 @@ impl Executor<'_> { /// Construct a new [`Executor`] #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] #[allow(dead_code, reason = "not all feature flags require this function")] - pub fn new() -> Self { + pub const fn new() -> Self { Self(ExecutorInner::new()) } From da66005b058efc5d49787913571d31be689699d9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 17:02:18 -0700 Subject: [PATCH 19/68] Fix single-threaded builds --- crates/bevy_tasks/src/async_executor.rs | 44 ++++++++++++++++++- .../src/single_threaded_task_pool.rs | 5 ++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 916ac6878a129..f48670943984f 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -6,12 +6,17 @@ clippy::unused_unit, reason = "False positive detection on {Async}CallOnDrop" )] +#![expect( + dead_code, + reason = "Not every function is used in all feature combinations." +)] use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; +use std::thread::ThreadId; use alloc::collections::VecDeque; use alloc::fmt; @@ -245,6 +250,18 @@ impl<'a> Executor<'a> { /// Spawns a non-Send task onto the executor. pub fn spawn_local(&self, future: impl Future + 'static) -> Task { + // SAFETY: future is 'static + unsafe { self.spawn_local_scoped(future) } + } + + /// Spawns a non-'static and non-Send task onto the executor. + /// + /// # Safety + /// The caller must ensure that the returned Task does not outlive 'a. + pub unsafe fn spawn_local_scoped( + &self, + future: impl Future + 'a, + ) -> Task { // Remove the task from the set of active tasks when the future finishes. // // SAFETY: There are no instances where the value is accessed mutably @@ -267,7 +284,9 @@ impl<'a> Executor<'a> { // // - `future` is not `Send`, but the produced `Runnable` does is bound // to thread-local storage and thus cannot leave this thread of execution. - // - `future` is `'static`. + // - `future` may not be `'static`, but the caller is required to ensure that + // the future does not outlive the borrowed non-metadata variables of the + // task. // - `self.schedule_local()` is not `Send` or `Sync` so all instances // must not leave the current thread of execution, and it does not // all of them are bound vy use of thread-local storage. @@ -294,6 +313,27 @@ impl<'a> Executor<'a> { } } + /// Attempts to run a task if at least one is scheduled. + /// + /// Running a scheduled task means simply polling its future once. + pub fn try_tick(&self) -> bool { + let state = self.state(); + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } + .or_else(|| state.queue.pop()) + .or_else(|| { + THREAD_LOCAL_STATE + .get_or_default() + .thread_locked_queue + .pop() + }) + .map(Runnable::run) + .is_some() + } + pub fn try_tick_local() -> bool { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. As the Runnable is run after @@ -417,7 +457,7 @@ impl Drop for Executor<'_> { } drop(active); - while self.state.queue.pop().is_some() {} + while state.queue.pop().is_some() {} } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 33dd8f3c0acd5..4755c95efd31c 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -254,6 +254,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -274,7 +275,9 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; - self.executor.spawn_local(f).detach(); + // SAFETY: The surrounding scope will not terminate until all local tasks are done + // ensuring that the borrowed variables do not outlive the detatched task. + unsafe { self.executor.spawn_local_scoped(f).detach() }; } } From 8d4e5cabb7aca09f9d23d6e14aef0a9c20a90b6d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:00:17 -0700 Subject: [PATCH 20/68] Fix builds for no_std builds --- crates/bevy_tasks/Cargo.toml | 6 +- crates/bevy_tasks/src/async_executor.rs | 25 +------- crates/bevy_tasks/src/edge_executor.rs | 9 +++ crates/bevy_tasks/src/executor.rs | 5 -- crates/bevy_tasks/src/lib.rs | 5 +- .../src/single_threaded_task_pool.rs | 60 +++++++++++-------- crates/bevy_tasks/src/task_pool.rs | 9 ++- crates/bevy_tasks/src/usages.rs | 3 +- 8 files changed, 58 insertions(+), 64 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 407505e5855b0..b4830a45674e7 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -15,11 +15,7 @@ default = ["std"] ## Enables multi-threading support. ## Without this feature, all tasks will be run on a single thread. -multi_threaded = [ - "std", - "dep:async-channel", - "dep:fastrand", -] +multi_threaded = ["std", "dep:async-channel", "dep:fastrand"] # Platform Compatibility diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index f48670943984f..9d6677118951b 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -6,9 +6,9 @@ clippy::unused_unit, reason = "False positive detection on {Async}CallOnDrop" )] -#![expect( +#![allow( dead_code, - reason = "Not every function is used in all feature combinations." + reason = "Not all functions are used with every feature combination" )] use core::marker::PhantomData; @@ -313,27 +313,6 @@ impl<'a> Executor<'a> { } } - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - pub fn try_tick(&self) -> bool { - let state = self.state(); - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. As the Runnable is run after - // this scope closes, the AsyncCallOnDrop around the future will be invoked - // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } - .or_else(|| state.queue.pop()) - .or_else(|| { - THREAD_LOCAL_STATE - .get_or_default() - .thread_locked_queue - .pop() - }) - .map(Runnable::run) - .is_some() - } - pub fn try_tick_local() -> bool { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. As the Runnable is run after diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 70e11c8a433cf..c182fc230770e 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -97,6 +97,15 @@ impl<'a, const C: usize> Executor<'a, C> { unsafe { self.spawn_unchecked(fut) } } + pub fn spawn_local(&self, fut: F) -> Task + where + F: Future + Send + 'a, + F::Output: Send + 'a, + { + // SAFETY: Original implementation missing safety documentation + unsafe { self.spawn_unchecked(fut) } + } + /// Attempts to run a task if at least one is scheduled. /// /// Running a scheduled task means simply polling its future once. diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs index d8f6f862ed66d..e8cb779e0355d 100644 --- a/crates/bevy_tasks/src/executor.rs +++ b/crates/bevy_tasks/src/executor.rs @@ -38,11 +38,6 @@ impl Executor<'_> { pub const fn new() -> Self { Self(ExecutorInner::new()) } - - #[inline] - pub fn try_tick_local() -> bool { - ExecutorInner::try_tick_local() - } } impl UnwindSafe for Executor<'_> {} diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 2e04705b13fb0..2ca5e8c50c241 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -61,12 +61,11 @@ cfg_if::cfg_if! { if #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { mod task_pool; - pub use task_pool::{Scope, TaskPool, TaskPoolBuilder}; - pub use async_executor::ThreadSpawner; + pub use task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; } else if #[cfg(any(target_arch = "wasm32", not(feature = "multi_threaded")))] { mod single_threaded_task_pool; - pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; + pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadSpawner}; } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 4755c95efd31c..bab9a7c4ce8e1 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -4,18 +4,19 @@ use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; -#[cfg(not(feature = "std"))] -use bevy_platform::sync::{Mutex, PoisonError}; - use crate::executor::Executor; static EXECUTOR: Executor = Executor::new(); -#[cfg(feature = "std")] -type ScopeResult = alloc::rc::Rc>>; +cfg_if::cfg_if! { + if #[cfg(feature = "std")] { + type ScopeResult = alloc::rc::Rc>>; + } else { + use bevy_platform::sync::{Mutex, PoisonError}; -#[cfg(not(feature = "std"))] -type ScopeResult = Arc>>; + type ScopeResult = Arc>>; + } +} /// Used to create a [`TaskPool`]. #[derive(Debug, Default, Clone)] @@ -27,8 +28,8 @@ pub struct TaskPoolBuilder {} /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread /// and so the [`ThreadExecutor`] does nothing. #[derive(Default)] -pub struct ThreadExecutor<'a>(PhantomData<&'a ()>); -impl<'a> ThreadExecutor<'a> { +pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); +impl<'a> ThreadSpawner<'a> { /// Creates a new `ThreadExecutor` pub fn new() -> Self { Self::default() @@ -79,8 +80,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn get_thread_executor() -> Arc> { - Arc::new(ThreadExecutor::new()) + pub fn get_thread_executor() -> Arc> { + Arc::new(ThreadSpawner::new()) } /// Create a `TaskPool` with the default configuration. @@ -119,7 +120,7 @@ impl TaskPool { pub fn scope_with_executor<'env, F, T>( &self, _tick_task_pool_executor: bool, - _thread_executor: Option<&ThreadExecutor>, + _thread_executor: Option<&ThreadSpawner>, f: F, ) -> Vec where @@ -155,7 +156,7 @@ impl TaskPool { f(scope_ref); // Loop until all tasks are done - while executor.try_tick() {} + while Self::try_tick_local() {} let results = scope.results.borrow(); results @@ -189,16 +190,10 @@ impl TaskPool { cfg_if::cfg_if! { if #[cfg(all(target_arch = "wasm32", feature = "web"))] { Task::wrap_future(future) - } else if #[cfg(feature = "std")] { - let task = EXECUTOR.spawn_local(future); - // Loop until all tasks are done - while EXECUTOR.try_tick() {} - - Task::new(task) } else { - EXECUTOR.spawn_local(future); + let task = EXECUTOR.spawn_local(future); // Loop until all tasks are done - while EXECUTOR.try_tick() {} + while Self::try_tick_local() {} Task::new(task) } @@ -215,6 +210,16 @@ impl TaskPool { { self.spawn(future) } + + pub(crate) fn try_tick_local() -> bool { + cfg_if::cfg_if! { + if #[cfg(feature = "std")] { + crate::async_executor::Executor::try_tick_local() + } else { + EXECUTOR.try_tick() + } + } + } } /// A `TaskPool` scope for running one or more non-`'static` futures. @@ -254,7 +259,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } - #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] + #[allow(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -275,9 +280,16 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { *lock = Some(temp_result); } }; + + #[cfg(feature = "std")] // SAFETY: The surrounding scope will not terminate until all local tasks are done - // ensuring that the borrowed variables do not outlive the detatched task. - unsafe { self.executor.spawn_local_scoped(f).detach() }; + // ensuring that the borrowed variables do not outlive the detached task. + unsafe { + self.executor.spawn_local_scoped(f).detach() + }; + + #[cfg(not(feature = "std"))] + self.executor.spawn(f).detach(); } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 972b554783b98..ab827f95812e3 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,14 +2,15 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; use std::thread::{self, JoinHandle}; -use crate::async_executor::ThreadSpawner; -use crate::executor::FallibleTask; +use crate::{async_executor::Executor, executor::FallibleTask}; use bevy_platform::sync::Arc; use crossbeam_queue::SegQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; +pub use crate::async_executor::ThreadSpawner; + struct CallOnDrop(Option>); impl Drop for CallOnDrop { @@ -419,6 +420,10 @@ impl TaskPool { { Task::new(self.executor.spawn_local(future)) } + + pub(crate) fn try_tick_local() -> bool { + Executor::try_tick_local() + } } impl Default for TaskPool { diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 8bd0bd3182cd2..a4d614a297f0c 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,5 +1,4 @@ use super::TaskPool; -use crate::executor::Executor; use bevy_platform::sync::OnceLock; use core::ops::Deref; @@ -85,7 +84,7 @@ taskpool! { #[cfg(not(all(target_arch = "wasm32", feature = "web")))] pub fn tick_global_task_pools_on_main_thread() { for _ in 0..100 { - if !Executor::try_tick_local() { + if !TaskPool::try_tick_local() { break; } } From 3cb694f598f320c42496fbcab51fc507aa116852 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:04:20 -0700 Subject: [PATCH 21/68] Shut up Clippy --- crates/bevy_tasks/src/async_executor.rs | 6 +++--- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 9d6677118951b..dde1c114f1e3b 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -208,7 +208,7 @@ impl<'a> Executor<'a> { /// Creates a new executor. pub const fn new() -> Executor<'a> { Executor { - state: AtomicPtr::new(std::ptr::null_mut()), + state: AtomicPtr::new(core::ptr::null_mut()), _marker: PhantomData, } } @@ -379,7 +379,7 @@ impl<'a> Executor<'a> { let state = Arc::new(State::new()); let ptr = Arc::into_raw(state).cast_mut(); if let Err(actual) = atomic_ptr.compare_exchange( - std::ptr::null_mut(), + core::ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire, @@ -414,7 +414,7 @@ impl<'a> Executor<'a> { // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; let clone = arc.clone(); - std::mem::forget(arc); + core::mem::forget(arc); clone } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index bab9a7c4ce8e1..b839b180cd654 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -27,7 +27,7 @@ pub struct TaskPoolBuilder {} /// tasks on a specific thread. But the wasm task pool just calls /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread /// and so the [`ThreadExecutor`] does nothing. -#[derive(Default)] +#[derive(Default, Clone)] pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); impl<'a> ThreadSpawner<'a> { /// Creates a new `ThreadExecutor` From d0ef3f4584272af875097094ff62079964919c80 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:11:47 -0700 Subject: [PATCH 22/68] Fix bevy_ecs builds --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index b839b180cd654..b3009ab2b9381 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -108,7 +108,7 @@ impl TaskPool { F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), T: Send + 'static, { - self.scope_with_executor(false, None, f) + self.scope_with_executor(None, f) } /// Allows spawning non-`'static` futures on the thread pool. The function takes a callback, @@ -119,7 +119,6 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] pub fn scope_with_executor<'env, F, T>( &self, - _tick_task_pool_executor: bool, _thread_executor: Option<&ThreadSpawner>, f: F, ) -> Vec From 25233ffa53ce554b5f4ab22832ce1f44fcf2b11d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 18:14:50 -0700 Subject: [PATCH 23/68] Whoops --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index b3009ab2b9381..45d9a6720dac8 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -80,8 +80,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn get_thread_executor() -> Arc> { - Arc::new(ThreadSpawner::new()) + pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + ThreadSpawner::new() } /// Create a `TaskPool` with the default configuration. @@ -119,7 +119,7 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] pub fn scope_with_executor<'env, F, T>( &self, - _thread_executor: Option<&ThreadSpawner>, + _thread_executor: Option, f: F, ) -> Vec where From 69a13893d4df1b7b1b139f25711b90e02fdc8ce4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 19:05:42 -0700 Subject: [PATCH 24/68] Arc is only used when no_std --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 45d9a6720dac8..e23e18f9a7307 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,5 +1,4 @@ use alloc::{string::String, vec::Vec}; -use bevy_platform::sync::Arc; use core::{cell::RefCell, future::Future, marker::PhantomData, mem}; use crate::Task; @@ -13,6 +12,7 @@ cfg_if::cfg_if! { type ScopeResult = alloc::rc::Rc>>; } else { use bevy_platform::sync::{Mutex, PoisonError}; + use bevy_platform::sync::Arc; type ScopeResult = Arc>>; } From 943f13511f7ee5e5ee74aca71d1aea9eec6b9512 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 19:51:45 -0700 Subject: [PATCH 25/68] Remove now unused mentions of LocalExecutor --- crates/bevy_tasks/src/async_executor.rs | 23 +--- crates/bevy_tasks/src/edge_executor.rs | 175 ------------------------ 2 files changed, 2 insertions(+), 196 deletions(-) diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index dde1c114f1e3b..932962d4563f6 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -82,7 +82,6 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { } }) } - struct LocalQueue { local_queue: VecDeque, local_active: Slab, @@ -938,7 +937,7 @@ unsafe fn flush_to_local(src: &SegQueue) { } } -/// Debug implementation for `Executor` and `LocalExecutor`. +/// Debug implementation for `Executor`. fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Get a reference to the state. let ptr = executor.state.load(Ordering::Acquire); @@ -963,7 +962,7 @@ fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_ debug_state(state, name, f) } -/// Debug implementation for `Executor` and `LocalExecutor`. +/// Debug implementation for `Executor`. fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// Debug wrapper for the number of active tasks. struct ActiveTasks<'a>(&'a Mutex>); @@ -1071,23 +1070,5 @@ mod test { is_sync(ex.current_thread_spawner()); is_send(THREAD_LOCAL_STATE.get_or_default()); is_sync(THREAD_LOCAL_STATE.get_or_default()); - - /// ```compile_fail - /// use crate::async_executor::LocalExecutor; - /// use futures_lite::future::pending; - /// - /// fn is_send(_: T) {} - /// fn is_sync(_: T) {} - /// - /// is_send::>(LocalExecutor::new()); - /// is_sync::>(LocalExecutor::new()); - /// - /// let ex = LocalExecutor::new(); - /// is_send(ex.run(pending::<()>())); - /// is_sync(ex.run(pending::<()>())); - /// is_send(ex.tick()); - /// is_sync(ex.tick()); - /// ``` - fn _negative_test() {} } } diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index c182fc230770e..800ae57a70798 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -307,141 +307,6 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} -/// A thread-local executor. -/// -/// The executor can only be run on the thread that created it. -/// -/// # Examples -/// -/// ```ignore -/// use edge_executor::{LocalExecutor, block_on}; -/// -/// let local_ex: LocalExecutor = Default::default(); -/// -/// block_on(local_ex.run(async { -/// println!("Hello world!"); -/// })); -/// ``` -pub struct LocalExecutor<'a, const C: usize = 64> { - executor: Executor<'a, C>, - _not_send: PhantomData>>, -} - -impl<'a, const C: usize> LocalExecutor<'a, C> { - /// Creates a single-threaded executor. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// ``` - pub const fn new() -> Self { - Self { - executor: Executor::::new(), - _not_send: PhantomData, - } - } - - /// Spawns a task onto the executor. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// ``` - /// - /// Note that if the executor's queue size is equal to the number of currently - /// spawned and running tasks, spawning this additional task might cause the executor to panic - /// later, when the task is scheduled for polling. - pub fn spawn(&self, fut: F) -> Task - where - F: Future + 'a, - F::Output: 'a, - { - // SAFETY: Original implementation missing safety documentation - unsafe { self.executor.spawn_unchecked(fut) } - } - - /// Attempts to run a task if at least one is scheduled. - /// - /// Running a scheduled task means simply polling its future once. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::LocalExecutor; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// assert!(!local_ex.try_tick()); // no tasks to run - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// assert!(local_ex.try_tick()); // a task was found - /// ``` - pub fn try_tick(&self) -> bool { - self.executor.try_tick() - } - - /// Runs a single task asynchronously. - /// - /// Running a task means simply polling its future once. - /// - /// If no tasks are scheduled when this method is called, it will wait until one is scheduled. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::{LocalExecutor, block_on}; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { - /// println!("Hello world"); - /// }); - /// block_on(local_ex.tick()); // runs the task - /// ``` - pub async fn tick(&self) { - self.executor.tick().await; - } - - /// Runs the executor asynchronously until the given future completes. - /// - /// # Examples - /// - /// ```ignore - /// use edge_executor::{LocalExecutor, block_on}; - /// - /// let local_ex: LocalExecutor = Default::default(); - /// - /// let task = local_ex.spawn(async { 1 + 2 }); - /// let res = block_on(local_ex.run(async { task.await * 2 })); - /// - /// assert_eq!(res, 6); - /// ``` - pub async fn run(&self, fut: F) -> F::Output - where - F: Future, - { - // SAFETY: Original implementation missing safety documentation - unsafe { self.executor.run_unchecked(fut) }.await - } -} - -impl<'a, const C: usize> Default for LocalExecutor<'a, C> { - fn default() -> Self { - Self::new() - } -} - struct State { #[cfg(all( target_has_atomic = "8", @@ -486,46 +351,6 @@ impl State { } } -#[cfg(test)] -mod different_executor_tests { - use core::cell::Cell; - - use futures_lite::future::{block_on, pending, poll_once}; - use futures_lite::pin; - - use super::LocalExecutor; - - #[test] - fn shared_queue_slot() { - block_on(async { - let was_polled = Cell::new(false); - let future = async { - was_polled.set(true); - pending::<()>().await; - }; - - let ex1: LocalExecutor = Default::default(); - let ex2: LocalExecutor = Default::default(); - - // Start the futures for running forever. - let (run1, run2) = (ex1.run(pending::<()>()), ex2.run(pending::<()>())); - pin!(run1); - pin!(run2); - assert!(poll_once(run1.as_mut()).await.is_none()); - assert!(poll_once(run2.as_mut()).await.is_none()); - - // Spawn the future on executor one and then poll executor two. - ex1.spawn(future).detach(); - assert!(poll_once(run2).await.is_none()); - assert!(!was_polled.get()); - - // Poll the first one. - assert!(poll_once(run1).await.is_none()); - assert!(was_polled.get()); - }); - } -} - #[cfg(test)] mod drop_tests { use alloc::string::String; From 5b9a40eb011f4ec8f85d1bf49ac13a63f89fce7e Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:06:05 -0700 Subject: [PATCH 26/68] Remove unused import --- crates/bevy_tasks/src/edge_executor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 800ae57a70798..b2cd152e79c84 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -13,7 +13,6 @@ // TODO: Create a more tailored replacement, possibly integrating [Fotre](https://github.com/NthTensor/Forte) -use alloc::rc::Rc; use core::{ future::{poll_fn, Future}, marker::PhantomData, From 355c96f9fdd32d8a8b843cdfddcf995345e33a69 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:19:10 -0700 Subject: [PATCH 27/68] Use pin_project_lite for web builds instead of pin_project --- crates/bevy_tasks/Cargo.toml | 2 +- crates/bevy_tasks/src/async_executor.rs | 3 +-- crates/bevy_tasks/src/wasm_task.rs | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index b4830a45674e7..77c385ae5e279 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -38,7 +38,7 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel"] +web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel", "dep:pin-project-lite"] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ diff --git a/crates/bevy_tasks/src/async_executor.rs b/crates/bevy_tasks/src/async_executor.rs index 932962d4563f6..a90a193f323cf 100644 --- a/crates/bevy_tasks/src/async_executor.rs +++ b/crates/bevy_tasks/src/async_executor.rs @@ -25,7 +25,6 @@ use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future, prelude::*}; -use pin_project_lite::pin_project; use slab::Slab; use thread_local::ThreadLocal; @@ -1023,7 +1022,7 @@ impl Drop for CallOnDrop { } } -pin_project! { +pin_project_lite::pin_project! { /// A wrapper around a future, running a closure when dropped. struct AsyncCallOnDrop { #[pin] diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index 0cc569c47913d..f2f804a1d80b1 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -74,8 +74,9 @@ impl Future for Task { type Panic = Box; -#[pin_project::pin_project] -struct CatchUnwind(#[pin] F); +pin_project_lite::pin_project! { + struct CatchUnwind(#[pin] F); +} impl Future for CatchUnwind { type Output = Result; From 564daf6d4822dc3751a80c7fe6f0da55e2b734f3 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:21:31 -0700 Subject: [PATCH 28/68] Fix TOML formatting --- crates/bevy_tasks/Cargo.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 77c385ae5e279..f83dcaf041db5 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -38,7 +38,12 @@ critical-section = ["bevy_platform/critical-section"] ## Enables use of browser APIs. ## Note this is currently only applicable on `wasm32` architectures. -web = ["bevy_platform/web", "dep:wasm-bindgen-futures", "dep:futures-channel", "dep:pin-project-lite"] +web = [ + "bevy_platform/web", + "dep:wasm-bindgen-futures", + "dep:futures-channel", + "dep:pin-project-lite", +] [dependencies] bevy_platform = { path = "../bevy_platform", version = "0.17.0-dev", default-features = false, features = [ From b100634a65f66371d8850719715eb3bfa164f7c9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 20:41:51 -0700 Subject: [PATCH 29/68] Make CatchUnwind pin_project_lite friendly --- crates/bevy_tasks/src/wasm_task.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index f2f804a1d80b1..a85d0dd8b0f8d 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -21,7 +21,7 @@ impl Task { wasm_bindgen_futures::spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind(AssertUnwindSafe(future)).await; + let value = CatchUnwind { inner: future }.await; let _ = sender.send(value); }); Self(receiver.into_future()) @@ -75,13 +75,16 @@ impl Future for Task { type Panic = Box; pin_project_lite::pin_project! { - struct CatchUnwind(#[pin] F); + struct CatchUnwind { + #[pin] + inner: F + } } impl Future for CatchUnwind { type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let f = AssertUnwindSafe(|| self.project().0.poll(cx)); + let f = AssertUnwindSafe(|| self.project().inner.poll(cx)); #[cfg(feature = "std")] let result = std::panic::catch_unwind(f)?; From 8700f7eb12474bff6ae3b379336fac00c8909ebc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sat, 2 Aug 2025 22:02:54 -0700 Subject: [PATCH 30/68] Add missing AssertUnwindSafe --- crates/bevy_tasks/src/wasm_task.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/wasm_task.rs b/crates/bevy_tasks/src/wasm_task.rs index a85d0dd8b0f8d..feee8184542d1 100644 --- a/crates/bevy_tasks/src/wasm_task.rs +++ b/crates/bevy_tasks/src/wasm_task.rs @@ -21,7 +21,10 @@ impl Task { wasm_bindgen_futures::spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind { inner: future }.await; + let value = CatchUnwind { + inner: AssertUnwindSafe(future), + } + .await; let _ = sender.send(value); }); Self(receiver.into_future()) From 6246ec841e16f12b1d374e2f14a1a91e590d2ace Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 4 Aug 2025 00:49:47 -0700 Subject: [PATCH 31/68] Another attempt at fixing CI --- crates/bevy_tasks/Cargo.toml | 6 +----- crates/bevy_tasks/src/single_threaded_task_pool.rs | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 1ff2e0e2cb4cd..308efd1be21e6 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -13,11 +13,7 @@ default = ["futures-lite"] # Enables multi-threading support. # Without this feature, all tasks will be run on a single thread. -multi_threaded = [ - "bevy_platform/std", - "dep:async-channel", - "bevy_executor", -] +multi_threaded = ["bevy_platform/std", "dep:async-channel", "bevy_executor"] # Uses a Bevy-specific fork of `async-executor` as a task execution backend. # This backend is incompatible with `no_std` targets. diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5b867c1c72088..664bce4607a31 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -5,7 +5,11 @@ use crate::Task; use crate::executor::Executor; -static EXECUTOR: Executor = Executor::new(); +crate::cfg::web! { + if {} else { + static EXECUTOR: Executor = Executor::new(); + } +} crate::cfg::std! { if { From 93c696b45d680c18f0bcf3334404576b17a77da1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 5 Aug 2025 02:16:06 -0700 Subject: [PATCH 32/68] Properly handle thread destruction and recycling --- crates/bevy_tasks/src/bevy_executor.rs | 64 +++++++++++++++++--------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index a90a193f323cf..a230d19240929 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -16,7 +16,7 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; -use std::thread::ThreadId; +use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; use alloc::fmt; @@ -24,7 +24,7 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; -use futures_lite::{future, prelude::*}; +use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -33,8 +33,11 @@ use thread_local::ThreadLocal; static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); pub(crate) fn install_runtime_into_current_thread() { - let tls = THREAD_LOCAL_STATE.get_or_default(); - tls.executor_thread.store(true, Ordering::Relaxed); + // Use LOCAL_QUEUE here to set the thread destructor + LOCAL_QUEUE.with(|_| { + let tls = THREAD_LOCAL_STATE.get_or_default(); + tls.executor_thread.store(true, Ordering::Relaxed); + }) } // Do not access this directly, use `with_local_queue` instead. @@ -67,8 +70,8 @@ cfg_if::cfg_if! { /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] -unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { - LOCAL_QUEUE.with(|tls| { +unsafe fn try_with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> Result { + LOCAL_QUEUE.try_with(|tls| { cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { f(&mut tls.borrow_mut()) @@ -81,14 +84,29 @@ unsafe fn with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> T { } }) } + struct LocalQueue { local_queue: VecDeque, local_active: Slab, } +impl Drop for LocalQueue { + fn drop(&mut self) { + // Unset the executor thread flag if it's been set. + if let Some(tls) = THREAD_LOCAL_STATE.get() { + tls.executor_thread.store(false, Ordering::Relaxed); + } + + for waker in self.local_active.drain() { + waker.wake(); + } + + while self.local_queue.pop_front().is_some() {} + } +} + struct ThreadLocalState { executor_thread: AtomicBool, - thread_id: ThreadId, stealable_queue: ArrayQueue, thread_locked_queue: SegQueue, } @@ -97,7 +115,6 @@ impl Default for ThreadLocalState { fn default() -> Self { Self { executor_thread: AtomicBool::new(false), - thread_id: std::thread::current().id(), stealable_queue: ArrayQueue::new(512), thread_locked_queue: SegQueue::new(), } @@ -176,10 +193,9 @@ impl<'a> ThreadSpawner<'a> { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - unsafe { - with_local_queue(|tls| tls.local_queue.push_back(runnable)); + if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { + state.notify_specific_thread(thread_id, false); } - state.notify_specific_thread(thread_id, false); } } } @@ -265,7 +281,7 @@ impl<'a> Executor<'a> { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. let (runnable, task) = unsafe { - with_local_queue(|tls| { + try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); // SAFETY: There are no instances where the value is accessed mutably @@ -273,7 +289,7 @@ impl<'a> Executor<'a> { // invoked after the surrounding scope has exited in either a // `try_tick_local` or `run` call. let future = AsyncCallOnDrop::new(future, move || { - with_local_queue(|tls| drop(tls.local_active.try_remove(index))); + try_with_local_queue(|tls| drop(tls.local_active.try_remove(index))).ok(); }); // Create the task and register it in the set of active tasks. @@ -295,7 +311,7 @@ impl<'a> Executor<'a> { entry.insert(runnable.waker()); (runnable, task) - }) + }).unwrap() }; runnable.schedule(); @@ -316,7 +332,9 @@ impl<'a> Executor<'a> { // from multiple locations simultaneously. As the Runnable is run after // this scope closes, the AsyncCallOnDrop around the future will be invoked // without overlapping mutable accssses. - unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) } + unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) } + .ok() + .flatten() .map(Runnable::run) .is_some() } @@ -337,7 +355,7 @@ impl<'a> Executor<'a> { if local_state.executor_thread.load(Ordering::Relaxed) { match local_state.stealable_queue.push(runnable) { Ok(()) => { - state.notify_specific_thread(local_state.thread_id, true); + state.notify_specific_thread(std::thread::current().id(), true); return; } Err(r) => r, @@ -357,15 +375,13 @@ impl<'a> Executor<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule_local(&self) -> impl Fn(Runnable) + 'static { let state = self.state_as_arc(); - let local_state: &'static ThreadLocalState = THREAD_LOCAL_STATE.get_or_default(); move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - unsafe { - with_local_queue(|tls| tls.local_queue.push_back(runnable)); + if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { + state.notify_specific_thread(std::thread::current().id(), false); } - state.notify_specific_thread(local_state.thread_id, false); } } @@ -777,8 +793,8 @@ impl Runner<'_> { .runnable_with(|| { // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. - let local_pop = unsafe { with_local_queue(|tls| tls.local_queue.pop_front()) }; - if let Some(r) = local_pop { + let local_pop = unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) }; + if let Ok(Some(r)) = local_pop { return Some(r); } @@ -925,7 +941,9 @@ unsafe fn flush_to_local(src: &SegQueue) { // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any // overlapping accesses. unsafe { - with_local_queue(|tls| { + // It's OK to ignore this error, no point in pushing to a + // queue for a thread that is already terminating. + let _ = try_with_local_queue(|tls| { // Steal tasks. for _ in 0..count { let Some(val) = src.queue_pop() else { break }; From 686d4294e8c016f897d8b1042f80418f95d2c5a4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 6 Aug 2025 00:48:11 -0700 Subject: [PATCH 33/68] Switch to a solution that doesn't require TLS access in thread destructors --- crates/bevy_tasks/src/bevy_executor.rs | 52 ++++++++----------- .../src/single_threaded_task_pool.rs | 12 ++--- crates/bevy_tasks/src/task_pool.rs | 2 +- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index a230d19240929..8833e5354364e 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -32,41 +32,40 @@ use thread_local::ThreadLocal; // See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); -pub(crate) fn install_runtime_into_current_thread() { +pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { // Use LOCAL_QUEUE here to set the thread destructor LOCAL_QUEUE.with(|_| { let tls = THREAD_LOCAL_STATE.get_or_default(); - tls.executor_thread.store(true, Ordering::Relaxed); - }) + let state = executor.state_as_arc(); + let state_ptr = Arc::into_raw(state).cast_mut(); + let old_ptr = tls.executor.swap(state_ptr, Ordering::Relaxed); + if !old_ptr.is_null() { + // SAFETY: If this pointer was not null, it was initialized from Arc::into_raw. + drop(unsafe { Arc::from_raw(old_ptr) }) + } + }); } // Do not access this directly, use `with_local_queue` instead. cfg_if::cfg_if! { if #[cfg(all(debug_assertions, not(miri)))] { use core::cell::RefCell; - - std::thread_local! { - static LOCAL_QUEUE: RefCell = const { - RefCell::new(LocalQueue { - local_queue: VecDeque::new(), - local_active:Slab::new(), - }) - }; - } + type LocalCell = RefCell; } else { use core::cell::UnsafeCell; - - std::thread_local! { - static LOCAL_QUEUE: UnsafeCell = const { - UnsafeCell::new(LocalQueue { - local_queue: VecDeque::new(), - local_active:Slab::new(), - }) - }; - } + type LocalCell = UnsafeCell; } } +std::thread_local! { + static LOCAL_QUEUE: LocalCell = const { + LocalCell::new(LocalQueue { + local_queue: VecDeque::new(), + local_active:Slab::new(), + }) + }; +} + /// # Safety /// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. #[inline(always)] @@ -92,11 +91,6 @@ struct LocalQueue { impl Drop for LocalQueue { fn drop(&mut self) { - // Unset the executor thread flag if it's been set. - if let Some(tls) = THREAD_LOCAL_STATE.get() { - tls.executor_thread.store(false, Ordering::Relaxed); - } - for waker in self.local_active.drain() { waker.wake(); } @@ -106,7 +100,7 @@ impl Drop for LocalQueue { } struct ThreadLocalState { - executor_thread: AtomicBool, + executor: AtomicPtr, stealable_queue: ArrayQueue, thread_locked_queue: SegQueue, } @@ -114,7 +108,7 @@ struct ThreadLocalState { impl Default for ThreadLocalState { fn default() -> Self { Self { - executor_thread: AtomicBool::new(false), + executor: AtomicPtr::new(core::ptr::null_mut()), stealable_queue: ArrayQueue::new(512), thread_locked_queue: SegQueue::new(), } @@ -352,7 +346,7 @@ impl<'a> Executor<'a> { // Attempt to push onto the local queue first in dedicated executor threads, // because we know that this thread is awake and always processing new tasks. let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - if local_state.executor_thread.load(Ordering::Relaxed) { + if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), Arc::as_ptr(&state)) { match local_state.stealable_queue.push(runnable) { Ok(()) => { state.notify_specific_thread(std::thread::current().id(), true); diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 664bce4607a31..c5327ffa0bf67 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -30,15 +30,9 @@ pub struct TaskPoolBuilder {} /// task pool. In the case of the multithreaded task pool this struct is used to spawn /// tasks on a specific thread. But the wasm task pool just calls /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread -/// and so the [`ThreadExecutor`] does nothing. -#[derive(Default, Clone)] +/// and so the [`ThreadSpawner`] does nothing. +#[derive(Clone)] pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); -impl<'a> ThreadSpawner<'a> { - /// Creates a new `ThreadExecutor` - pub fn new() -> Self { - Self::default() - } -} impl TaskPoolBuilder { /// Creates a new `TaskPoolBuilder` instance @@ -85,7 +79,7 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { - ThreadSpawner::new() + ThreadSpawner(PhantomData) } /// Create a `TaskPool` with the default configuration. diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index e0a31f3fdedf7..cace1736b4902 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -178,7 +178,7 @@ impl TaskPool { thread_builder .spawn(move || { - crate::bevy_executor::install_runtime_into_current_thread(); + crate::bevy_executor::install_runtime_into_current_thread(&ex); if let Some(on_thread_spawn) = on_thread_spawn { on_thread_spawn(); From aa64f0336f2c4d80d68700908bf85967687601ea Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 6 Aug 2025 02:21:24 -0700 Subject: [PATCH 34/68] Try to provide a blocking solution to TaskPool::scope in single threaded spaces --- crates/bevy_tasks/src/bevy_executor.rs | 55 +++++++++++-------- .../src/single_threaded_task_pool.rs | 12 +++- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 8833e5354364e..85c8ad01f0a67 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -16,6 +16,7 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; +use core::cell::UnsafeCell; use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; @@ -24,6 +25,7 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; +use futures_lite::future::block_on; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -41,25 +43,14 @@ pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { let old_ptr = tls.executor.swap(state_ptr, Ordering::Relaxed); if !old_ptr.is_null() { // SAFETY: If this pointer was not null, it was initialized from Arc::into_raw. - drop(unsafe { Arc::from_raw(old_ptr) }) + drop(unsafe { Arc::from_raw(old_ptr) }); } }); } -// Do not access this directly, use `with_local_queue` instead. -cfg_if::cfg_if! { - if #[cfg(all(debug_assertions, not(miri)))] { - use core::cell::RefCell; - type LocalCell = RefCell; - } else { - use core::cell::UnsafeCell; - type LocalCell = UnsafeCell; - } -} - std::thread_local! { - static LOCAL_QUEUE: LocalCell = const { - LocalCell::new(LocalQueue { + static LOCAL_QUEUE: UnsafeCell = const { + UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), local_active:Slab::new(), }) @@ -71,16 +62,10 @@ std::thread_local! { #[inline(always)] unsafe fn try_with_local_queue(f: impl FnOnce(&mut LocalQueue) -> T) -> Result { LOCAL_QUEUE.try_with(|tls| { - cfg_if::cfg_if! { - if #[cfg(all(debug_assertions, not(miri)))] { - f(&mut tls.borrow_mut()) - } else { - // SAFETY: This value is in thread local storage and thus can only be accessed - // from one thread. The caller guarantees that this function is not used with - // LOCAL_QUEUE in any way. - f(unsafe { &mut *tls.get() }) - } - } + // SAFETY: This value is in thread local storage and thus can only be accessed + // from one thread. The caller guarantees that this function is not used with + // LOCAL_QUEUE in any way. + f(unsafe { &mut *tls.get() }) }) } @@ -333,6 +318,28 @@ impl<'a> Executor<'a> { .is_some() } + pub fn flush_local() { + // If this is called during the thread destructor, there's nothing to run anyway. + // All of the tasks will be dropped soon. + // + // SAFETY: There are no instances where the value is accessed mutably + // from multiple locations simultaneously. As the Runnable is run after + // this scope closes, the AsyncCallOnDrop around the future will be invoked + // without overlapping mutable accssses. + let _ = LOCAL_QUEUE.try_with(|local| unsafe { + block_on(async { + while !(&*local.get()).local_active.is_empty() { + match (&mut *local.get()).local_queue.pop_front() { + Some(runnable) => { + runnable.run(); + }, + None => future::yield_now().await, + } + } + }) + }); + } + /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index c5327ffa0bf67..195154597069c 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -153,7 +153,7 @@ impl TaskPool { f(scope_ref); // Loop until all tasks are done - while Self::try_tick_local() {} + Self::flush_local(); let results = scope.results.borrow(); results @@ -217,6 +217,16 @@ impl TaskPool { } } } + + fn flush_local() { + crate::cfg::bevy_executor! { + if { + crate::bevy_executor::Executor::flush_local(); + } else { + while EXECUTOR.try_tick() {} + } + } + } } /// A `TaskPool` scope for running one or more non-`'static` futures. From 79c31b7e6d9fcf480007f7b55362be706f7f87b1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 7 Aug 2025 23:46:37 -0700 Subject: [PATCH 35/68] CI fixes --- crates/bevy_tasks/src/bevy_executor.rs | 4 ++-- .../bevy_tasks/src/single_threaded_task_pool.rs | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 85c8ad01f0a67..d0815c3198598 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -336,7 +336,7 @@ impl<'a> Executor<'a> { None => future::yield_now().await, } } - }) + }); }); } @@ -915,7 +915,7 @@ impl WorkQueue for SegQueue { } /// Steals some items from one queue into another. -fn steal>(src: &Q, dest: &ArrayQueue) { +fn steal_and_pop>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. let mut count = src.stealable_count(); diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 195154597069c..62e21ef970ddf 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -208,12 +208,16 @@ impl TaskPool { self.spawn(future) } - pub(crate) fn try_tick_local() -> bool { - crate::cfg::bevy_executor! { - if { - crate::bevy_executor::Executor::try_tick_local() - } else { - EXECUTOR.try_tick() + crate::cfg::web! { + if {} else { + pub(crate) fn try_tick_local() -> bool { + crate::cfg::bevy_executor! { + if { + crate::bevy_executor::Executor::try_tick_local() + } else { + EXECUTOR.try_tick() + } + } } } } From 4dcb37c941323fdebf432ef69497bf460a3e0f9d Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:21:54 -0700 Subject: [PATCH 36/68] Fix up the build, and hopefully CI --- crates/bevy_tasks/src/bevy_executor.rs | 24 +-------- crates/bevy_tasks/src/edge_executor.rs | 4 +- crates/bevy_tasks/src/executor.rs | 52 ------------------- crates/bevy_tasks/src/lib.rs | 1 - .../src/single_threaded_task_pool.rs | 38 +++++--------- crates/bevy_tasks/src/task_pool.rs | 19 +++---- 6 files changed, 28 insertions(+), 110 deletions(-) delete mode 100644 crates/bevy_tasks/src/executor.rs diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index d0815c3198598..eeea29ef9c006 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -318,28 +318,6 @@ impl<'a> Executor<'a> { .is_some() } - pub fn flush_local() { - // If this is called during the thread destructor, there's nothing to run anyway. - // All of the tasks will be dropped soon. - // - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. As the Runnable is run after - // this scope closes, the AsyncCallOnDrop around the future will be invoked - // without overlapping mutable accssses. - let _ = LOCAL_QUEUE.try_with(|local| unsafe { - block_on(async { - while !(&*local.get()).local_active.is_empty() { - match (&mut *local.get()).local_queue.pop_front() { - Some(runnable) => { - runnable.run(); - }, - None => future::yield_now().await, - } - } - }); - }); - } - /// Runs the executor until the given future completes. pub async fn run(&self, future: impl Future) -> T { self.state().run(future).await @@ -915,7 +893,7 @@ impl WorkQueue for SegQueue { } /// Steals some items from one queue into another. -fn steal_and_pop>(src: &Q, dest: &ArrayQueue) { +fn steal>(src: &Q, dest: &ArrayQueue) { // Half of `src`'s length rounded up. let mut count = src.stealable_count(); diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index e063a47533774..b8f72ba31f40c 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -48,6 +48,7 @@ use futures_lite::FutureExt; /// drop(signal); /// })); /// ``` +#[derive(Debug)] pub struct Executor<'a, const C: usize = 64> { state: LazyLock>>, _invariant: PhantomData>, @@ -170,7 +171,7 @@ impl<'a, const C: usize> Executor<'a, C> { /// ``` pub async fn run(&self, fut: F) -> F::Output where - F: Future + Send + 'a, + F: Future + 'a, { // SAFETY: Original implementation missing safety documentation unsafe { self.run_unchecked(fut).await } @@ -306,6 +307,7 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} +#[derive(Debug)] struct State { #[cfg(all( target_has_atomic = "8", diff --git a/crates/bevy_tasks/src/executor.rs b/crates/bevy_tasks/src/executor.rs deleted file mode 100644 index 7ea6e64e32229..0000000000000 --- a/crates/bevy_tasks/src/executor.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! Provides a fundamental executor primitive appropriate for the target platform -//! and feature set selected. -//! By default, the `async_executor` feature will be enabled, which will rely on -//! [`async-executor`] for the underlying implementation. This requires `std`, -//! so is not suitable for `no_std` contexts. Instead, you must use `edge_executor`, -//! which relies on the alternate [`edge-executor`] backend. -//! -//! [`async-executor`]: https://crates.io/crates/async-executor -//! [`edge-executor`]: https://crates.io/crates/edge-executor - -use core::{ - fmt, - panic::{RefUnwindSafe, UnwindSafe}, -}; -use derive_more::{Deref, DerefMut}; - -crate::cfg::bevy_executor! { - if { - type ExecutorInner<'a> = crate::bevy_executor::Executor<'a>; - } else { - type ExecutorInner<'a> = crate::edge_executor::Executor<'a, 64>; - } -} - -crate::cfg::multi_threaded! { - pub use async_task::FallibleTask; -} - -/// Wrapper around a multi-threading-aware async executor. -/// spawning will generally require tasks to be `send` and `sync` to allow multiple -/// threads to send/receive/advance tasks. -#[derive(Deref, DerefMut)] -pub(crate) struct Executor<'a>(ExecutorInner<'a>); - -impl Executor<'_> { - /// Construct a new [`Executor`] - #[expect(clippy::allow_attributes, reason = "This lint may not always trigger.")] - #[allow(dead_code, reason = "not all feature flags require this function")] - pub const fn new() -> Self { - Self(ExecutorInner::new()) - } -} - -impl UnwindSafe for Executor<'_> {} - -impl RefUnwindSafe for Executor<'_> {} - -impl fmt::Debug for Executor<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Executor").finish() - } -} diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 2854655354ab3..d03665c94bc86 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -72,7 +72,6 @@ use alloc::boxed::Box; pub type BoxedFuture<'a, T> = core::pin::Pin + 'a>>; // Modules -mod executor; pub mod futures; mod iter; mod slice; diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 59921e8281393..434895a7bf3d0 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,28 +1,18 @@ use alloc::{string::String, vec::Vec}; -use bevy_platform::sync::Arc; use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; -use crate::executor::LocalExecutor; use crate::{block_on, Task}; -crate::cfg::std! { - if { - use std::thread_local; - - use crate::executor::LocalExecutor as Executor; - - thread_local! { - static LOCAL_EXECUTOR: Executor<'static> = const { Executor::new() }; - } +crate::cfg::bevy_executor! { + if { + use crate::bevy_executor::Executor; } else { - - // Because we do not have thread-locals without std, we cannot use LocalExecutor here. - use crate::executor::Executor; - - static LOCAL_EXECUTOR: Executor<'static> = const { Executor::new() }; + use crate::edge_executor::Executor; } } +static EXECUTOR: Executor<'static> = const { Executor::new() }; + /// Used to create a [`TaskPool`]. #[derive(Debug, Default, Clone)] pub struct TaskPoolBuilder {} @@ -133,9 +123,8 @@ impl TaskPool { // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor = LocalExecutor::new(); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor_ref: &'env LocalExecutor<'env> = unsafe { mem::transmute(&executor) }; + let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; let results: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -159,7 +148,8 @@ impl TaskPool { f(scope_ref); // Wait until the scope is complete - block_on(executor.run(async { + block_on(executor_ref.run(async { + std::println!("Pending: {}", pending_tasks.get()); while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } @@ -215,7 +205,7 @@ impl TaskPool { pub(crate) fn try_tick_local() -> bool { crate::cfg::bevy_executor! { if { - crate::bevy_executor::Executor::try_tick_local() + Executor::try_tick_local() } else { EXECUTOR.try_tick() } @@ -227,7 +217,7 @@ impl TaskPool { fn flush_local() { crate::cfg::bevy_executor! { if { - crate::bevy_executor::Executor::flush_local(); + Executor::flush_local(); } else { while EXECUTOR.try_tick() {} } @@ -240,7 +230,7 @@ impl TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor_ref: &'scope LocalExecutor<'scope>, + executor_ref: &'scope Executor<'scope>, // The number of pending tasks spawned on the scope pending_tasks: &'scope Cell, // Vector to gather results of all futures spawned during scope run @@ -310,10 +300,10 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. unsafe { - self.executor.spawn_local_scoped(f).detach() + self.executor_ref.spawn_local_scoped(f).detach() }; } else { - self.executor.spawn_local(f).detach(); + self.executor_ref.spawn_local(f).detach(); } } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index cace1736b4902..086e05d4e0614 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,7 +2,8 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; use std::thread::{self, JoinHandle}; -use crate::{bevy_executor::Executor, executor::FallibleTask}; +use crate::bevy_executor::Executor; +use async_task::FallibleTask; use bevy_platform::sync::Arc; use crossbeam_queue::SegQueue; use futures_lite::FutureExt; @@ -129,7 +130,7 @@ impl TaskPoolBuilder { #[derive(Debug)] pub struct TaskPool { /// The executor for the pool. - executor: Arc>, + executor: Arc>, // The inner state of the pool. threads: Vec>, @@ -151,7 +152,7 @@ impl TaskPool { fn new_internal(builder: TaskPoolBuilder) -> Self { let (shutdown_tx, shutdown_rx) = async_channel::unbounded::<()>(); - let executor = Arc::new(crate::executor::Executor::new()); + let executor = Arc::new(Executor::new()); let num_threads = builder .num_threads @@ -343,14 +344,14 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor: &crate::executor::Executor = &self.executor; + let executor: &Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor: &'env crate::executor::Executor = unsafe { mem::transmute(executor) }; - let spawned: SegQueue>>> = + let executor: &'env Executor = unsafe { mem::transmute(executor) }; + let spawned: SegQueue>>> = SegQueue::new(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env SegQueue>>> = + let spawned: &'env SegQueue>>> = unsafe { mem::transmute(&spawned) }; let scope = Scope { @@ -451,10 +452,10 @@ impl Drop for TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor: &'scope crate::executor::Executor<'scope>, + executor: &'scope Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope SegQueue>>>, + spawned: &'scope SegQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, From 0923f0268a23dd7966f17f0e2d9b8ccd579b1cfc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:33:17 -0700 Subject: [PATCH 37/68] Shut up Clippy --- crates/bevy_tasks/Cargo.toml | 1 - crates/bevy_tasks/src/bevy_executor.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index f3a155e325bc1..50c88da9ba3e1 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -45,7 +45,6 @@ derive_more = { version = "2", default-features = false, features = [ "deref", "deref_mut", ] } -cfg-if = "1.0.0" slab = { version = "0.4", optional = true } pin-project-lite = { version = "0.2", optional = true } thread_local = { version = "1.1", optional = true } diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index eeea29ef9c006..2fb68aad68d12 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -2,10 +2,6 @@ unsafe_code, reason = "Executor code requires unsafe code for dealing with non-'static lifetimes" )] -#![expect( - clippy::unused_unit, - reason = "False positive detection on {Async}CallOnDrop" -)] #![allow( dead_code, reason = "Not all functions are used with every feature combination" @@ -25,7 +21,6 @@ use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; use crossbeam_queue::{ArrayQueue, SegQueue}; -use futures_lite::future::block_on; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; From 43a09d7688eb41cda4951ee4aca971b2802346ca Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 01:46:34 -0700 Subject: [PATCH 38/68] Remove test println --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 434895a7bf3d0..5207e8452439d 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -149,7 +149,6 @@ impl TaskPool { // Wait until the scope is complete block_on(executor_ref.run(async { - std::println!("Pending: {}", pending_tasks.get()); while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } From 8f313f77838f73d073d3f3e549420d04d1a0f00b Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 02:05:50 -0700 Subject: [PATCH 39/68] Fix for portable atomics --- crates/bevy_tasks/src/edge_executor.rs | 2 -- crates/bevy_tasks/src/single_threaded_task_pool.rs | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index b8f72ba31f40c..fc0cb5d770d5d 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -48,7 +48,6 @@ use futures_lite::FutureExt; /// drop(signal); /// })); /// ``` -#[derive(Debug)] pub struct Executor<'a, const C: usize = 64> { state: LazyLock>>, _invariant: PhantomData>, @@ -307,7 +306,6 @@ unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} // SAFETY: Original implementation missing safety documentation unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} -#[derive(Debug)] struct State { #[cfg(all( target_has_atomic = "8", diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5207e8452439d..139f180bd832f 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,4 +1,4 @@ -use alloc::{string::String, vec::Vec}; +use alloc::{string::String, vec::Vec, fmt}; use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; use crate::{block_on, Task}; @@ -227,7 +227,6 @@ impl TaskPool { /// A `TaskPool` scope for running one or more non-`'static` futures. /// /// For more information, see [`TaskPool::scope`]. -#[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor_ref: &'scope Executor<'scope>, // The number of pending tasks spawned on the scope @@ -308,6 +307,17 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { } } +impl <'scope, 'env: 'scope, T> fmt::Debug for Scope<'scope, 'env, T> +where T: fmt::Debug +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("pending_tasks", &self.pending_tasks) + .field("results_ref", &self.results_ref) + .finish() + } +} + crate::cfg::std! { if { pub trait MaybeSend {} From 4a8b6b0c7d7a550144885e14738df58be9de8ab6 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 10:42:52 -0700 Subject: [PATCH 40/68] Fix for web builds --- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/single_threaded_task_pool.rs | 10 ---------- crates/bevy_tasks/src/task.rs | 14 ++++++++++---- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 50c88da9ba3e1..ce5e38c184c7d 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -58,6 +58,7 @@ crossbeam-queue = { version = "0.3", default-features = false, features = [ [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" +pin-project-lite = "0.2" [target.'cfg(not(all(target_has_atomic = "8", target_has_atomic = "16", target_has_atomic = "32", target_has_atomic = "64", target_has_atomic = "ptr")))'.dependencies] async-task = { version = "4.4.0", default-features = false, features = [ diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 139f180bd832f..e7a5d4dc46125 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -212,16 +212,6 @@ impl TaskPool { } } } - - fn flush_local() { - crate::cfg::bevy_executor! { - if { - Executor::flush_local(); - } else { - while EXECUTOR.try_tick() {} - } - } - } } /// A `TaskPool` scope for running one or more non-`'static` futures. diff --git a/crates/bevy_tasks/src/task.rs b/crates/bevy_tasks/src/task.rs index dd649ba47dca3..4e9826a82f709 100644 --- a/crates/bevy_tasks/src/task.rs +++ b/crates/bevy_tasks/src/task.rs @@ -38,7 +38,9 @@ cfg::web! { spawn_local(async move { // Catch any panics that occur when polling the future so they can // be propagated back to the task handle. - let value = CatchUnwind(AssertUnwindSafe(future)).await; + let value = CatchUnwind { + inner: AssertUnwindSafe(future) + }.await; let _ = sender.send(value); }); Self(receiver) @@ -173,13 +175,17 @@ cfg::web! { type Panic = Box; - #[pin_project::pin_project] - struct CatchUnwind(#[pin] F); + pin_project_lite::pin_project! { + struct CatchUnwind { + #[pin] + inner: F + } + } impl Future for CatchUnwind { type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let f = AssertUnwindSafe(|| self.project().0.poll(cx)); + let f = AssertUnwindSafe(|| self.project().inner.poll(cx)); let result = cfg::std! { if { From 7e5cf6caa7297f2bec6faa5812701b332feac9dc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:05:55 -0700 Subject: [PATCH 41/68] Reduce async type genreation, code indirection, and handle single-threaded cancellation --- crates/bevy_tasks/src/bevy_executor.rs | 133 ++++++++---------- .../src/single_threaded_task_pool.rs | 42 +++--- 2 files changed, 79 insertions(+), 96 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 2fb68aad68d12..317e333f9f16c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -314,8 +314,23 @@ impl<'a> Executor<'a> { } /// Runs the executor until the given future completes. - pub async fn run(&self, future: impl Future) -> T { - self.state().run(future).await + pub fn run<'b, T>(&'b self, future: impl Future + 'b) -> impl Future + 'b { + let mut runner = Runner::new(self.state()); + + // A future that runs tasks forever. + let run_forever = async move { + let mut rng = fastrand::Rng::new(); + loop { + for _ in 0..200 { + let runnable = runner.runnable(&mut rng).await; + runnable.run(); + } + future::yield_now().await; + } + }; + + // Run `future` and `run_forever` concurrently until `future` completes. + future.or(run_forever) } /// Returns a function that schedules a runnable task when it gets woken up. @@ -500,25 +515,6 @@ impl State { w.wake(); } } - - pub async fn run(&self, future: impl Future) -> T { - let mut runner = Runner::new(self); - let mut rng = fastrand::Rng::new(); - - // A future that runs tasks forever. - let run_forever = async { - loop { - for _ in 0..200 { - let runnable = runner.runnable(&mut rng).await; - runnable.run(); - } - future::yield_now().await; - } - }; - - // Run `future` and `run_forever` concurrently until `future` completes. - future.or(run_forever).await - } } impl fmt::Debug for State { @@ -677,31 +673,39 @@ impl Ticker<'_> { } /// Waits for the next runnable task to run, given a function that searches for a task. - async fn runnable_with(&mut self, mut search: impl FnMut() -> Option) -> Runnable { - future::poll_fn(|cx| { - loop { - match search() { - None => { - // Move to sleeping and unnotified state. - if !self.sleep(cx.waker()) { - // If already sleeping and unnotified, return. - return Poll::Pending; + /// + /// # Safety + /// Caller must not access LOCAL_QUEUE either directly or with try_with_local_queue` in any way inside `search`. + unsafe fn runnable_with(&mut self, mut search: impl FnMut(&mut LocalQueue) -> Option) -> impl Future { + future::poll_fn(move |cx| { + // SAFETY: Caller must ensure that there's no instances where LOCAL_QUEUE is accessed mutably + // from multiple locations simultaneously. + unsafe { + try_with_local_queue(|tls| { + loop { + match search(tls) { + None => { + // Move to sleeping and unnotified state. + if !self.sleep(cx.waker()) { + // If already sleeping and unnotified, return. + return Poll::Pending; + } + } + Some(r) => { + // Wake up. + self.wake(); + + // Notify another ticker now to pick up where this ticker left off, just in + // case running the task takes a long time. + self.state.notify(); + + return Poll::Ready(r); + } } } - Some(r) => { - // Wake up. - self.wake(); - - // Notify another ticker now to pick up where this ticker left off, just in - // case running the task takes a long time. - self.state.notify(); - - return Poll::Ready(r); - } - } + }).unwrap() } }) - .await } } @@ -739,7 +743,7 @@ struct Runner<'a> { ticks: usize, // The thread local state of the executor for the current thread. - local_state: &'static ThreadLocalState, + local_state: &'a ThreadLocalState, } impl Runner<'_> { @@ -761,14 +765,14 @@ impl Runner<'_> { } /// Waits for the next runnable task to run. - async fn runnable(&mut self, rng: &mut fastrand::Rng) -> Runnable { - let runnable = self + fn runnable(&mut self, rng: &mut fastrand::Rng) -> impl Future { + // SAFETY: The provided search function does not access LOCAL_QUEUE in any way, and thus cannot + // alias. + let runnable = unsafe { + self .ticker - .runnable_with(|| { - // SAFETY: There are no instances where the value is accessed mutably - // from multiple locations simultaneously. - let local_pop = unsafe { try_with_local_queue(|tls| tls.local_queue.pop_front()) }; - if let Ok(Some(r)) = local_pop { + .runnable_with(|tls| { + if let Some(r) = tls.local_queue.pop_back() { return Some(r); } @@ -813,16 +817,13 @@ impl Runner<'_> { // // Instead, flush all queued tasks into the local queue to // minimize the effort required to scan for these tasks. - // - // SAFETY: This is not being used at the same time as any - // access to LOCAL_QUEUE. - unsafe { flush_to_local(&self.local_state.thread_locked_queue) }; + flush_to_local(&self.local_state.thread_locked_queue, tls); return Some(r); } None }) - .await; + }; // Bump the tick counter. self.ticks = self.ticks.wrapping_add(1); @@ -904,26 +905,14 @@ fn steal>(src: &Q, dest: &ArrayQueue) { } } -/// Flushes all of the items from a queue into the thread local queue. -/// -/// # Safety -/// This must not be accessed at the same time as `LOCAL_QUEUE` in any way. -unsafe fn flush_to_local(src: &SegQueue) { +fn flush_to_local(src: &SegQueue, dst: &mut LocalQueue) { let count = src.len(); if count > 0 { - // SAFETY: Caller assures that `LOCAL_QUEUE` does not have any - // overlapping accesses. - unsafe { - // It's OK to ignore this error, no point in pushing to a - // queue for a thread that is already terminating. - let _ = try_with_local_queue(|tls| { - // Steal tasks. - for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; - tls.local_queue.push_front(val); - } - }); + // Steal tasks. + for _ in 0..count { + let Some(val) = src.queue_pop() else { break }; + dst.local_queue.push_front(val); } } } diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index e7a5d4dc46125..5c892f97b5629 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -1,5 +1,5 @@ use alloc::{string::String, vec::Vec, fmt}; -use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem}; +use core::{cell::{RefCell, Cell}, future::Future, marker::PhantomData, mem, task::{Poll, Context, Waker}, pin::Pin}; use crate::{block_on, Task}; @@ -126,9 +126,10 @@ impl TaskPool { // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; - let results: RefCell>> = RefCell::new(Vec::new()); + // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks. + let tasks: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let results_ref: &'env RefCell>> = unsafe { mem::transmute(&results) }; + let tasks_ref: &'env RefCell>> = unsafe { mem::transmute(&tasks) }; let pending_tasks: Cell = Cell::new(0); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -136,8 +137,8 @@ impl TaskPool { let mut scope = Scope { executor_ref, + tasks_ref, pending_tasks, - results_ref, scope: PhantomData, env: PhantomData, }; @@ -154,10 +155,14 @@ impl TaskPool { } })); - results + let mut context = Context::from_waker(Waker::noop()); + tasks .take() .into_iter() - .map(|result| result.unwrap()) + .map(|mut task| match Pin::new(&mut task).poll(&mut context) { + Poll::Ready(result) => result, + Poll::Pending => unreachable!(), + }) .collect() } @@ -222,7 +227,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { // The number of pending tasks spawned on the scope pending_tasks: &'scope Cell, // Vector to gather results of all futures spawned during scope run - results_ref: &'env RefCell>>, + tasks_ref: &'env RefCell>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -263,35 +268,24 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { let pending_tasks = self.pending_tasks; pending_tasks.update(|i| i + 1); - // add a spot to keep the result, and record the index - let results_ref = self.results_ref; - let mut results = results_ref.borrow_mut(); - let task_number = results.len(); - results.push(None); - drop(results); - // create the job closure let f = async move { let result = f.await; - // store the result in the allocated slot - let mut results = results_ref.borrow_mut(); - results[task_number] = Some(result); - drop(results); - // decrement the pending tasks count pending_tasks.update(|i| i - 1); + + result }; + let mut tasks = self.tasks_ref.borrow_mut(); crate::cfg::bevy_executor! { if { // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. - unsafe { - self.executor_ref.spawn_local_scoped(f).detach() - }; + tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); } else { - self.executor_ref.spawn_local(f).detach(); + tasks.push(self.executor_ref.spawn_local(f)); } } } @@ -303,7 +297,7 @@ where T: fmt::Debug fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Scope") .field("pending_tasks", &self.pending_tasks) - .field("results_ref", &self.results_ref) + .field("tasks_ref", &self.tasks_ref) .finish() } } From eb689b59831ac324840d1a49f112903c2a94b37a Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:14:50 -0700 Subject: [PATCH 42/68] Fix docs and move expect attribute --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 317e333f9f16c..453252eb7f35a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -675,7 +675,7 @@ impl Ticker<'_> { /// Waits for the next runnable task to run, given a function that searches for a task. /// /// # Safety - /// Caller must not access LOCAL_QUEUE either directly or with try_with_local_queue` in any way inside `search`. + /// Caller must not access `LOCAL_QUEUE` either directly or with `try_with_local_queue` in any way inside `search`. unsafe fn runnable_with(&mut self, mut search: impl FnMut(&mut LocalQueue) -> Option) -> impl Future { future::poll_fn(move |cx| { // SAFETY: Caller must ensure that there's no instances where LOCAL_QUEUE is accessed mutably diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 5c892f97b5629..8931856ed8cdc 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -257,7 +257,6 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { self.spawn_on_scope(f); } - #[allow(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] /// Spawns a scoped future that runs on the thread the scope called from. The /// scope *must* outlive the provided future. The results of the future will be /// returned as a part of [`TaskPool::scope`]'s return value. @@ -281,6 +280,7 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { let mut tasks = self.tasks_ref.borrow_mut(); crate::cfg::bevy_executor! { if { + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] // SAFETY: The surrounding scope will not terminate until all local tasks are done // ensuring that the borrowed variables do not outlive the detached task. tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); From 7ec07af71b82df333da0913014fb7e7d63e0e999 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 22:35:04 -0700 Subject: [PATCH 43/68] Complete the comment --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 8931856ed8cdc..62f9f75e1fa23 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -126,7 +126,7 @@ impl TaskPool { // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; - // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks. + // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks are cancelled. let tasks: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime let tasks_ref: &'env RefCell>> = unsafe { mem::transmute(&tasks) }; From 4a4434c2b731662b25f241a08c247d9dd517151f Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 23:37:42 -0700 Subject: [PATCH 44/68] Add async_executor's tests --- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/bevy_executor.rs | 203 ++++++++++++++++++++++++- 2 files changed, 203 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index ce5e38c184c7d..5b12d0c817372 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -76,6 +76,7 @@ futures-lite = { version = "2.0.1", default-features = false, features = [ "std", ] } async-channel = "2.3.0" +async-io = "2.0.0" [lints] workspace = true diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 453252eb7f35a..34ebfcebe5f0c 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1031,8 +1031,16 @@ impl Future for AsyncCallOnDrop { #[cfg(test)] mod test { - use super::Executor; + use super::*; use super::THREAD_LOCAL_STATE; + use alloc::{string::String, boxed::Box}; + use futures_lite::{future, pin}; + use std::panic::catch_unwind; + use core::sync::atomic::{AtomicUsize, Ordering}; + use bevy_platform::sync::Mutex; + use core::task::{Poll, Waker}; + use async_task::Task; + use std::time::Duration; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -1051,4 +1059,197 @@ mod test { is_send(THREAD_LOCAL_STATE.get_or_default()); is_sync(THREAD_LOCAL_STATE.get_or_default()); } + + #[test] + fn executor_cancels_everything() { + static DROP: AtomicUsize = AtomicUsize::new(0); + static WAKER: Mutex> = Mutex::new(None); + + let ex = Executor::new(); + + let task = ex.spawn(async { + let _guard = CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }); + + future::poll_fn(|cx| { + *WAKER.lock().unwrap() = Some(cx.waker().clone()); + Poll::Pending::<()> + }) + .await; + }); + + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert!(WAKER.lock().unwrap().is_some()); + assert_eq!(DROP.load(Ordering::SeqCst), 0); + + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + + assert!(catch_unwind(|| future::block_on(task)).is_err()); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + #[test] + fn await_task_after_dropping_executor() { + let s: String = "hello".into(); + + let ex = Executor::new(); + let task: Task<&str> = ex.spawn(async { &*s }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + + drop(ex); + assert_eq!(future::block_on(task), "hello"); + drop(s); + } + + #[test] + fn drop_executor_and_then_drop_finished_task() { + static DROP: AtomicUsize = AtomicUsize::new(0); + + let ex = Executor::new(); + let task = ex.spawn(async { + CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }) + }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(task); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + #[test] + fn drop_finished_task_and_then_drop_executor() { + static DROP: AtomicUsize = AtomicUsize::new(0); + + let ex = Executor::new(); + let task = ex.spawn(async { + CallOnDrop(|| { + DROP.fetch_add(1, Ordering::SeqCst); + }) + }); + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + assert_eq!(DROP.load(Ordering::SeqCst), 0); + drop(task); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + drop(ex); + assert_eq!(DROP.load(Ordering::SeqCst), 1); + } + + fn do_run>(mut f: impl FnMut(Arc>) -> Fut) { + // This should not run for longer than two minutes. + #[cfg(not(miri))] + let _stop_timeout = { + let (stop_timeout, stopper) = async_channel::bounded::<()>(1); + std::thread::spawn(move || { + future::block_on(async move { + let timeout = async { + async_io::Timer::after(Duration::from_secs(2 * 60)).await; + std::eprintln!("test timed out after 2m"); + std::process::exit(1) + }; + + let _ = stopper.recv().or(timeout).await; + }) + }); + stop_timeout + }; + + let ex = Arc::new(Executor::new()); + + // Test 1: Use the `run` command. + future::block_on(ex.run(f(ex.clone()))); + + // Test 2: Run on many threads. + std::thread::scope(|scope| { + let (_signal, shutdown) = async_channel::bounded::<()>(1); + + for _ in 0..16 { + let shutdown = shutdown.clone(); + let ex = &ex; + scope.spawn(move || future::block_on(ex.run(shutdown.recv()))); + } + + future::block_on(f(ex.clone())); + }); + } + + #[test] + fn smoke() { + do_run(|ex| async move { ex.spawn(async {}).await }); + } + + #[test] + fn yield_now() { + do_run(|ex| async move { ex.spawn(future::yield_now()).await }) + } + + #[test] + fn timer() { + do_run(|ex| async move { + ex.spawn(async_io::Timer::after(Duration::from_millis(5))) + .await; + }) + } + + #[test] + fn test_panic_propagation() { + let ex = Executor::new(); + let task = ex.spawn(async { panic!("should be caught by the task") }); + + // Running the executor should not panic. + future::block_on(ex.run(async { + for _ in 0..10 { + future::yield_now().await + } + })); + + // Polling the task should. + assert!(future::block_on(task.catch_unwind()).is_err()); + } + + #[test] + fn two_queues() { + future::block_on(async { + // Create an executor with two runners. + let ex = Executor::new(); + let (run1, run2) = ( + ex.run(future::pending::<()>()), + ex.run(future::pending::<()>()), + ); + let mut run1 = Box::pin(run1); + pin!(run2); + + // Poll them both. + assert!(future::poll_once(run1.as_mut()).await.is_none()); + assert!(future::poll_once(run2.as_mut()).await.is_none()); + + // Drop the first one, which should leave the local queue in the `None` state. + drop(run1); + assert!(future::poll_once(run2.as_mut()).await.is_none()); + }); + } } From 2ce5537641ac5ec09705d4b147af52c11a3299bc Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 10 Aug 2025 23:40:50 -0700 Subject: [PATCH 45/68] Run Miri on bevy_tasks in CI --- .github/workflows/ci.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ad9a1ce76777..a16c1111ea73f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,13 @@ jobs: miri: runs-on: macos-latest timeout-minutes: 60 + env: + # -Zrandomize-layout makes sure we dont rely on the layout of anything that might change + RUSTFLAGS: -Zrandomize-layout + # https://github.com/rust-lang/miri#miri--z-flags-and-environment-variables + # -Zmiri-disable-isolation is needed because our executor uses `fastrand` which accesses system time. + # -Zmiri-ignore-leaks is necessary because a bunch of tests don't join all threads before finishing. + MIRIFLAGS: -Zmiri-ignore-leaks -Zmiri-disable-isolation steps: - uses: actions/checkout@v4 - uses: actions/cache/restore@v4 @@ -113,15 +120,10 @@ jobs: components: miri - name: CI job # To run the tests one item at a time for troubleshooting, use + # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_tasks --lib -- --exact # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact + run: cargo miri test -p bevy_tasks --features bevy_executor --features multi_threaded run: cargo miri test -p bevy_ecs --features bevy_utils/debug - env: - # -Zrandomize-layout makes sure we dont rely on the layout of anything that might change - RUSTFLAGS: -Zrandomize-layout - # https://github.com/rust-lang/miri#miri--z-flags-and-environment-variables - # -Zmiri-disable-isolation is needed because our executor uses `fastrand` which accesses system time. - # -Zmiri-ignore-leaks is necessary because a bunch of tests don't join all threads before finishing. - MIRIFLAGS: -Zmiri-ignore-leaks -Zmiri-disable-isolation check-compiles: runs-on: ubuntu-latest From 8af9886f9fa2d7917929add2dc45816fddc10928 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 11:06:18 -0700 Subject: [PATCH 46/68] Make stealing a non-blocking operation --- crates/bevy_tasks/src/bevy_executor.rs | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 34ebfcebe5f0c..fe07ae862190f 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -47,7 +47,7 @@ std::thread_local! { static LOCAL_QUEUE: UnsafeCell = const { UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), - local_active:Slab::new(), + local_active: Slab::new(), }) }; } @@ -788,26 +788,26 @@ impl Runner<'_> { } // Try stealing from other runners. - let stealer_queues = self.state.stealer_queues.read().unwrap(); - - // Pick a random starting point in the iterator list and rotate the list. - let n = stealer_queues.len(); - let start = rng.usize(..n); - let iter = stealer_queues - .iter() - .chain(stealer_queues.iter()) - .skip(start) - .take(n); - - // Remove this runner's local queue. - let iter = - iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); - - // Try stealing from each local queue in the list. - for local in iter { - steal(*local, &self.local_state.stealable_queue); - if let Some(r) = self.local_state.stealable_queue.pop() { - return Some(r); + if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { + // Pick a random starting point in the iterator list and rotate the list. + let n = stealer_queues.len(); + let start = rng.usize(..n); + let iter = stealer_queues + .iter() + .chain(stealer_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { + return Some(r); + } } } From 74d301c84d67ba85004b18a51fb3b1b3a746e2bf Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:20:43 -0700 Subject: [PATCH 47/68] Cache-pad the thread locals and disable multithreaded polling when the feature is not enabled --- crates/bevy_tasks/Cargo.toml | 2 + crates/bevy_tasks/src/bevy_executor.rs | 91 ++++++++++++++------------ 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 5b12d0c817372..62c4d78e3f84e 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -21,6 +21,7 @@ bevy_executor = [ "dep:fastrand", "dep:slab", "dep:thread_local", + "dep:crossbeam-utils", "dep:pin-project-lite", "async-task/std", ] @@ -55,6 +56,7 @@ atomic-waker = { version = "1", default-features = false } crossbeam-queue = { version = "0.3", default-features = false, features = [ "alloc", ] } +crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index fe07ae862190f..345874c21feb5 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -24,6 +24,7 @@ use crossbeam_queue::{ArrayQueue, SegQueue}; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; +use crossbeam_utils::CachePadded; // ThreadLocalState *must* stay `Sync` due to a currently existing soundness hole. // See: https://github.com/Amanieu/thread_local-rs/issues/75 @@ -44,11 +45,11 @@ pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { } std::thread_local! { - static LOCAL_QUEUE: UnsafeCell = const { - UnsafeCell::new(LocalQueue { + static LOCAL_QUEUE: CachePadded> = const { + CachePadded::new(UnsafeCell::new(LocalQueue { local_queue: VecDeque::new(), local_active: Slab::new(), - }) + })) }; } @@ -765,7 +766,7 @@ impl Runner<'_> { } /// Waits for the next runnable task to run. - fn runnable(&mut self, rng: &mut fastrand::Rng) -> impl Future { + fn runnable(&mut self, _rng: &mut fastrand::Rng) -> impl Future { // SAFETY: The provided search function does not access LOCAL_QUEUE in any way, and thus cannot // alias. let runnable = unsafe { @@ -776,49 +777,53 @@ impl Runner<'_> { return Some(r); } - // Try the local queue. - if let Some(r) = self.local_state.stealable_queue.pop() { - return Some(r); - } - - // Try stealing from the global queue. - if let Some(r) = self.state.queue.pop() { - steal(&self.state.queue, &self.local_state.stealable_queue); - return Some(r); - } - - // Try stealing from other runners. - if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { - // Pick a random starting point in the iterator list and rotate the list. - let n = stealer_queues.len(); - let start = rng.usize(..n); - let iter = stealer_queues - .iter() - .chain(stealer_queues.iter()) - .skip(start) - .take(n); - - // Remove this runner's local queue. - let iter = - iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); - - // Try stealing from each local queue in the list. - for local in iter { - steal(*local, &self.local_state.stealable_queue); + crate::cfg::multi_threaded! { + if{ + // Try the local queue. if let Some(r) = self.local_state.stealable_queue.pop() { return Some(r); } - } - } - if let Some(r) = self.local_state.thread_locked_queue.pop() { - // Do not steal from this queue. If other threads steal - // from this current thread, the task will be moved. - // - // Instead, flush all queued tasks into the local queue to - // minimize the effort required to scan for these tasks. - flush_to_local(&self.local_state.thread_locked_queue, tls); - return Some(r); + // Try stealing from the global queue. + if let Some(r) = self.state.queue.pop() { + steal(&self.state.queue, &self.local_state.stealable_queue); + return Some(r); + } + + // Try stealing from other runners. + if let Ok(stealer_queues) = self.state.stealer_queues.try_read() { + // Pick a random starting point in the iterator list and rotate the list. + let n = stealer_queues.len(); + let start = _rng.usize(..n); + let iter = stealer_queues + .iter() + .chain(stealer_queues.iter()) + .skip(start) + .take(n); + + // Remove this runner's local queue. + let iter = + iter.filter(|local| !core::ptr::eq(**local, &self.local_state.stealable_queue)); + + // Try stealing from each local queue in the list. + for local in iter { + steal(*local, &self.local_state.stealable_queue); + if let Some(r) = self.local_state.stealable_queue.pop() { + return Some(r); + } + } + } + + if let Some(r) = self.local_state.thread_locked_queue.pop() { + // Do not steal from this queue. If other threads steal + // from this current thread, the task will be moved. + // + // Instead, flush all queued tasks into the local queue to + // minimize the effort required to scan for these tasks. + flush_to_local(&self.local_state.thread_locked_queue, tls); + return Some(r); + } + } else {} } None From 898166bb30c7c47d1d5f36ba833181f49c5c2cf9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:28:49 -0700 Subject: [PATCH 48/68] Fix Miri job to run them in sequence --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50fde2ced1ebb..bce2f347c7be8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,11 +118,13 @@ jobs: with: toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} components: miri - - name: CI job + - name: CI job (Tasks) # To run the tests one item at a time for troubleshooting, use # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_tasks --lib -- --exact - # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact run: cargo miri test -p bevy_tasks --features bevy_executor --features multi_threaded + - name: CI job (ECS) + # To run the tests one item at a time for troubleshooting, use + # cargo --quiet test --lib -- --list | sed 's/: test$//' | MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-weak-memory-emulation" xargs -n1 cargo miri test -p bevy_ecs --lib -- --exact run: cargo miri test -p bevy_ecs --features bevy_utils/debug check-compiles: From 2b967899f1a4a58b12687d8602dae59f6ee9265b Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:40:46 -0700 Subject: [PATCH 49/68] Fix up README and Clippy --- crates/bevy_tasks/README.md | 8 +++++--- crates/bevy_tasks/src/bevy_executor.rs | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/bevy_tasks/README.md b/crates/bevy_tasks/README.md index 04815df35e6ed..530489bfaa115 100644 --- a/crates/bevy_tasks/README.md +++ b/crates/bevy_tasks/README.md @@ -14,8 +14,8 @@ a single thread and having that thread await the completion of those tasks. This generating the tasks from a slice of data. This library is intended for games and makes no attempt to ensure fairness or ordering of spawned tasks. -It is based on [`async-executor`][async-executor], a lightweight executor that allows the end user to manage their own threads. -`async-executor` is based on async-task, a core piece of async-std. +It is based on a fork of [`async-executor`][async-executor], a lightweight executor that allows the end user to manage their own threads. +`async-executor` is based on [`async-task`][async-task], a core piece of [`smol`][smol]. ## Usage @@ -40,4 +40,6 @@ To enable `no_std` support in this crate, you will need to disable default featu [bevy]: https://bevy.org [rayon]: https://github.com/rayon-rs/rayon -[async-executor]: https://github.com/stjepang/async-executor +[async-executor]: https://github.com/smol-rs/async-executor +[smol]: https://github.com/smol-rs/smol +[async-task]: https://github.com/smol-rs/async-task diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 345874c21feb5..731b178195dd8 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1045,7 +1045,7 @@ mod test { use bevy_platform::sync::Mutex; use core::task::{Poll, Waker}; use async_task::Task; - use std::time::Duration; + use core::time::Duration; fn _ensure_send_and_sync() { fn is_send(_: T) {} @@ -1086,7 +1086,7 @@ mod test { future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1108,7 +1108,7 @@ mod test { let task: Task<&str> = ex.spawn(async { &*s }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1130,7 +1130,7 @@ mod test { }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1153,7 +1153,7 @@ mod test { }); future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); @@ -1173,12 +1173,13 @@ mod test { future::block_on(async move { let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; + #[expect(print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; let _ = stopper.recv().or(timeout).await; - }) + }); }); stop_timeout }; @@ -1209,7 +1210,7 @@ mod test { #[test] fn yield_now() { - do_run(|ex| async move { ex.spawn(future::yield_now()).await }) + do_run(|ex| async move { ex.spawn(future::yield_now()).await }); } #[test] @@ -1217,7 +1218,7 @@ mod test { do_run(|ex| async move { ex.spawn(async_io::Timer::after(Duration::from_millis(5))) .await; - }) + }); } #[test] @@ -1228,7 +1229,7 @@ mod test { // Running the executor should not panic. future::block_on(ex.run(async { for _ in 0..10 { - future::yield_now().await + future::yield_now().await; } })); From f5737f74b8be8d234cf5c8b543f8b4f09604da87 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 11 Aug 2025 18:52:12 -0700 Subject: [PATCH 50/68] It's clippy --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 731b178195dd8..71b5b2acaea7a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1173,7 +1173,7 @@ mod test { future::block_on(async move { let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; - #[expect(print_stderr, reason = "Explicitly used to warn about timed out tests")] + #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; From 84edd4f024da022fbb0801de93f915402c9ec29f Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 14 Aug 2025 20:23:09 -0700 Subject: [PATCH 51/68] Address UB in spawn_scoped_local from OOMs --- crates/bevy_tasks/src/bevy_executor.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 71b5b2acaea7a..7500cc8cec2c5 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -13,6 +13,7 @@ use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::task::{Context, Poll, Waker}; use core::cell::UnsafeCell; +use core::mem; use std::thread::{AccessError, ThreadId}; use alloc::collections::VecDeque; @@ -259,6 +260,8 @@ impl<'a> Executor<'a> { try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); + let builder = Builder::new().propagate_panic(true); + // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. This AsyncCallOnDrop will be // invoked after the surrounding scope has exited in either a @@ -267,6 +270,14 @@ impl<'a> Executor<'a> { try_with_local_queue(|tls| drop(tls.local_active.try_remove(index))).ok(); }); + // This is a critical section which will result in UB by aliasing active + // if the AsyncCallOnDrop is called while still in this function. + // + // To avoid this, this guard will abort the process if it does + // panic. Rust's drop order will ensure that this will run before + // executor, and thus before the above AsyncCallOnDrop is dropped. + let _panic_guard = AbortOnPanic; + // Create the task and register it in the set of active tasks. // // SAFETY: @@ -280,11 +291,12 @@ impl<'a> Executor<'a> { // must not leave the current thread of execution, and it does not // all of them are bound vy use of thread-local storage. // - `self.schedule_local()` is `'static`, as checked below. - let (runnable, task) = Builder::new() - .propagate_panic(true) + let (runnable, task) = builder .spawn_unchecked(|()| future, self.schedule_local()); entry.insert(runnable.waker()); + mem::forget(_panic_guard); + (runnable, task) }).unwrap() }; @@ -999,6 +1011,15 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re .finish() } +struct AbortOnPanic; + +impl Drop for AbortOnPanic { + fn drop(&mut self) { + // Panicking while unwinding will force an abort. + panic!("Aborting due to allocator error"); + } +} + /// Runs a closure when dropped. struct CallOnDrop(F); From dc7ce63e3da308bea81c4a443f8dff93c02d3e9a Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 14 Aug 2025 20:23:58 -0700 Subject: [PATCH 52/68] Move expect to outer block --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 7500cc8cec2c5..39b7a4a9dda28 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -1192,9 +1192,9 @@ mod test { let (stop_timeout, stopper) = async_channel::bounded::<()>(1); std::thread::spawn(move || { future::block_on(async move { + #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] let timeout = async { async_io::Timer::after(Duration::from_secs(2 * 60)).await; - #[expect(clippy::print_stderr, reason = "Explicitly used to warn about timed out tests")] std::eprintln!("test timed out after 2m"); std::process::exit(1) }; From 00b62c27bb25604e6b133a039d3575be3b783296 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 18 Aug 2025 17:06:23 -0700 Subject: [PATCH 53/68] Fix lint --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 39b7a4a9dda28..6ce0900e7b434 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -430,7 +430,7 @@ impl<'a> Executor<'a> { // Arc when accessed through state_ptr. let arc = unsafe { Arc::from_raw(self.state_ptr()) }; let clone = arc.clone(); - core::mem::forget(arc); + mem::forget(arc); clone } } From 5d122a01a97384040e45ab0bafaa60541cb91f03 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 18 Aug 2025 22:25:31 -0700 Subject: [PATCH 54/68] Go back to using ConcurrentQueue --- crates/bevy_tasks/Cargo.toml | 5 +- crates/bevy_tasks/src/bevy_executor.rs | 79 +++++++++----------------- crates/bevy_tasks/src/edge_executor.rs | 6 +- 3 files changed, 31 insertions(+), 59 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index b44a608c39650..aa2b47a3e4bb2 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -25,6 +25,7 @@ bevy_executor = [ "dep:pin-project-lite", "futures-lite", "async-task/std", + "concurrent-queue/std", ] # Provide an implementation of `block_on` from `futures-lite`. @@ -54,9 +55,7 @@ fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } -crossbeam-queue = { version = "0.3", default-features = false, features = [ - "alloc", -] } +concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 6ce0900e7b434..0c8a669563a55 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -21,7 +21,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use crossbeam_queue::{ArrayQueue, SegQueue}; +use concurrent_queue::{ConcurrentQueue, PushError}; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -83,16 +83,16 @@ impl Drop for LocalQueue { struct ThreadLocalState { executor: AtomicPtr, - stealable_queue: ArrayQueue, - thread_locked_queue: SegQueue, + stealable_queue: ConcurrentQueue, + thread_locked_queue: ConcurrentQueue, } impl Default for ThreadLocalState { fn default() -> Self { Self { executor: AtomicPtr::new(core::ptr::null_mut()), - stealable_queue: ArrayQueue::new(512), - thread_locked_queue: SegQueue::new(), + stealable_queue: ConcurrentQueue::bounded(512), + thread_locked_queue: ConcurrentQueue::unbounded(), } } } @@ -104,7 +104,7 @@ impl Default for ThreadLocalState { #[derive(Clone, Debug)] pub struct ThreadSpawner<'a> { thread_id: ThreadId, - target_queue: &'static SegQueue, + target_queue: &'static ConcurrentQueue, state: Arc, _marker: PhantomData<&'a ()>, } @@ -360,7 +360,7 @@ impl<'a> Executor<'a> { state.notify_specific_thread(std::thread::current().id(), true); return; } - Err(r) => r, + Err(r) => r.into_inner(), } } else { runnable @@ -452,17 +452,17 @@ impl Drop for Executor<'_> { } drop(active); - while state.queue.pop().is_some() {} + while state.queue.pop().is_ok() {} } } /// The state of a executor. struct State { /// The global queue. - queue: SegQueue, + queue: ConcurrentQueue, /// Local queues created by runners. - stealer_queues: RwLock>>, + stealer_queues: RwLock>>, /// Set to `true` when a sleeping ticker is notified or no tickers are sleeping. notified: AtomicBool, @@ -478,7 +478,7 @@ impl State { /// Creates state for a new executor. const fn new() -> State { State { - queue: SegQueue::new(), + queue: ConcurrentQueue::unbounded(), stealer_queues: RwLock::new(Vec::new()), notified: AtomicBool::new(true), sleepers: Mutex::new(Sleepers { @@ -792,12 +792,12 @@ impl Runner<'_> { crate::cfg::multi_threaded! { if{ // Try the local queue. - if let Some(r) = self.local_state.stealable_queue.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } // Try stealing from the global queue. - if let Some(r) = self.state.queue.pop() { + if let Ok(r) = self.state.queue.pop() { steal(&self.state.queue, &self.local_state.stealable_queue); return Some(r); } @@ -820,13 +820,13 @@ impl Runner<'_> { // Try stealing from each local queue in the list. for local in iter { steal(*local, &self.local_state.stealable_queue); - if let Some(r) = self.local_state.stealable_queue.pop() { + if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); } } } - if let Some(r) = self.local_state.thread_locked_queue.pop() { + if let Ok(r) = self.local_state.thread_locked_queue.pop() { // Do not steal from this queue. If other threads steal // from this current thread, the task will be moved. // @@ -870,65 +870,38 @@ impl Drop for Runner<'_> { } // Re-schedule remaining tasks in the local queue. - while let Some(r) = self.local_state.stealable_queue.pop() { + while let Ok(r) = self.local_state.stealable_queue.pop() { r.schedule(); } } } -trait WorkQueue { - fn stealable_count(&self) -> usize; - fn queue_pop(&self) -> Option; -} - -impl WorkQueue for ArrayQueue { - #[inline] - fn stealable_count(&self) -> usize { - self.len().div_ceil(2) - } - - #[inline] - fn queue_pop(&self) -> Option { - self.pop() - } -} - -impl WorkQueue for SegQueue { - #[inline] - fn stealable_count(&self) -> usize { - self.len() - } - - #[inline] - fn queue_pop(&self) -> Option { - self.pop() - } -} - /// Steals some items from one queue into another. -fn steal>(src: &Q, dest: &ArrayQueue) { +fn steal(src: &ConcurrentQueue, dest: &ConcurrentQueue) { // Half of `src`'s length rounded up. - let mut count = src.stealable_count(); + let mut count = src.len(); if count > 0 { - // Don't steal more than fits into the queue. - count = count.min(dest.capacity() - dest.len()); + if let Some(capacity) = dest.capacity() { + // Don't steal more than fits into the queue. + count = count.min(capacity- dest.len()); + } // Steal tasks. for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; + let Ok(val) = src.pop() else { break }; assert!(dest.push(val).is_ok()); } } } -fn flush_to_local(src: &SegQueue, dst: &mut LocalQueue) { +fn flush_to_local(src: &ConcurrentQueue, dst: &mut LocalQueue) { let count = src.len(); if count > 0 { // Steal tasks. for _ in 0..count { - let Some(val) = src.queue_pop() else { break }; + let Ok(val) = src.pop() else { break }; dst.local_queue.push_front(val); } } @@ -975,7 +948,7 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } /// Debug wrapper for the local runners. - struct LocalRunners<'a>(&'a RwLock>>); + struct LocalRunners<'a>(&'a RwLock>>); impl fmt::Debug for LocalRunners<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index dc17077aef66d..be0589c7beff4 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -209,7 +209,7 @@ impl<'a, const C: usize> Executor<'a, C> { target_has_atomic = "ptr" ))] { - runnable = self.state().queue.pop(); + runnable = self.state().queue.pop().ok(); } #[cfg(not(all( @@ -314,7 +314,7 @@ struct State { target_has_atomic = "64", target_has_atomic = "ptr" ))] - queue: crossbeam_queue::ArrayQueue, + queue: concurrent_queue::ConcurrentQueue, #[cfg(not(all( target_has_atomic = "8", target_has_atomic = "16", @@ -336,7 +336,7 @@ impl State { target_has_atomic = "64", target_has_atomic = "ptr" ))] - queue: crossbeam_queue::ArrayQueue::new(C), + queue: concurrent_queue::ConcurrentQueue::bounded(C), #[cfg(not(all( target_has_atomic = "8", target_has_atomic = "16", From 73798b1908efe2a564afa7a60e0d3c6b34be548b Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 00:53:15 -0700 Subject: [PATCH 55/68] Don't panic --- crates/bevy_tasks/src/bevy_executor.rs | 24 +++++++++-------- crates/bevy_tasks/src/task_pool.rs | 36 +++++++++++++------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 0c8a669563a55..750575c71c879 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -21,7 +21,7 @@ use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; -use concurrent_queue::{ConcurrentQueue, PushError}; +use concurrent_queue::ConcurrentQueue; use futures_lite::{future,FutureExt}; use slab::Slab; use thread_local::ThreadLocal; @@ -156,7 +156,8 @@ impl<'a> ThreadSpawner<'a> { // Instead of directly scheduling this task, it's put into the onto the // thread locked queue to be moved to the target thread, where it will // either be run immediately or flushed into the thread's local queue. - self.target_queue.push(runnable); + let result = self.target_queue.push(runnable); + debug_assert!(result.is_ok()); task } @@ -369,7 +370,8 @@ impl<'a> Executor<'a> { runnable }; // Otherwise push onto the global queue instead. - state.queue.push(runnable); + let result = state.queue.push(runnable); + debug_assert!(result.is_ok()); state.notify(); } } @@ -503,7 +505,7 @@ impl State { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_ok() { - let waker = self.sleepers.lock().unwrap().notify(); + let waker = self.sleepers.lock().unwrap_or_else(PoisonError::into_inner).notify(); if let Some(w) = waker { w.wake(); } @@ -513,7 +515,7 @@ impl State { /// Notifies a sleeping ticker. #[inline] fn notify_specific_thread(&self, thread_id: ThreadId, allow_stealing: bool) { - let mut sleepers = self.sleepers.lock().unwrap(); + let mut sleepers = self.sleepers.lock().unwrap_or_else(PoisonError::into_inner); let mut waker = sleepers.notify_specific_thread(thread_id); if waker.is_none() && allow_stealing @@ -649,7 +651,7 @@ impl Ticker<'_> { /// /// Returns `false` if the ticker was already sleeping and unnotified. fn sleep(&mut self, waker: &Waker) -> bool { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); match self.sleeping { // Move to sleeping state. @@ -675,7 +677,7 @@ impl Ticker<'_> { /// Moves the ticker into woken state. fn wake(&mut self) { if self.sleeping != 0 { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); sleepers.remove(self.sleeping); self.state @@ -716,7 +718,7 @@ impl Ticker<'_> { } } } - }).unwrap() + }).unwrap_or(Poll::Pending) } }) } @@ -726,7 +728,7 @@ impl Drop for Ticker<'_> { fn drop(&mut self) { // If this ticker is in sleeping state, it must be removed from the sleepers list. if self.sleeping != 0 { - let mut sleepers = self.state.sleepers.lock().unwrap(); + let mut sleepers = self.state.sleepers.lock().unwrap_or_else(PoisonError::into_inner); let notified = sleepers.remove(self.sleeping); self.state @@ -772,7 +774,7 @@ impl Runner<'_> { state .stealer_queues .write() - .unwrap() + .unwrap_or_else(PoisonError::into_inner) .push(&local_state.stealable_queue); runner } @@ -790,7 +792,7 @@ impl Runner<'_> { } crate::cfg::multi_threaded! { - if{ + if { // Try the local queue. if let Ok(r) = self.local_state.stealable_queue.pop() { return Some(r); diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 086e05d4e0614..270089b59f34f 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -5,7 +5,7 @@ use std::thread::{self, JoinHandle}; use crate::bevy_executor::Executor; use async_task::FallibleTask; use bevy_platform::sync::Arc; -use crossbeam_queue::SegQueue; +use concurrent_queue::ConcurrentQueue; use futures_lite::FutureExt; use crate::{block_on, Task}; @@ -347,11 +347,11 @@ impl TaskPool { let executor: &Executor = &self.executor; // SAFETY: As above, all futures must complete in this function so we can change the lifetime let executor: &'env Executor = unsafe { mem::transmute(executor) }; - let spawned: SegQueue>>> = - SegQueue::new(); + let spawned: ConcurrentQueue>>> = + ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env SegQueue>>> = + let spawned: &'env ConcurrentQueue>>> = unsafe { mem::transmute(&spawned) }; let scope = Scope { @@ -374,14 +374,11 @@ impl TaskPool { } else { block_on(self.executor.run(async move { let mut results = Vec::with_capacity(spawned.len()); - while let Some(task) = spawned.pop() { - if let Some(res) = task.await { - match res { - Ok(res) => results.push(res), - Err(payload) => std::panic::resume_unwind(payload), - } - } else { - panic!("Failed to catch panic!"); + while let Ok(task) = spawned.pop() { + match task.await { + Some(Ok(res)) => results.push(res), + Some(Err(payload)) => std::panic::resume_unwind(payload), + None => panic!("Failed to catch panic!"), } } results @@ -455,7 +452,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope Executor<'scope>, external_spawner: ThreadSpawner<'scope>, scope_spawner: ThreadSpawner<'scope>, - spawned: &'scope SegQueue>>>, + spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, @@ -475,7 +472,8 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .executor .spawn(AssertUnwindSafe(f).catch_unwind()) .fallible(); - self.spawned.push(task); + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } #[expect( @@ -496,7 +494,8 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) .fallible() }; - self.spawned.push(task); + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } #[expect( @@ -519,8 +518,9 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { .fallible() }; // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so it is safe to unwrap - self.spawned.push(task); + // close and use an unbounded queue, so pushing should always succeed. + let result = self.spawned.push(task); + debug_assert!(result.is_ok()); } } @@ -530,7 +530,7 @@ where { fn drop(&mut self) { block_on(async { - while let Some(task) = self.spawned.pop() { + while let Ok(task) = self.spawned.pop() { task.cancel().await; } }); From 909f2e53324e0d95e4931bb1b585c497a16e8086 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 00:56:10 -0700 Subject: [PATCH 56/68] Formatting --- crates/bevy_tasks/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index aa2b47a3e4bb2..0eb81732728c0 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -55,7 +55,7 @@ fastrand = { version = "2.3", optional = true, default-features = false } async-channel = { version = "2.3.0", optional = true } async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } -concurrent-queue = { version = "2.5", default-features = false } +concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] From f75719f56643843e0b202371223c8a575d114a71 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 01:06:22 -0700 Subject: [PATCH 57/68] Static Executors --- crates/bevy_app/src/task_pool_plugin.rs | 6 +- crates/bevy_ecs/src/lib.rs | 6 +- .../src/schedule/executor/multi_threaded.rs | 6 +- crates/bevy_ecs/src/schedule/mod.rs | 4 +- crates/bevy_tasks/src/bevy_executor.rs | 321 +++--------------- crates/bevy_tasks/src/edge_executor.rs | 53 +-- .../src/single_threaded_task_pool.rs | 28 +- crates/bevy_tasks/src/task_pool.rs | 66 ++-- crates/bevy_tasks/src/usages.rs | 23 +- crates/bevy_transform/src/systems.rs | 16 +- 10 files changed, 168 insertions(+), 361 deletions(-) diff --git a/crates/bevy_app/src/task_pool_plugin.rs b/crates/bevy_app/src/task_pool_plugin.rs index 8014790f07772..f38d09d7f5e91 100644 --- a/crates/bevy_app/src/task_pool_plugin.rs +++ b/crates/bevy_app/src/task_pool_plugin.rs @@ -190,7 +190,7 @@ impl TaskPoolOptions { builder }; - builder.build() + builder }); } @@ -220,7 +220,7 @@ impl TaskPoolOptions { builder }; - builder.build() + builder }); } @@ -250,7 +250,7 @@ impl TaskPoolOptions { builder }; - builder.build() + builder }); } } diff --git a/crates/bevy_ecs/src/lib.rs b/crates/bevy_ecs/src/lib.rs index 974c371bf31d0..8cddefae56bd2 100644 --- a/crates/bevy_ecs/src/lib.rs +++ b/crates/bevy_ecs/src/lib.rs @@ -170,7 +170,7 @@ mod tests { }; use alloc::{string::String, sync::Arc, vec, vec::Vec}; use bevy_platform::collections::HashSet; - use bevy_tasks::{ComputeTaskPool, TaskPool}; + use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use core::{ any::TypeId, marker::PhantomData, @@ -495,7 +495,7 @@ mod tests { #[test] fn par_for_each_dense() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); let e1 = world.spawn(A(1)).id(); let e2 = world.spawn(A(2)).id(); @@ -517,7 +517,7 @@ mod tests { #[test] fn par_for_each_sparse() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); let e1 = world.spawn(SparseStored(1)).id(); let e2 = world.spawn(SparseStored(2)).id(); diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index b16e6e22a7b53..9b06549860b2f 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, vec::Vec}; use bevy_platform::cell::SyncUnsafeCell; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadSpawner}; +use bevy_tasks::{ComputeTaskPool, Scope, TaskPoolBuilder, ThreadSpawner}; use concurrent_queue::ConcurrentQueue; use core::{any::Any, panic::AssertUnwindSafe}; use fixedbitset::FixedBitSet; @@ -274,7 +274,7 @@ impl SystemExecutor for MultiThreadedExecutor { let environment = &Environment::new(self, schedule, world); - ComputeTaskPool::get_or_init(TaskPool::default).scope_with_executor( + ComputeTaskPool::get_or_init(TaskPoolBuilder::default).scope_with_executor( thread_executor, |scope| { let context = Context { @@ -863,7 +863,7 @@ unsafe fn evaluate_and_fold_conditions( /// New-typed [`ThreadSpawner`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Clone)] -pub struct MainThreadSpawner(pub ThreadSpawner<'static>); +pub struct MainThreadSpawner(pub ThreadSpawner); impl Default for MainThreadSpawner { fn default() -> Self { diff --git a/crates/bevy_ecs/src/schedule/mod.rs b/crates/bevy_ecs/src/schedule/mod.rs index 1b01e031ef978..f19efb49dfa3b 100644 --- a/crates/bevy_ecs/src/schedule/mod.rs +++ b/crates/bevy_ecs/src/schedule/mod.rs @@ -110,12 +110,12 @@ mod tests { #[cfg(not(miri))] fn parallel_execution() { use alloc::sync::Arc; - use bevy_tasks::{ComputeTaskPool, TaskPool}; + use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use std::sync::Barrier; let mut world = World::default(); let mut schedule = Schedule::default(); - let thread_count = ComputeTaskPool::get_or_init(TaskPool::default).thread_num(); + let thread_count = ComputeTaskPool::get_or_init(TaskPoolBuilder::default).thread_num(); let barrier = Arc::new(Barrier::new(thread_count)); diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 750575c71c879..d167a39fa9e1a 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -7,7 +7,6 @@ reason = "Not all functions are used with every feature combination" )] -use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; @@ -20,7 +19,7 @@ use alloc::collections::VecDeque; use alloc::fmt; use async_task::{Builder, Runnable, Task}; use bevy_platform::prelude::Vec; -use bevy_platform::sync::{Arc, Mutex, MutexGuard, PoisonError, RwLock, TryLockError}; +use bevy_platform::sync::{Mutex, PoisonError, RwLock, TryLockError}; use concurrent_queue::ConcurrentQueue; use futures_lite::{future,FutureExt}; use slab::Slab; @@ -31,17 +30,12 @@ use crossbeam_utils::CachePadded; // See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); -pub(crate) fn install_runtime_into_current_thread(executor: &Executor) { +pub(crate) fn install_runtime_into_current_thread(executor: &'static Executor) { // Use LOCAL_QUEUE here to set the thread destructor LOCAL_QUEUE.with(|_| { let tls = THREAD_LOCAL_STATE.get_or_default(); - let state = executor.state_as_arc(); - let state_ptr = Arc::into_raw(state).cast_mut(); - let old_ptr = tls.executor.swap(state_ptr, Ordering::Relaxed); - if !old_ptr.is_null() { - // SAFETY: If this pointer was not null, it was initialized from Arc::into_raw. - drop(unsafe { Arc::from_raw(old_ptr) }); - } + let state_ptr: *const State = &executor.state; + tls.executor.swap(state_ptr.cast_mut(), Ordering::Relaxed); }); } @@ -102,14 +96,13 @@ impl Default for ThreadLocalState { /// /// [`TaskPool::current_thread_spawner`]: crate::TaskPool::current_thread_spawner #[derive(Clone, Debug)] -pub struct ThreadSpawner<'a> { +pub struct ThreadSpawner { thread_id: ThreadId, target_queue: &'static ConcurrentQueue, - state: Arc, - _marker: PhantomData<&'a ()>, + state: &'static State, } -impl<'a> ThreadSpawner<'a> { +impl ThreadSpawner { /// Spawns a task onto the specific target thread. pub fn spawn( &self, @@ -123,18 +116,10 @@ impl<'a> ThreadSpawner<'a> { /// /// # Safety /// The caller must ensure that the returned Task does not outlive 'a. - pub unsafe fn spawn_scoped( + pub unsafe fn spawn_scoped<'a, T: Send + 'a>( &self, future: impl Future + Send + 'a, ) -> Task { - let mut active = self.state.active(); - - // Remove the task from the set of active tasks when the future finishes. - let entry = active.vacant_entry(); - let index = entry.key(); - let state = self.state.clone(); - let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); - // Create the task and register it in the set of active tasks. // // SAFETY: @@ -151,7 +136,6 @@ impl<'a> ThreadSpawner<'a> { .propagate_panic(true) .spawn_unchecked(|()| future, self.schedule()) }; - entry.insert(runnable.waker()); // Instead of directly scheduling this task, it's put into the onto the // thread locked queue to be moved to the target thread, where it will @@ -164,7 +148,7 @@ impl<'a> ThreadSpawner<'a> { /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { let thread_id = self.thread_id; - let state = self.state.clone(); + let state = self.state; move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed @@ -178,42 +162,35 @@ impl<'a> ThreadSpawner<'a> { } /// An async executor. -pub struct Executor<'a> { +pub struct Executor { /// The executor state. - state: AtomicPtr, - - /// Makes the `'a` lifetime invariant. - _marker: PhantomData<&'a ()>, + state: State, } -impl UnwindSafe for Executor<'_> {} -impl RefUnwindSafe for Executor<'_> {} +impl UnwindSafe for Executor {} +impl RefUnwindSafe for Executor {} -impl fmt::Debug for Executor<'_> { +impl fmt::Debug for Executor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { debug_executor(self, "Executor", f) } } -impl<'a> Executor<'a> { +impl Executor { /// Creates a new executor. - pub const fn new() -> Executor<'a> { + pub const fn new() -> Executor { Executor { - state: AtomicPtr::new(core::ptr::null_mut()), - _marker: PhantomData, + state: State::new() } } /// Spawns a task onto the executor. - pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { - let mut active = self.state().active(); - - // Remove the task from the set of active tasks when the future finishes. - let entry = active.vacant_entry(); - let index = entry.key(); - let state = self.state_as_arc(); - let future = AsyncCallOnDrop::new(future, move || drop(state.active().try_remove(index))); + pub fn spawn(&'static self, future: impl Future + Send + 'static) -> Task { + // SAFETY: Both `T` and `future` are 'static. + unsafe { self.spawn_scoped(future) } + } + pub unsafe fn spawn_scoped<'a, T: Send + 'a>(&'static self, future: impl Future + Send + 'a) -> Task { // Create the task and register it in the set of active tasks. // // SAFETY: @@ -233,14 +210,13 @@ impl<'a> Executor<'a> { .propagate_panic(true) .spawn_unchecked(|()| future, self.schedule()) }; - entry.insert(runnable.waker()); runnable.schedule(); task } /// Spawns a non-Send task onto the executor. - pub fn spawn_local(&self, future: impl Future + 'static) -> Task { + pub fn spawn_local(&'static self, future: impl Future + 'static) -> Task { // SAFETY: future is 'static unsafe { self.spawn_local_scoped(future) } } @@ -249,8 +225,8 @@ impl<'a> Executor<'a> { /// /// # Safety /// The caller must ensure that the returned Task does not outlive 'a. - pub unsafe fn spawn_local_scoped( - &self, + pub unsafe fn spawn_local_scoped<'a, T: 'a>( + &'static self, future: impl Future + 'a, ) -> Task { // Remove the task from the set of active tasks when the future finishes. @@ -306,12 +282,11 @@ impl<'a> Executor<'a> { task } - pub fn current_thread_spawner(&self) -> ThreadSpawner<'a> { + pub fn current_thread_spawner(&'static self) -> ThreadSpawner { ThreadSpawner { thread_id: std::thread::current().id(), target_queue: &THREAD_LOCAL_STATE.get_or_default().thread_locked_queue, - state: self.state_as_arc(), - _marker: PhantomData, + state: &self.state, } } @@ -328,8 +303,8 @@ impl<'a> Executor<'a> { } /// Runs the executor until the given future completes. - pub fn run<'b, T>(&'b self, future: impl Future + 'b) -> impl Future + 'b { - let mut runner = Runner::new(self.state()); + pub fn run<'b, T>(&'static self, future: impl Future + 'b) -> impl Future + 'b { + let mut runner = Runner::new(&self.state); // A future that runs tasks forever. let run_forever = async move { @@ -348,14 +323,14 @@ impl<'a> Executor<'a> { } /// Returns a function that schedules a runnable task when it gets woken up. - fn schedule(&self) -> impl Fn(Runnable) + Send + Sync + 'static { - let state = self.state_as_arc(); + fn schedule(&'static self) -> impl Fn(Runnable) + Send + Sync + 'static { + let state = &self.state; move |runnable| { // Attempt to push onto the local queue first in dedicated executor threads, // because we know that this thread is awake and always processing new tasks. let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), Arc::as_ptr(&state)) { + if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), state) { match local_state.stealable_queue.push(runnable) { Ok(()) => { state.notify_specific_thread(std::thread::current().id(), true); @@ -377,8 +352,8 @@ impl<'a> Executor<'a> { } /// Returns a function that schedules a runnable task when it gets woken up. - fn schedule_local(&self) -> impl Fn(Runnable) + 'static { - let state = self.state_as_arc(); + fn schedule_local(&'static self) -> impl Fn(Runnable) + 'static { + let state = &self.state; move |runnable| { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably @@ -388,74 +363,6 @@ impl<'a> Executor<'a> { } } } - - /// Returns a pointer to the inner state. - #[inline] - fn state_ptr(&self) -> *const State { - #[cold] - fn alloc_state(atomic_ptr: &AtomicPtr) -> *mut State { - let state = Arc::new(State::new()); - let ptr = Arc::into_raw(state).cast_mut(); - if let Err(actual) = atomic_ptr.compare_exchange( - core::ptr::null_mut(), - ptr, - Ordering::AcqRel, - Ordering::Acquire, - ) { - // SAFETY: This was just created from Arc::into_raw. - drop(unsafe { Arc::from_raw(ptr) }); - actual - } else { - ptr - } - } - - let mut ptr = self.state.load(Ordering::Acquire); - if ptr.is_null() { - ptr = alloc_state(&self.state); - } - ptr - } - - /// Returns a reference to the inner state. - #[inline] - fn state(&self) -> &State { - // SAFETY: So long as an Executor lives, it's state pointer will always be valid - // when accessed through state_ptr. - unsafe { &*self.state_ptr() } - } - - // Clones the inner state Arc - #[inline] - fn state_as_arc(&self) -> Arc { - // SAFETY: So long as an Executor lives, it's state pointer will always be a valid - // Arc when accessed through state_ptr. - let arc = unsafe { Arc::from_raw(self.state_ptr()) }; - let clone = arc.clone(); - mem::forget(arc); - clone - } -} - -impl Drop for Executor<'_> { - fn drop(&mut self) { - let ptr = *self.state.get_mut(); - if ptr.is_null() { - return; - } - - // SAFETY: As ptr is not null, it was allocated via Arc::new and converted - // via Arc::into_raw in state_ptr. - let state = unsafe { Arc::from_raw(ptr) }; - - let mut active = state.active(); - for w in active.drain() { - w.wake(); - } - drop(active); - - while state.queue.pop().is_ok() {} - } } /// The state of a executor. @@ -471,9 +378,6 @@ struct State { /// A list of sleeping tickers. sleepers: Mutex, - - /// Currently active tasks. - active: Mutex>, } impl State { @@ -488,15 +392,9 @@ impl State { wakers: Vec::new(), free_ids: Vec::new(), }), - active: Mutex::new(Slab::new()), } } - /// Returns a reference to currently active tasks. - fn active(&self) -> MutexGuard<'_, Slab> { - self.active.lock().unwrap_or_else(PoisonError::into_inner) - } - /// Notifies a sleeping ticker. #[inline] fn notify(&self) { @@ -910,28 +808,8 @@ fn flush_to_local(src: &ConcurrentQueue, dst: &mut LocalQueue) { } /// Debug implementation for `Executor`. -fn debug_executor(executor: &Executor<'_>, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Get a reference to the state. - let ptr = executor.state.load(Ordering::Acquire); - if ptr.is_null() { - // The executor has not been initialized. - struct Uninitialized; - - impl fmt::Debug for Uninitialized { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("") - } - } - - return f.debug_tuple(name).field(&Uninitialized).finish(); - } - - // SAFETY: If the state pointer is not null, it must have been - // allocated properly by Arc::new and converted via Arc::into_raw - // in state_ptr. - let state = unsafe { &*ptr }; - - debug_state(state, name, f) +fn debug_executor(executor: &Executor, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_state(&executor.state, name, f) } /// Debug implementation for `Executor`. @@ -979,7 +857,6 @@ fn debug_state(state: &State, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Re } f.debug_struct(name) - .field("active", &ActiveTasks(&state.active)) .field("global_tasks", &state.queue.len()) .field("stealer_queues", &LocalRunners(&state.stealer_queues)) .field("sleepers", &SleepCount(&state.sleepers)) @@ -1043,124 +920,42 @@ mod test { use async_task::Task; use core::time::Duration; + static EX: Executor = Executor::new(); + fn _ensure_send_and_sync() { fn is_send(_: T) {} fn is_sync(_: T) {} fn is_static(_: T) {} - is_send::>(Executor::new()); - is_sync::>(Executor::new()); + is_send::(Executor::new()); + is_sync::(Executor::new()); - let ex = Executor::new(); - is_send(ex.schedule()); - is_sync(ex.schedule()); - is_static(ex.schedule()); - is_send(ex.current_thread_spawner()); - is_sync(ex.current_thread_spawner()); + is_send(EX.schedule()); + is_sync(EX.schedule()); + is_static(EX.schedule()); + is_send(EX.current_thread_spawner()); + is_sync(EX.current_thread_spawner()); is_send(THREAD_LOCAL_STATE.get_or_default()); is_sync(THREAD_LOCAL_STATE.get_or_default()); } - #[test] - fn executor_cancels_everything() { - static DROP: AtomicUsize = AtomicUsize::new(0); - static WAKER: Mutex> = Mutex::new(None); - - let ex = Executor::new(); - - let task = ex.spawn(async { - let _guard = CallOnDrop(|| { - DROP.fetch_add(1, Ordering::SeqCst); - }); - - future::poll_fn(|cx| { - *WAKER.lock().unwrap() = Some(cx.waker().clone()); - Poll::Pending::<()> - }) - .await; - }); - - future::block_on(ex.run(async { - for _ in 0..10 { - future::yield_now().await; - } - })); - - assert!(WAKER.lock().unwrap().is_some()); - assert_eq!(DROP.load(Ordering::SeqCst), 0); - - drop(ex); - assert_eq!(DROP.load(Ordering::SeqCst), 1); - - assert!(catch_unwind(|| future::block_on(task)).is_err()); - assert_eq!(DROP.load(Ordering::SeqCst), 1); - } - #[test] fn await_task_after_dropping_executor() { let s: String = "hello".into(); - let ex = Executor::new(); - let task: Task<&str> = ex.spawn(async { &*s }); - future::block_on(ex.run(async { + // SAFETY: We make sure that the task does not outlive the borrow on `s`. + let task: Task<&str> = unsafe { EX.spawn_scoped(async { &*s }) }; + future::block_on(EX.run(async { for _ in 0..10 { future::yield_now().await; } })); - - drop(ex); assert_eq!(future::block_on(task), "hello"); drop(s); } - #[test] - fn drop_executor_and_then_drop_finished_task() { - static DROP: AtomicUsize = AtomicUsize::new(0); - - let ex = Executor::new(); - let task = ex.spawn(async { - CallOnDrop(|| { - DROP.fetch_add(1, Ordering::SeqCst); - }) - }); - future::block_on(ex.run(async { - for _ in 0..10 { - future::yield_now().await; - } - })); - - assert_eq!(DROP.load(Ordering::SeqCst), 0); - drop(ex); - assert_eq!(DROP.load(Ordering::SeqCst), 0); - drop(task); - assert_eq!(DROP.load(Ordering::SeqCst), 1); - } - - #[test] - fn drop_finished_task_and_then_drop_executor() { - static DROP: AtomicUsize = AtomicUsize::new(0); - - let ex = Executor::new(); - let task = ex.spawn(async { - CallOnDrop(|| { - DROP.fetch_add(1, Ordering::SeqCst); - }) - }); - future::block_on(ex.run(async { - for _ in 0..10 { - future::yield_now().await; - } - })); - - assert_eq!(DROP.load(Ordering::SeqCst), 0); - drop(task); - assert_eq!(DROP.load(Ordering::SeqCst), 1); - drop(ex); - assert_eq!(DROP.load(Ordering::SeqCst), 1); - } - - fn do_run>(mut f: impl FnMut(Arc>) -> Fut) { + fn do_run>(mut f: impl FnMut(&'static Executor) -> Fut) { // This should not run for longer than two minutes. #[cfg(not(miri))] let _stop_timeout = { @@ -1180,10 +975,8 @@ mod test { stop_timeout }; - let ex = Arc::new(Executor::new()); - // Test 1: Use the `run` command. - future::block_on(ex.run(f(ex.clone()))); + future::block_on(EX.run(f(&EX))); // Test 2: Run on many threads. std::thread::scope(|scope| { @@ -1191,11 +984,11 @@ mod test { for _ in 0..16 { let shutdown = shutdown.clone(); - let ex = &ex; + let ex = &EX; scope.spawn(move || future::block_on(ex.run(shutdown.recv()))); } - future::block_on(f(ex.clone())); + future::block_on(f(&EX)); }); } @@ -1219,11 +1012,10 @@ mod test { #[test] fn test_panic_propagation() { - let ex = Executor::new(); - let task = ex.spawn(async { panic!("should be caught by the task") }); + let task = EX.spawn(async { panic!("should be caught by the task") }); // Running the executor should not panic. - future::block_on(ex.run(async { + future::block_on(EX.run(async { for _ in 0..10 { future::yield_now().await; } @@ -1237,10 +1029,9 @@ mod test { fn two_queues() { future::block_on(async { // Create an executor with two runners. - let ex = Executor::new(); let (run1, run2) = ( - ex.run(future::pending::<()>()), - ex.run(future::pending::<()>()), + EX.run(future::pending::<()>()), + EX.run(future::pending::<()>()), ); let mut run1 = Box::pin(run1); pin!(run2); diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index be0589c7beff4..612fe65b580dc 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -48,12 +48,11 @@ use futures_lite::FutureExt; /// drop(signal); /// })); /// ``` -pub struct Executor<'a, const C: usize = 64> { - state: LazyLock>>, - _invariant: PhantomData>, +pub struct Executor { + state: LazyLock>, } -impl<'a, const C: usize> Executor<'a, C> { +impl Executor { /// Creates a new executor. /// /// # Examples @@ -65,8 +64,7 @@ impl<'a, const C: usize> Executor<'a, C> { /// ``` pub const fn new() -> Self { Self { - state: LazyLock::new(|| Arc::new(State::new())), - _invariant: PhantomData, + state: LazyLock::new(|| State::new()), } } @@ -87,7 +85,16 @@ impl<'a, const C: usize> Executor<'a, C> { /// Note that if the executor's queue size is equal to the number of currently /// spawned and running tasks, spawning this additional task might cause the executor to panic /// later, when the task is scheduled for polling. - pub fn spawn(&self, fut: F) -> Task + pub fn spawn(&'static self, fut: F) -> Task + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + // SAFETY: Original implementation missing safety documentation + unsafe { self.spawn_unchecked(fut) } + } + + pub unsafe fn spawn_scoped<'a, F>(&'static self, fut: F) -> Task where F: Future + Send + 'a, F::Output: Send + 'a, @@ -96,7 +103,16 @@ impl<'a, const C: usize> Executor<'a, C> { unsafe { self.spawn_unchecked(fut) } } - pub fn spawn_local(&self, fut: F) -> Task + pub fn spawn_local(&'static self, fut: F) -> Task + where + F: Future + 'static, + F::Output: 'static, + { + // SAFETY: Original implementation missing safety documentation + unsafe { self.spawn_unchecked(fut) } + } + + pub unsafe fn spawn_local_scoped<'a, F>(&'static self, fut: F) -> Task where F: Future + 'a, F::Output: 'a, @@ -168,7 +184,7 @@ impl<'a, const C: usize> Executor<'a, C> { /// /// assert_eq!(res, 6); /// ``` - pub async fn run(&self, fut: F) -> F::Output + pub async fn run<'a, F>(&'static self, fut: F) -> F::Output where F: Future + 'a, { @@ -183,7 +199,7 @@ impl<'a, const C: usize> Executor<'a, C> { /// Polls the first task scheduled for execution by the executor. fn poll_runnable(&self, ctx: &Context<'_>) -> Poll { - self.state().waker.register(ctx.waker()); + LazyLock::get(&self.state).waker.register(ctx.waker()); if let Some(runnable) = self.try_runnable() { Poll::Ready(runnable) @@ -209,7 +225,7 @@ impl<'a, const C: usize> Executor<'a, C> { target_has_atomic = "ptr" ))] { - runnable = self.state().queue.pop().ok(); + runnable = LazyLock::get(&self.state).queue.pop(); } #[cfg(not(all( @@ -229,12 +245,12 @@ impl<'a, const C: usize> Executor<'a, C> { /// # Safety /// /// Original implementation missing safety documentation - unsafe fn spawn_unchecked(&self, fut: F) -> Task + unsafe fn spawn_unchecked(&'static self, fut: F) -> Task where F: Future, { let schedule = { - let state = self.state().clone(); + let state = &self.state; move |runnable| { #[cfg(all( @@ -288,23 +304,18 @@ impl<'a, const C: usize> Executor<'a, C> { run_forever.or(fut).await } - - /// Returns a reference to the inner state. - fn state(&self) -> &Arc> { - &self.state - } } -impl<'a, const C: usize> Default for Executor<'a, C> { +impl Default for Executor { fn default() -> Self { Self::new() } } // SAFETY: Original implementation missing safety documentation -unsafe impl<'a, const C: usize> Send for Executor<'a, C> {} +unsafe impl Send for Executor {} // SAFETY: Original implementation missing safety documentation -unsafe impl<'a, const C: usize> Sync for Executor<'a, C> {} +unsafe impl Sync for Executor {} struct State { #[cfg(all( diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 62f9f75e1fa23..2f49f83332ce8 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -11,7 +11,7 @@ crate::cfg::bevy_executor! { } } -static EXECUTOR: Executor<'static> = const { Executor::new() }; +static EXECUTOR: Executor = const { Executor::new() }; /// Used to create a [`TaskPool`]. #[derive(Debug, Default, Clone)] @@ -60,6 +60,10 @@ impl TaskPoolBuilder { pub fn build(self) -> TaskPool { TaskPool::new_internal() } + + pub(crate) fn build_static(self, _executor: &'static Executor) -> TaskPool { + TaskPool::new_internal() + } } /// A thread pool for executing tasks. Tasks are futures that are being automatically driven by @@ -123,9 +127,6 @@ impl TaskPool { // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor_ref: &'env Executor<'env> = unsafe { mem::transmute(&EXECUTOR) }; - // Kept around to ensure that, in the case of an unwinding panic, all scheduled Tasks are cancelled. let tasks: RefCell>> = RefCell::new(Vec::new()); // SAFETY: As above, all futures must complete in this function so we can change the lifetime @@ -136,7 +137,7 @@ impl TaskPool { let pending_tasks: &'env Cell = unsafe { mem::transmute(&pending_tasks) }; let mut scope = Scope { - executor_ref, + executor_ref: &EXECUTOR, tasks_ref, pending_tasks, scope: PhantomData, @@ -223,7 +224,7 @@ impl TaskPool { /// /// For more information, see [`TaskPool::scope`]. pub struct Scope<'scope, 'env: 'scope, T> { - executor_ref: &'scope Executor<'scope>, + executor_ref: &'static Executor, // The number of pending tasks spawned on the scope pending_tasks: &'scope Cell, // Vector to gather results of all futures spawned during scope run @@ -278,16 +279,11 @@ impl<'scope, 'env, T: Send + 'env> Scope<'scope, 'env, T> { }; let mut tasks = self.tasks_ref.borrow_mut(); - crate::cfg::bevy_executor! { - if { - #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] - // SAFETY: The surrounding scope will not terminate until all local tasks are done - // ensuring that the borrowed variables do not outlive the detached task. - tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); - } else { - tasks.push(self.executor_ref.spawn_local(f)); - } - } + + #[expect(unsafe_code, reason = "Executor::spawn_local_scoped is unsafe")] + // SAFETY: The surrounding scope will not terminate until all local tasks are done + // ensuring that the borrowed variables do not outlive the detached task. + tasks.push(unsafe { self.executor_ref.spawn_local_scoped(f) }); } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 270089b59f34f..b835934407abb 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -113,7 +113,11 @@ impl TaskPoolBuilder { /// Creates a new [`TaskPool`] based on the current options. pub fn build(self) -> TaskPool { - TaskPool::new_internal(self) + TaskPool::new_internal(self, Box::leak(Box::new(Executor::new()))) + } + + pub(crate) fn build_static(self, executor: &'static Executor) -> TaskPool { + TaskPool::new_internal(self, executor) } } @@ -130,7 +134,7 @@ impl TaskPoolBuilder { #[derive(Debug)] pub struct TaskPool { /// The executor for the pool. - executor: Arc>, + executor: &'static Executor, // The inner state of the pool. threads: Vec>, @@ -140,27 +144,24 @@ pub struct TaskPool { impl TaskPool { /// Creates a [`ThreadSpawner`] for this current thread of execution. /// Can be used to spawn new tasks to execute exclusively on this thread. - pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { + pub fn current_thread_spawner(&self) -> ThreadSpawner { self.executor.current_thread_spawner() } /// Create a `TaskPool` with the default configuration. - pub fn new() -> Self { + pub(crate) fn new() -> Self { TaskPoolBuilder::new().build() } - fn new_internal(builder: TaskPoolBuilder) -> Self { + fn new_internal(builder: TaskPoolBuilder, executor: &'static Executor) -> Self { let (shutdown_tx, shutdown_rx) = async_channel::unbounded::<()>(); - let executor = Arc::new(Executor::new()); - let num_threads = builder .num_threads .unwrap_or_else(crate::available_parallelism); let threads = (0..num_threads) .map(|i| { - let ex = Arc::clone(&executor); let shutdown_rx = shutdown_rx.clone(); let thread_name = if let Some(thread_name) = builder.thread_name.as_deref() { @@ -179,7 +180,7 @@ impl TaskPool { thread_builder .spawn(move || { - crate::bevy_executor::install_runtime_into_current_thread(&ex); + crate::bevy_executor::install_runtime_into_current_thread(executor); if let Some(on_thread_spawn) = on_thread_spawn { on_thread_spawn(); @@ -188,7 +189,7 @@ impl TaskPool { let _destructor = CallOnDrop(on_thread_destroy); loop { let res = - std::panic::catch_unwind(|| block_on(ex.run(shutdown_rx.recv()))); + std::panic::catch_unwind(|| block_on(executor.run(shutdown_rx.recv()))); if let Ok(value) = res { // Use unwrap_err because we expect a Closed error value.unwrap_err(); @@ -309,7 +310,7 @@ impl TaskPool { /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, - external_spawner: Option>, + external_spawner: Option, f: F, ) -> Vec where @@ -329,8 +330,8 @@ impl TaskPool { #[expect(unsafe_code, reason = "Required to transmute lifetimes.")] fn scope_with_executor_inner<'env, F, T>( &self, - external_spawner: ThreadSpawner<'env>, - scope_spawner: ThreadSpawner<'env>, + external_spawner: ThreadSpawner, + scope_spawner: ThreadSpawner, f: F, ) -> Vec where @@ -344,9 +345,6 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let executor: &Executor = &self.executor; - // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let executor: &'env Executor = unsafe { mem::transmute(executor) }; let spawned: ConcurrentQueue>>> = ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function @@ -355,7 +353,7 @@ impl TaskPool { unsafe { mem::transmute(&spawned) }; let scope = Scope { - executor, + executor: self.executor, external_spawner, scope_spawner, spawned, @@ -424,12 +422,6 @@ impl TaskPool { } } -impl Default for TaskPool { - fn default() -> Self { - Self::new() - } -} - impl Drop for TaskPool { fn drop(&mut self) { self.shutdown_tx.close(); @@ -449,9 +441,9 @@ impl Drop for TaskPool { /// For more information, see [`TaskPool::scope`]. #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { - executor: &'scope Executor<'scope>, - external_spawner: ThreadSpawner<'scope>, - scope_spawner: ThreadSpawner<'scope>, + executor: &'static Executor, + external_spawner: ThreadSpawner, + scope_spawner: ThreadSpawner, spawned: &'scope ConcurrentQueue>>>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -459,6 +451,10 @@ pub struct Scope<'scope, 'env: 'scope, T> { } impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { + #[expect( + unsafe_code, + reason = "Executor::spawn otherwise requires 'static Futures" + )] /// Spawns a scoped future onto the thread pool. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of /// [`TaskPool::scope`]'s return value. @@ -468,10 +464,13 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn + 'scope + Send>(&self, f: Fut) { - let task = self - .executor - .spawn(AssertUnwindSafe(f).catch_unwind()) - .fallible(); + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + let task = unsafe { + self.executor + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .fallible() + }; let result = self.spawned.push(task); debug_assert!(result.is_ok()); } @@ -577,6 +576,7 @@ mod tests { #[test] fn test_thread_callbacks() { let counter = Arc::new(AtomicI32::new(0)); + static EX: Executor = Executor::new(); let start_counter = counter.clone(); { let barrier = Arc::new(Barrier::new(11)); @@ -588,7 +588,7 @@ mod tests { start_counter.fetch_add(1, Ordering::Relaxed); barrier.clone().wait(); }) - .build(); + .build_static(&EX); last_barrier.wait(); assert_eq!(10, counter.load(Ordering::Relaxed)); } @@ -600,7 +600,7 @@ mod tests { .on_thread_destroy(move || { end_counter.fetch_sub(1, Ordering::Relaxed); }) - .build(); + .build_static(&EX); assert_eq!(10, counter.load(Ordering::Relaxed)); } assert_eq!(-10, counter.load(Ordering::Relaxed)); @@ -618,7 +618,7 @@ mod tests { .on_thread_destroy(move || { end_counter.fetch_sub(1, Ordering::Relaxed); }) - .build(); + .build_static(&EX); last_barrier.wait(); assert_eq!(-5, counter.load(Ordering::Relaxed)); } diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index e96cbdc6b80b4..c62967280874e 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,9 +1,18 @@ -use super::TaskPool; +use super::{TaskPool, TaskPoolBuilder}; use bevy_platform::sync::OnceLock; use core::ops::Deref; +crate::cfg::bevy_executor! { + if { + use crate::bevy_executor::Executor; + } else { + use crate::edge_executor::Executor; + } +} + macro_rules! taskpool { - ($(#[$attr:meta])* ($static:ident, $type:ident)) => { + ($(#[$attr:meta])* ($static:ident, $executor:ident, $type:ident)) => { + static $executor: Executor = Executor::new(); static $static: OnceLock<$type> = OnceLock::new(); $(#[$attr])* @@ -12,8 +21,8 @@ macro_rules! taskpool { impl $type { #[doc = concat!(" Gets the global [`", stringify!($type), "`] instance, or initializes it with `f`.")] - pub fn get_or_init(f: impl FnOnce() -> TaskPool) -> &'static Self { - $static.get_or_init(|| Self(f())) + pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static Self { + $static.get_or_init(|| Self(f().build_static(&$executor))) } #[doc = concat!(" Attempts to get the global [`", stringify!($type), "`] instance, \ @@ -56,7 +65,7 @@ taskpool! { /// See [`TaskPool`] documentation for details on Bevy tasks. /// [`AsyncComputeTaskPool`] should be preferred if the work does not have to be /// completed before the next frame. - (COMPUTE_TASK_POOL, ComputeTaskPool) + (COMPUTE_TASK_POOL, COMPUTE_EXECUTOR, ComputeTaskPool) } taskpool! { @@ -64,7 +73,7 @@ taskpool! { /// /// See [`TaskPool`] documentation for details on Bevy tasks. /// Use [`ComputeTaskPool`] if the work must be complete before advancing to the next frame. - (ASYNC_COMPUTE_TASK_POOL, AsyncComputeTaskPool) + (ASYNC_COMPUTE_TASK_POOL, ASYNC_COMPUTE_EXECUTOR, AsyncComputeTaskPool) } taskpool! { @@ -72,7 +81,7 @@ taskpool! { /// "woken" state) /// /// See [`TaskPool`] documentation for details on Bevy tasks. - (IO_TASK_POOL, IoTaskPool) + (IO_TASK_POOL, IO_EXECUTOR, IoTaskPool) } crate::cfg::web! { diff --git a/crates/bevy_transform/src/systems.rs b/crates/bevy_transform/src/systems.rs index 62038b37ed90f..2e9a113e66d8a 100644 --- a/crates/bevy_transform/src/systems.rs +++ b/crates/bevy_transform/src/systems.rs @@ -252,7 +252,7 @@ mod parallel { // TODO: this implementation could be used in no_std if there are equivalents of these. use alloc::{sync::Arc, vec::Vec}; use bevy_ecs::{entity::UniqueEntityIter, prelude::*, system::lifetimeless::Read}; - use bevy_tasks::{ComputeTaskPool, TaskPool}; + use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use bevy_utils::Parallel; use core::sync::atomic::{AtomicI32, Ordering}; use std::sync::{ @@ -320,7 +320,7 @@ mod parallel { } // Spawn workers on the task pool to recursively propagate the hierarchy in parallel. - let task_pool = ComputeTaskPool::get_or_init(TaskPool::default); + let task_pool = ComputeTaskPool::get_or_init(TaskPoolBuilder::default); task_pool.scope(|s| { (1..task_pool.thread_num()) // First worker is run locally instead of the task pool. .for_each(|_| s.spawn(async { propagation_worker(&queue, &nodes) })); @@ -559,13 +559,13 @@ mod test { use bevy_app::prelude::*; use bevy_ecs::{prelude::*, world::CommandQueue}; use bevy_math::{vec3, Vec3}; - use bevy_tasks::{ComputeTaskPool, TaskPool}; + use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use crate::systems::*; #[test] fn correct_parent_removed() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let offset_global_transform = |offset| GlobalTransform::from(Transform::from_xyz(offset, offset, offset)); @@ -626,7 +626,7 @@ mod test { #[test] fn did_propagate() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let mut schedule = Schedule::default(); @@ -702,7 +702,7 @@ mod test { #[test] fn correct_children() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let mut schedule = Schedule::default(); @@ -783,7 +783,7 @@ mod test { #[test] fn correct_transforms_when_no_children() { let mut app = App::new(); - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); app.add_systems( Update, @@ -834,7 +834,7 @@ mod test { #[test] #[should_panic] fn panic_when_hierarchy_cycle() { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); // We cannot directly edit ChildOf and Children, so we use a temp world to break the // hierarchy's invariants. let mut temp = World::new(); From 64e2e46888776b4d2a341fb70f45c82ebadb675c Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 19:34:16 -0700 Subject: [PATCH 58/68] Cleanup --- crates/bevy_tasks/src/bevy_executor.rs | 4 ---- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- crates/bevy_tasks/src/task_pool.rs | 2 +- crates/bevy_tasks/src/usages.rs | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index d167a39fa9e1a..f80b8c6afe7dd 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -913,10 +913,6 @@ mod test { use super::THREAD_LOCAL_STATE; use alloc::{string::String, boxed::Box}; use futures_lite::{future, pin}; - use std::panic::catch_unwind; - use core::sync::atomic::{AtomicUsize, Ordering}; - use bevy_platform::sync::Mutex; - use core::task::{Poll, Waker}; use async_task::Task; use core::time::Duration; diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 2f49f83332ce8..b529d6f2efe31 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -23,7 +23,7 @@ pub struct TaskPoolBuilder {} /// `wasm_bindgen_futures::spawn_local` for spawning which just runs tasks on the main thread /// and so the [`ThreadSpawner`] does nothing. #[derive(Clone)] -pub struct ThreadSpawner<'a>(PhantomData<&'a ()>); +pub struct ThreadSpawner; impl TaskPoolBuilder { /// Creates a new `TaskPoolBuilder` instance diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index b835934407abb..424eaa1f3e621 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -149,7 +149,7 @@ impl TaskPool { } /// Create a `TaskPool` with the default configuration. - pub(crate) fn new() -> Self { + pub fn new() -> Self { TaskPoolBuilder::new().build() } diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index c62967280874e..1fa09bd139d87 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -3,7 +3,7 @@ use bevy_platform::sync::OnceLock; use core::ops::Deref; crate::cfg::bevy_executor! { - if { + if { use crate::bevy_executor::Executor; } else { use crate::edge_executor::Executor; From ba4643331f412bfb2804dde77bd81e4001626613 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 19:53:10 -0700 Subject: [PATCH 59/68] Try to fix CI again --- crates/bevy_tasks/src/edge_executor.rs | 48 ++----------------- .../src/single_threaded_task_pool.rs | 6 +-- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/crates/bevy_tasks/src/edge_executor.rs b/crates/bevy_tasks/src/edge_executor.rs index 612fe65b580dc..7215b0b7bb8e7 100644 --- a/crates/bevy_tasks/src/edge_executor.rs +++ b/crates/bevy_tasks/src/edge_executor.rs @@ -199,7 +199,7 @@ impl Executor { /// Polls the first task scheduled for execution by the executor. fn poll_runnable(&self, ctx: &Context<'_>) -> Poll { - LazyLock::get(&self.state).waker.register(ctx.waker()); + self.state.waker.register(ctx.waker()); if let Some(runnable) = self.try_runnable() { Poll::Ready(runnable) @@ -225,7 +225,7 @@ impl Executor { target_has_atomic = "ptr" ))] { - runnable = LazyLock::get(&self.state).queue.pop(); + runnable = self.state.queue.pop().ok(); } #[cfg(not(all( @@ -236,7 +236,7 @@ impl Executor { target_has_atomic = "ptr" )))] { - runnable = self.state().queue.dequeue(); + runnable = self.state.queue.dequeue(); } runnable @@ -361,46 +361,6 @@ impl State { } } -#[cfg(test)] -mod different_executor_tests { - use core::cell::Cell; - - use bevy_tasks::{block_on, futures_lite::{pending, poll_once}}; - use futures_lite::pin; - - use super::LocalExecutor; - - #[test] - fn shared_queue_slot() { - block_on(async { - let was_polled = Cell::new(false); - let future = async { - was_polled.set(true); - pending::<()>().await; - }; - - let ex1: LocalExecutor = Default::default(); - let ex2: LocalExecutor = Default::default(); - - // Start the futures for running forever. - let (run1, run2) = (ex1.run(pending::<()>()), ex2.run(pending::<()>())); - pin!(run1); - pin!(run2); - assert!(poll_once(run1.as_mut()).await.is_none()); - assert!(poll_once(run2.as_mut()).await.is_none()); - - // Spawn the future on executor one and then poll executor two. - ex1.spawn(future).detach(); - assert!(poll_once(run2).await.is_none()); - assert!(!was_polled.get()); - - // Poll the first one. - assert!(poll_once(run1).await.is_none()); - assert!(was_polled.get()); - }); - } -} - #[cfg(test)] mod drop_tests { use alloc::string::String; @@ -417,7 +377,7 @@ mod drop_tests { #[test] fn leaked_executor_leaks_everything() { static DROP: AtomicUsize = AtomicUsize::new(0); - static WAKER: LazyLock>> = LazyLock::new(Default::default); + static WAKER: Mutex> = Mutex::new(None); let ex: Executor = Default::default(); diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index b529d6f2efe31..877757d253ad9 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -73,8 +73,8 @@ pub struct TaskPool {} impl TaskPool { /// Just create a new `ThreadExecutor` for wasm - pub fn current_thread_spawner(&self) -> ThreadSpawner<'static> { - ThreadSpawner(PhantomData) + pub fn current_thread_spawner(&self) -> ThreadSpawner { + ThreadSpawner } /// Create a `TaskPool` with the default configuration. @@ -150,7 +150,7 @@ impl TaskPool { f(scope_ref); // Wait until the scope is complete - block_on(executor_ref.run(async { + block_on(EX.run(async { while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } From 02fc4a516df4bdd06a1072b30b8497c9a92cb248 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 19:59:36 -0700 Subject: [PATCH 60/68] Clippy and docs --- crates/bevy_ecs/src/query/state.rs | 2 +- crates/bevy_tasks/src/bevy_executor.rs | 6 +++++- crates/bevy_tasks/src/task_pool.rs | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index 09821a718c668..9e19417d6723d 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -1345,7 +1345,7 @@ impl QueryState { /// #[derive(Component, PartialEq, Debug)] /// struct A(usize); /// - /// # bevy_tasks::ComputeTaskPool::get_or_init(|| bevy_tasks::TaskPool::new()); + /// # bevy_tasks::ComputeTaskPool::get_or_init(|| bevy_tasks::TaskPoolBuilder::default()); /// /// let mut world = World::new(); /// diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index f80b8c6afe7dd..d7bd8f7684aba 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -184,12 +184,16 @@ impl Executor { } } - /// Spawns a task onto the executor. + /// Spawns a 'static and Send task onto the executor. pub fn spawn(&'static self, future: impl Future + Send + 'static) -> Task { // SAFETY: Both `T` and `future` are 'static. unsafe { self.spawn_scoped(future) } } + /// Spawns a non-'static Send task onto the executor. + /// + /// # Safety + /// The caller must ensure that the returned Task does not outlive 'a. pub unsafe fn spawn_scoped<'a, T: Send + 'a>(&'static self, future: impl Future + Send + 'a) -> Task { // Create the task and register it in the set of active tasks. // diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 424eaa1f3e621..9fcb733ef697a 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -422,6 +422,12 @@ impl TaskPool { } } +impl Default for TaskPool { + fn default() -> Self { + Self::new() + } +} + impl Drop for TaskPool { fn drop(&mut self) { self.shutdown_tx.close(); From 66c6f5f32d9a2096a3b1032753c72985f5956f9c Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 20:03:32 -0700 Subject: [PATCH 61/68] is_multiple_of lint --- crates/bevy_tasks/src/bevy_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index d7bd8f7684aba..9c68c79ec8901 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -749,7 +749,7 @@ impl Runner<'_> { // Bump the tick counter. self.ticks = self.ticks.wrapping_add(1); - if self.ticks % 64 == 0 { + if self.ticks.is_multiple_of(64) { // Steal tasks from the global queue to ensure fair task scheduling. steal(&self.state.queue, &self.local_state.stealable_queue); } From a7362406a6f5d9e4ed1b548a02afe75b1ed830e1 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 19 Aug 2025 20:06:41 -0700 Subject: [PATCH 62/68] Fix benchmarks --- benches/benches/bevy_ecs/iteration/heavy_compute.rs | 4 ++-- benches/benches/bevy_ecs/iteration/par_iter_simple.rs | 4 ++-- .../bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs | 4 ++-- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benches/benches/bevy_ecs/iteration/heavy_compute.rs b/benches/benches/bevy_ecs/iteration/heavy_compute.rs index e057b20a431be..21483463cdd96 100644 --- a/benches/benches/bevy_ecs/iteration/heavy_compute.rs +++ b/benches/benches/bevy_ecs/iteration/heavy_compute.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPool}; +use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use criterion::Criterion; use glam::*; @@ -20,7 +20,7 @@ pub fn heavy_compute(c: &mut Criterion) { group.warm_up_time(core::time::Duration::from_millis(500)); group.measurement_time(core::time::Duration::from_secs(4)); group.bench_function("base", |b| { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); diff --git a/benches/benches/bevy_ecs/iteration/par_iter_simple.rs b/benches/benches/bevy_ecs/iteration/par_iter_simple.rs index 92259cb98fecf..089a9854e76d4 100644 --- a/benches/benches/bevy_ecs/iteration/par_iter_simple.rs +++ b/benches/benches/bevy_ecs/iteration/par_iter_simple.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPool}; +use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use glam::*; #[derive(Component, Copy, Clone)] @@ -26,7 +26,7 @@ fn insert_if_bit_enabled(entity: &mut EntityWorldMut, i: u16) { impl<'w> Benchmark<'w> { pub fn new(fragment: u16) -> Self { - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); diff --git a/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs b/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs index 9dbcba87852f7..8b90783dc5c51 100644 --- a/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs +++ b/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPool}; +use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; use rand::{prelude::SliceRandom, SeedableRng}; use rand_chacha::ChaCha8Rng; @@ -18,7 +18,7 @@ pub struct Benchmark<'w>(World, QueryState<(&'w mut TableData, &'w SparseData)>) impl<'w> Benchmark<'w> { pub fn new() -> Self { let mut world = World::new(); - ComputeTaskPool::get_or_init(TaskPool::default); + ComputeTaskPool::get_or_init(TaskPoolBuilder::default); let mut v = vec![]; for _ in 0..100000 { diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 877757d253ad9..9cf26de935a4f 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -150,7 +150,7 @@ impl TaskPool { f(scope_ref); // Wait until the scope is complete - block_on(EX.run(async { + block_on(EXECUTOR.run(async { while pending_tasks.get() != 0 { futures_lite::future::yield_now().await; } From 30ca9dd8311cd67161af523660ec65c382dbc870 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 02:41:16 -0700 Subject: [PATCH 63/68] Limit the number of actively running tasks of each priority --- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/bevy_executor.rs | 214 ++++++++++++++++++++----- crates/bevy_tasks/src/lib.rs | 25 +++ crates/bevy_tasks/src/task.rs | 31 ++-- crates/bevy_tasks/src/task_pool.rs | 19 ++- 5 files changed, 231 insertions(+), 59 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 0eb81732728c0..2af9bebd89a10 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -57,6 +57,7 @@ async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } +bitflags = "2.9" [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 9c68c79ec8901..3608f0a8710b1 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -9,15 +9,18 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; -use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use core::task::{Context, Poll, Waker}; use core::cell::UnsafeCell; use core::mem; +use std::boxed::Box; use std::thread::{AccessError, ThreadId}; +use crate::{Metadata, TaskPriority}; use alloc::collections::VecDeque; use alloc::fmt; -use async_task::{Builder, Runnable, Task}; +use core::num::{NonZero, NonZeroUsize}; +use async_task::Builder; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Mutex, PoisonError, RwLock, TryLockError}; use concurrent_queue::ConcurrentQueue; @@ -26,6 +29,9 @@ use slab::Slab; use thread_local::ThreadLocal; use crossbeam_utils::CachePadded; +type Runnable = async_task::Runnable; +type Task = async_task::Task; + // ThreadLocalState *must* stay `Sync` due to a currently existing soundness hole. // See: https://github.com/Amanieu/thread_local-rs/issues/75 static THREAD_LOCAL_STATE: ThreadLocal = ThreadLocal::new(); @@ -107,7 +113,7 @@ impl ThreadSpawner { pub fn spawn( &self, future: impl Future + Send + 'static, - ) -> Task { + ) -> crate::Task { // SAFETY: T and `future` are both 'static, so the Task is guaranteed to not outlive it. unsafe { self.spawn_scoped(future) } } @@ -119,7 +125,14 @@ impl ThreadSpawner { pub unsafe fn spawn_scoped<'a, T: Send + 'a>( &self, future: impl Future + Send + 'a, - ) -> Task { + ) -> crate::Task { + let builder = Builder::new() + .propagate_panic(true) + .metadata(Metadata { + priority: TaskPriority::Compute, + is_send: false, + }); + // Create the task and register it in the set of active tasks. // // SAFETY: @@ -132,9 +145,7 @@ impl ThreadSpawner { // Therefore we do not need to worry about what is done with the // `Waker`. let (runnable, task) = unsafe { - Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule()) + builder.spawn_unchecked(|_| future, self.schedule()) }; // Instead of directly scheduling this task, it's put into the onto the @@ -142,7 +153,7 @@ impl ThreadSpawner { // either be run immediately or flushed into the thread's local queue. let result = self.target_queue.push(runnable); debug_assert!(result.is_ok()); - task + crate::Task::new(task) } /// Returns a function that schedules a runnable task when it gets woken up. @@ -185,16 +196,18 @@ impl Executor { } /// Spawns a 'static and Send task onto the executor. - pub fn spawn(&'static self, future: impl Future + Send + 'static) -> Task { + pub fn spawn(&'static self, future: impl Future + Send + 'static, metadata: Metadata) -> Task { // SAFETY: Both `T` and `future` are 'static. - unsafe { self.spawn_scoped(future) } + unsafe { self.spawn_scoped(future, metadata) } } /// Spawns a non-'static Send task onto the executor. /// /// # Safety /// The caller must ensure that the returned Task does not outlive 'a. - pub unsafe fn spawn_scoped<'a, T: Send + 'a>(&'static self, future: impl Future + Send + 'a) -> Task { + pub unsafe fn spawn_scoped<'a, T: Send + 'a>(&'static self, future: impl Future + Send + 'a, mut metadata: Metadata) -> Task { + metadata.is_send = true; + let builder = Builder::new().propagate_panic(true).metadata(metadata); // Create the task and register it in the set of active tasks. // // SAFETY: @@ -210,9 +223,7 @@ impl Executor { // Therefore we do not need to worry about what is done with the // `Waker`. let (runnable, task) = unsafe { - Builder::new() - .propagate_panic(true) - .spawn_unchecked(|()| future, self.schedule()) + builder.spawn_unchecked(|_| future, self.schedule()) }; runnable.schedule(); @@ -220,9 +231,9 @@ impl Executor { } /// Spawns a non-Send task onto the executor. - pub fn spawn_local(&'static self, future: impl Future + 'static) -> Task { + pub fn spawn_local(&'static self, future: impl Future + 'static, metadata: Metadata) -> Task { // SAFETY: future is 'static - unsafe { self.spawn_local_scoped(future) } + unsafe { self.spawn_local_scoped(future, metadata) } } /// Spawns a non-'static and non-Send task onto the executor. @@ -232,7 +243,9 @@ impl Executor { pub unsafe fn spawn_local_scoped<'a, T: 'a>( &'static self, future: impl Future + 'a, + mut metadata: Metadata, ) -> Task { + metadata.is_send = false; // Remove the task from the set of active tasks when the future finishes. // // SAFETY: There are no instances where the value is accessed mutably @@ -241,7 +254,7 @@ impl Executor { try_with_local_queue(|tls| { let entry = tls.local_active.vacant_entry(); let index = entry.key(); - let builder = Builder::new().propagate_panic(true); + let builder = Builder::new().propagate_panic(true).metadata(metadata); // SAFETY: There are no instances where the value is accessed mutably // from multiple locations simultaneously. This AsyncCallOnDrop will be @@ -273,7 +286,7 @@ impl Executor { // all of them are bound vy use of thread-local storage. // - `self.schedule_local()` is `'static`, as checked below. let (runnable, task) = builder - .spawn_unchecked(|()| future, self.schedule_local()); + .spawn_unchecked(|_| future, self.schedule_local()); entry.insert(runnable.waker()); mem::forget(_panic_guard); @@ -308,15 +321,26 @@ impl Executor { /// Runs the executor until the given future completes. pub fn run<'b, T>(&'static self, future: impl Future + 'b) -> impl Future + 'b { + const MAX_CONSECUTIVE_FAILURES: usize = 5; let mut runner = Runner::new(&self.state); // A future that runs tasks forever. let run_forever = async move { let mut rng = fastrand::Rng::new(); loop { + let mut failed = 0; for _ in 0..200 { let runnable = runner.runnable(&mut rng).await; - runnable.run(); + + if !Self::execute(&self.state, runnable) { + failed += 1; + } else { + failed = 0; + } + + if failed >= MAX_CONSECUTIVE_FAILURES { + break; + } } future::yield_now().await; } @@ -326,32 +350,31 @@ impl Executor { future.or(run_forever) } + fn execute(state: &'static State, runnable: Runnable) -> bool { + let metadata = runnable.metadata(); + // SAFETY: This can never be outo bounds. + let semaphore = unsafe { state.priority_limits.get_unchecked(metadata.priority.to_index()) }; + match semaphore.acquire() { + Permit::Unrestricted | Permit::Held(_) => { + runnable.run(); + true + }, + Permit::Blocked => if metadata.is_send { + Self::queue_send(state, runnable); + false + } else { + Self::queue_local(state, runnable); + false + }, + } + } + /// Returns a function that schedules a runnable task when it gets woken up. fn schedule(&'static self) -> impl Fn(Runnable) + Send + Sync + 'static { let state = &self.state; move |runnable| { - // Attempt to push onto the local queue first in dedicated executor threads, - // because we know that this thread is awake and always processing new tasks. - let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { - if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), state) { - match local_state.stealable_queue.push(runnable) { - Ok(()) => { - state.notify_specific_thread(std::thread::current().id(), true); - return; - } - Err(r) => r.into_inner(), - } - } else { - runnable - } - } else { - runnable - }; - // Otherwise push onto the global queue instead. - let result = state.queue.push(runnable); - debug_assert!(result.is_ok()); - state.notify(); + Self::queue_send(state, runnable); } } @@ -359,12 +382,60 @@ impl Executor { fn schedule_local(&'static self) -> impl Fn(Runnable) + 'static { let state = &self.state; move |runnable| { + Self::queue_local(state, runnable); + } + } + + fn queue_send(state: &'static State, runnable: Runnable) { + debug_assert!(runnable.metadata().is_send); + if runnable.metadata().priority == TaskPriority::RunNow { // SAFETY: This value is in thread local storage and thus can only be accessed // from one thread. There are no instances where the value is accessed mutably // from multiple locations simultaneously. - if unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) }.is_ok() { + if unsafe { try_with_local_queue(|tls| tls.local_queue.push_front(runnable)) }.is_ok() { state.notify_specific_thread(std::thread::current().id(), false); } + return; + } + + // Attempt to push onto the local queue first in dedicated executor threads, + // because we know that this thread is awake and always processing new tasks. + let runnable = if let Some(local_state) = THREAD_LOCAL_STATE.get() { + if core::ptr::eq(local_state.executor.load(Ordering::Relaxed), state) { + match local_state.stealable_queue.push(runnable) { + Ok(()) => { + state.notify_specific_thread(std::thread::current().id(), true); + return; + } + Err(r) => r.into_inner(), + } + } else { + runnable + } + } else { + runnable + }; + // Otherwise push onto the global queue instead. + let result = state.queue.push(runnable); + debug_assert!(result.is_ok()); + state.notify(); + } + + fn queue_local(state: &'static State, runnable: Runnable) { + debug_assert!(!runnable.metadata().is_send); + let result = if runnable.metadata().priority == TaskPriority::RunNow { + // SAFETY: This value is in thread local storage and thus can only be accessed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { try_with_local_queue(|tls| tls.local_queue.push_front(runnable)) } + } else { + // SAFETY: This value is in thread local storage and thus can only be accessed + // from one thread. There are no instances where the value is accessed mutably + // from multiple locations simultaneously. + unsafe { try_with_local_queue(|tls| tls.local_queue.push_back(runnable)) } + }; + if result.is_ok() { + state.notify_specific_thread(std::thread::current().id(), false); } } } @@ -382,6 +453,9 @@ struct State { /// A list of sleeping tickers. sleepers: Mutex, + + // Semaphores for each priority level. + priority_limits: [CachePadded; TaskPriority::MAX] } impl State { @@ -396,6 +470,13 @@ impl State { wakers: Vec::new(), free_ids: Vec::new(), }), + priority_limits: [ + CachePadded::new(AtomicSemaphore::new(None)), + CachePadded::new(AtomicSemaphore::new(None)), + CachePadded::new(AtomicSemaphore::new(None)), + CachePadded::new(AtomicSemaphore::new(None)), + CachePadded::new(AtomicSemaphore::new(None)) + ], } } @@ -811,6 +892,57 @@ fn flush_to_local(src: &ConcurrentQueue, dst: &mut LocalQueue) { } } +struct AtomicSemaphore { + available: AtomicUsize, + limit: Option, +} + +impl AtomicSemaphore { + pub const fn new(limit: Option) -> Self { + match limit { + Some(limit) => Self { + available: AtomicUsize::new(limit.get()), + limit: Some(limit), + }, + None => Self { + available: AtomicUsize::new(0), + limit: None, + } + } + } + + pub fn acquire<'a>(&'a self) -> Permit<'a> { + if self.limit.is_none() { + return Permit::Unrestricted; + } + let mut current = self.available.load(Ordering::Acquire); + if current == 0 { + return Permit::Blocked; + } + loop { + match self.available.compare_exchange_weak(current, current - 1, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => return Permit::Held(self), + Err(0) => return Permit::Blocked, + Err(actual) => current = actual, + } + } + } +} + +enum Permit<'a> { + Unrestricted, + Held(&'a AtomicSemaphore), + Blocked, +} + +impl<'a> Drop for Permit<'a> { + fn drop(&mut self) { + if let Permit::Held(sempahore) = self { + sempahore.available.fetch_add(1, Ordering::AcqRel); + } + } +} + /// Debug implementation for `Executor`. fn debug_executor(executor: &Executor, name: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { debug_state(&executor.state, name, f) diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index d03665c94bc86..b66aa70247074 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -176,3 +176,28 @@ pub fn available_parallelism() -> usize { } }} } + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum TaskPriority { + BlockingIO, + BlockingCompute, + AsyncIO, + #[default] Compute, + RunNow, +} + +impl TaskPriority { + const MAX: usize = TaskPriority::RunNow as u8 as usize + 1; + + #[inline] + fn to_index(self) -> usize { + self as u8 as usize + } +} + +#[derive(Debug, Default)] +pub(crate) struct Metadata { + pub priority: TaskPriority, + pub is_send: bool, +} diff --git a/crates/bevy_tasks/src/task.rs b/crates/bevy_tasks/src/task.rs index 4e9826a82f709..c28e4170928e8 100644 --- a/crates/bevy_tasks/src/task.rs +++ b/crates/bevy_tasks/src/task.rs @@ -7,6 +7,19 @@ use core::{ use crate::cfg; +crate::cfg::switch! { + crate::cfg::web => { + type TaskInner = async_channel::Receiver>; + } + crate::cfg::bevy_executor => { + use crate::Metadata; + type TaskInner = async_task::Task; + } + _ => { + type TaskInner = async_task::Task; + } +} + /// Wraps `async_executor::Task`, a spawned future. /// /// Tasks are also futures themselves and yield the output of the spawned future. @@ -16,15 +29,7 @@ use crate::cfg; /// /// Tasks that panic get immediately canceled. Awaiting a canceled task also causes a panic. #[must_use = "Tasks are canceled when dropped, use `.detach()` to run them in the background."] -pub struct Task( - cfg::web! { - if { - async_channel::Receiver> - } else { - async_task::Task - } - }, -); +pub struct Task(TaskInner); // Custom constructors for web and non-web platforms cfg::web! { @@ -49,9 +54,15 @@ cfg::web! { } else { impl Task { /// Creates a new task from a given `async_executor::Task` - pub(crate) fn new(task: async_task::Task) -> Self { + #[inline] + pub(crate) fn new(task: TaskInner) -> Self { Self(task) } + + #[inline] + pub(crate) fn into_inner(self) -> TaskInner { + self.0 + } } } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 9fcb733ef697a..2f9de97801441 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,7 +2,7 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; use std::thread::{self, JoinHandle}; -use crate::bevy_executor::Executor; +use crate::{bevy_executor::Executor, Metadata}; use async_task::FallibleTask; use bevy_platform::sync::Arc; use concurrent_queue::ConcurrentQueue; @@ -345,11 +345,10 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. // Any usages of the references passed into `Scope` must be accessed through // the transmuted reference for the rest of this function. - let spawned: ConcurrentQueue>>> = - ConcurrentQueue::unbounded(); + let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); // shadow the variable so that the owned value cannot be used for the rest of the function // SAFETY: As above, all futures must complete in this function so we can change the lifetime - let spawned: &'env ConcurrentQueue>>> = + let spawned: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) }; let scope = Scope { @@ -396,7 +395,7 @@ impl TaskPool { where T: Send + 'static, { - Task::new(self.executor.spawn(future)) + Task::new(self.executor.spawn(future, Metadata::default())) } /// Spawns a static future on the thread-local async executor for the @@ -414,7 +413,7 @@ impl TaskPool { where T: 'static, { - Task::new(self.executor.spawn_local(future)) + Task::new(self.executor.spawn_local(future, Metadata::default())) } pub(crate) fn try_tick_local() -> bool { @@ -442,6 +441,8 @@ impl Drop for TaskPool { } } +type ScopeTask = FallibleTask>, Metadata>; + /// A [`TaskPool`] scope for running one or more non-`'static` futures. /// /// For more information, see [`TaskPool::scope`]. @@ -450,7 +451,7 @@ pub struct Scope<'scope, 'env: 'scope, T> { executor: &'static Executor, external_spawner: ThreadSpawner, scope_spawner: ThreadSpawner, - spawned: &'scope ConcurrentQueue>>>, + spawned: &'scope ConcurrentQueue>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, env: PhantomData<&'env mut &'env ()>, @@ -474,7 +475,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { // Task does not outlive 'scope. let task = unsafe { self.executor - .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .spawn_scoped(AssertUnwindSafe(f).catch_unwind(), Metadata::default()) .fallible() }; let result = self.spawned.push(task); @@ -497,6 +498,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { let task = unsafe { self.scope_spawner .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() .fallible() }; let result = self.spawned.push(task); @@ -520,6 +522,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { let task = unsafe { self.external_spawner .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() .fallible() }; // ConcurrentQueue only errors when closed or full, but we never From 9150b699d286648e136a8bcb328fe774e90ed914 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 10:09:18 -0700 Subject: [PATCH 64/68] Start providing ways to specify priority --- crates/bevy_tasks/src/bevy_executor.rs | 3 +- crates/bevy_tasks/src/lib.rs | 29 +++++++++ crates/bevy_tasks/src/task_pool.rs | 36 ++++++----- crates/bevy_tasks/src/usages.rs | 86 +------------------------- 4 files changed, 51 insertions(+), 103 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 3608f0a8710b1..599490bfab616 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -13,13 +13,12 @@ use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use core::task::{Context, Poll, Waker}; use core::cell::UnsafeCell; use core::mem; -use std::boxed::Box; use std::thread::{AccessError, ThreadId}; use crate::{Metadata, TaskPriority}; use alloc::collections::VecDeque; use alloc::fmt; -use core::num::{NonZero, NonZeroUsize}; +use core::num::NonZeroUsize; use async_task::Builder; use bevy_platform::prelude::Vec; use bevy_platform::sync::{Mutex, PoisonError, RwLock, TryLockError}; diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index b66aa70247074..ba90714a2b9fd 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -66,6 +66,8 @@ pub trait ConditionalSendFuture: Future + ConditionalSend {} impl ConditionalSendFuture for T {} +use core::marker::PhantomData; + use alloc::boxed::Box; /// An owned and dynamically typed Future used when you can't statically type your result or need to add some indirection. @@ -201,3 +203,30 @@ pub(crate) struct Metadata { pub priority: TaskPriority, pub is_send: bool, } + +pub struct TaskBuilder { + priority: TaskPriority, + marker_: PhantomData<*const T> +} + +impl TaskBuilder { + pub fn with_priority(mut self, priority: TaskPriority) -> Self { + self.priority = priority; + self + } + + pub fn build_metadata(self) -> Metadata { + Metadata { + priority: self.priority, + is_send: false, + } + } +} + +pub struct ScopeTaskBuilder<'a: 'scope, 'scope: 'env, 'env, T> { + scope: &'a Scope<'scope, 'env, T>, +} + +impl<'a, 'scope, 'env> ScopeTaskBuilder<'a, 'scope, 'env, T> { + +} \ No newline at end of file diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 2f9de97801441..f0fec26b338b3 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; -use std::thread::{self, JoinHandle}; +use std::{sync::OnceLock, thread::{self, JoinHandle}}; use crate::{bevy_executor::Executor, Metadata}; use async_task::FallibleTask; @@ -12,6 +12,9 @@ use crate::{block_on, Task}; pub use crate::bevy_executor::ThreadSpawner; +static EXECUTOR: Executor = Executor::new(); +static TASK_POOL: OnceLock = OnceLock::new(); + struct CallOnDrop(Option>); impl Drop for CallOnDrop { @@ -148,9 +151,16 @@ impl TaskPool { self.executor.current_thread_spawner() } - /// Create a `TaskPool` with the default configuration. - pub fn new() -> Self { - TaskPoolBuilder::new().build() + pub fn try_get() -> Option<&'static TaskPool> { + TASK_POOL.get() + } + + pub fn get() -> &'static TaskPool { + Self::get_or_init(Default::default) + } + + pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static TaskPool { + TASK_POOL.get_or_init(|| f().build_static(&EXECUTOR)) } fn new_internal(builder: TaskPoolBuilder, executor: &'static Executor) -> Self { @@ -421,12 +431,6 @@ impl TaskPool { } } -impl Default for TaskPool { - fn default() -> Self { - Self::new() - } -} - impl Drop for TaskPool { fn drop(&mut self) { self.shutdown_tx.close(); @@ -553,7 +557,7 @@ mod tests { #[test] fn test_spawn() { - let pool = TaskPool::new(); + let pool = TaskPool::get(); let foo = Box::new(42); let foo = &*foo; @@ -636,7 +640,7 @@ mod tests { #[test] fn test_mixed_spawn_on_scope_and_spawn() { - let pool = TaskPool::new(); + let pool = TaskPool::get(); let foo = Box::new(42); let foo = &*foo; @@ -681,7 +685,7 @@ mod tests { #[test] fn test_thread_locality() { - let pool = Arc::new(TaskPool::new()); + let pool = TaskPool::get(); let count = Arc::new(AtomicI32::new(0)); let barrier = Arc::new(Barrier::new(101)); let thread_check_failed = Arc::new(AtomicBool::new(false)); @@ -718,7 +722,7 @@ mod tests { #[test] fn test_nested_spawn() { - let pool = TaskPool::new(); + let pool = TaskPool::get(); let foo = Box::new(42); let foo = &*foo; @@ -756,7 +760,7 @@ mod tests { #[test] fn test_nested_locality() { - let pool = Arc::new(TaskPool::new()); + let pool = TaskPool::get(); let count = Arc::new(AtomicI32::new(0)); let barrier = Arc::new(Barrier::new(101)); let thread_check_failed = Arc::new(AtomicBool::new(false)); @@ -795,7 +799,7 @@ mod tests { // This test will often freeze on other executors. #[test] fn test_nested_scopes() { - let pool = TaskPool::new(); + let pool = TaskPool::get(); let count = Arc::new(AtomicI32::new(0)); pool.scope(|scope| { diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 1fa09bd139d87..2150395b2eae0 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -1,88 +1,4 @@ -use super::{TaskPool, TaskPoolBuilder}; -use bevy_platform::sync::OnceLock; -use core::ops::Deref; - -crate::cfg::bevy_executor! { - if { - use crate::bevy_executor::Executor; - } else { - use crate::edge_executor::Executor; - } -} - -macro_rules! taskpool { - ($(#[$attr:meta])* ($static:ident, $executor:ident, $type:ident)) => { - static $executor: Executor = Executor::new(); - static $static: OnceLock<$type> = OnceLock::new(); - - $(#[$attr])* - #[derive(Debug)] - pub struct $type(TaskPool); - - impl $type { - #[doc = concat!(" Gets the global [`", stringify!($type), "`] instance, or initializes it with `f`.")] - pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static Self { - $static.get_or_init(|| Self(f().build_static(&$executor))) - } - - #[doc = concat!(" Attempts to get the global [`", stringify!($type), "`] instance, \ - or returns `None` if it is not initialized.")] - pub fn try_get() -> Option<&'static Self> { - $static.get() - } - - #[doc = concat!(" Gets the global [`", stringify!($type), "`] instance.")] - #[doc = ""] - #[doc = " # Panics"] - #[doc = " Panics if the global instance has not been initialized yet."] - pub fn get() -> &'static Self { - $static.get().expect( - concat!( - "The ", - stringify!($type), - " has not been initialized yet. Please call ", - stringify!($type), - "::get_or_init beforehand." - ) - ) - } - } - - impl Deref for $type { - type Target = TaskPool; - - fn deref(&self) -> &Self::Target { - &self.0 - } - } - }; -} - -taskpool! { - /// A newtype for a task pool for CPU-intensive work that must be completed to - /// deliver the next frame - /// - /// See [`TaskPool`] documentation for details on Bevy tasks. - /// [`AsyncComputeTaskPool`] should be preferred if the work does not have to be - /// completed before the next frame. - (COMPUTE_TASK_POOL, COMPUTE_EXECUTOR, ComputeTaskPool) -} - -taskpool! { - /// A newtype for a task pool for CPU-intensive work that may span across multiple frames - /// - /// See [`TaskPool`] documentation for details on Bevy tasks. - /// Use [`ComputeTaskPool`] if the work must be complete before advancing to the next frame. - (ASYNC_COMPUTE_TASK_POOL, ASYNC_COMPUTE_EXECUTOR, AsyncComputeTaskPool) -} - -taskpool! { - /// A newtype for a task pool for IO-intensive work (i.e. tasks that spend very little time in a - /// "woken" state) - /// - /// See [`TaskPool`] documentation for details on Bevy tasks. - (IO_TASK_POOL, IO_EXECUTOR, IoTaskPool) -} +use super::TaskPool; crate::cfg::web! { if {} else { From 0b4af841d09209f9e31a421369e40b5284bfa01e Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 19:29:51 -0700 Subject: [PATCH 65/68] Update usages --- Cargo.toml | 10 +- .../bevy_ecs/iteration/heavy_compute.rs | 4 +- .../bevy_ecs/iteration/par_iter_simple.rs | 4 +- .../par_iter_simple_foreach_hybrid.rs | 4 +- crates/bevy_app/src/task_pool_plugin.rs | 241 ++++++------------ crates/bevy_asset/src/processor/mod.rs | 49 ++-- crates/bevy_asset/src/server/loaders.rs | 6 +- crates/bevy_asset/src/server/mod.rs | 115 +++++---- .../system_information_diagnostics_plugin.rs | 57 +++-- crates/bevy_ecs/src/batching.rs | 4 +- crates/bevy_ecs/src/event/iterators.rs | 10 +- crates/bevy_ecs/src/event/mut_iterators.rs | 10 +- crates/bevy_ecs/src/lib.rs | 6 +- crates/bevy_ecs/src/query/par_iter.rs | 30 +-- crates/bevy_ecs/src/query/state.rs | 24 +- .../src/schedule/executor/multi_threaded.rs | 24 +- crates/bevy_ecs/src/schedule/mod.rs | 4 +- .../src/system/commands/parallel_scope.rs | 2 +- crates/bevy_gltf/src/loader/mod.rs | 31 ++- crates/bevy_pbr/src/meshlet/from_mesh.rs | 4 +- crates/bevy_remote/src/http.rs | 10 +- crates/bevy_render/src/lib.rs | 3 +- crates/bevy_render/src/pipelined_rendering.rs | 6 +- .../src/render_resource/pipeline_cache.rs | 9 +- crates/bevy_render/src/renderer/mod.rs | 30 +-- .../bevy_render/src/view/window/screenshot.rs | 8 +- crates/bevy_tasks/Cargo.toml | 2 +- crates/bevy_tasks/README.md | 4 +- crates/bevy_tasks/src/bevy_executor.rs | 53 ++-- crates/bevy_tasks/src/lib.rs | 83 +++++- crates/bevy_tasks/src/task_pool.rs | 224 ++++++++++------ crates/bevy_transform/src/systems.rs | 16 +- examples/README.md | 108 ++++---- examples/animation/animation_graph.rs | 10 +- examples/asset/multi_asset_sync.rs | 6 +- examples/async_tasks/async_compute.rs | 12 +- examples/ecs/parallel_query.rs | 2 +- examples/scene/scene.rs | 7 +- 38 files changed, 683 insertions(+), 549 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f18cbf6a3743a..4527f955e7422 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1982,13 +1982,13 @@ wasm = true # Async Tasks [[example]] -name = "async_compute" -path = "examples/async_tasks/async_compute.rs" +name = "blocking_compute" +path = "examples/async_tasks/blocking_compute.rs" doc-scrape-examples = true -[package.metadata.example.async_compute] -name = "Async Compute" -description = "How to use `AsyncComputeTaskPool` to complete longer running tasks" +[package.metadata.example.blocking_compute] +name = "Blocking Compute" +description = "How to use `TaskPool` to complete longer running tasks" category = "Async Tasks" wasm = false diff --git a/benches/benches/bevy_ecs/iteration/heavy_compute.rs b/benches/benches/bevy_ecs/iteration/heavy_compute.rs index 21483463cdd96..72141e4010fe4 100644 --- a/benches/benches/bevy_ecs/iteration/heavy_compute.rs +++ b/benches/benches/bevy_ecs/iteration/heavy_compute.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; +use bevy_tasks::{TaskPool, TaskPoolBuilder}; use criterion::Criterion; use glam::*; @@ -20,7 +20,7 @@ pub fn heavy_compute(c: &mut Criterion) { group.warm_up_time(core::time::Duration::from_millis(500)); group.measurement_time(core::time::Duration::from_secs(4)); group.bench_function("base", |b| { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); diff --git a/benches/benches/bevy_ecs/iteration/par_iter_simple.rs b/benches/benches/bevy_ecs/iteration/par_iter_simple.rs index 089a9854e76d4..aa7976c2f9730 100644 --- a/benches/benches/bevy_ecs/iteration/par_iter_simple.rs +++ b/benches/benches/bevy_ecs/iteration/par_iter_simple.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; +use bevy_tasks::{TaskPool, TaskPoolBuilder}; use glam::*; #[derive(Component, Copy, Clone)] @@ -26,7 +26,7 @@ fn insert_if_bit_enabled(entity: &mut EntityWorldMut, i: u16) { impl<'w> Benchmark<'w> { pub fn new(fragment: u16) -> Self { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); diff --git a/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs b/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs index 8b90783dc5c51..729a3dec89ec9 100644 --- a/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs +++ b/benches/benches/bevy_ecs/iteration/par_iter_simple_foreach_hybrid.rs @@ -1,5 +1,5 @@ use bevy_ecs::prelude::*; -use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; +use bevy_tasks::{TaskPool, TaskPoolBuilder}; use rand::{prelude::SliceRandom, SeedableRng}; use rand_chacha::ChaCha8Rng; @@ -18,7 +18,7 @@ pub struct Benchmark<'w>(World, QueryState<(&'w mut TableData, &'w SparseData)>) impl<'w> Benchmark<'w> { pub fn new() -> Self { let mut world = World::new(); - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut v = vec![]; for _ in 0..100000 { diff --git a/crates/bevy_app/src/task_pool_plugin.rs b/crates/bevy_app/src/task_pool_plugin.rs index f38d09d7f5e91..a65b8cfa321c0 100644 --- a/crates/bevy_app/src/task_pool_plugin.rs +++ b/crates/bevy_app/src/task_pool_plugin.rs @@ -1,8 +1,8 @@ use crate::{App, Plugin}; -use alloc::string::ToString; -use bevy_platform::sync::Arc; -use bevy_tasks::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, TaskPoolBuilder}; +use alloc::{string::ToString, vec::Vec}; +use bevy_platform::{collections::HashMap, sync::Arc}; +use bevy_tasks::{TaskPool, TaskPoolBuilder, TaskPriority}; use core::fmt::Debug; use log::trace; @@ -21,7 +21,7 @@ cfg_if::cfg_if! { } } -/// Setup of default task pools: [`AsyncComputeTaskPool`], [`ComputeTaskPool`], [`IoTaskPool`]. +/// Setup of default task pools: [`AsyncTaskPool`], [`TaskPool`], [`IoTaskPool`]. #[derive(Default)] pub struct TaskPoolPlugin { /// Options for the [`TaskPool`](bevy_tasks::TaskPool) created at application start. @@ -40,7 +40,7 @@ impl Plugin for TaskPoolPlugin { /// Defines a simple way to determine how many threads to use given the number of remaining cores /// and number of total cores -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct TaskPoolThreadAssignmentPolicy { /// Force using at least this many threads pub min_threads: usize, @@ -49,22 +49,6 @@ pub struct TaskPoolThreadAssignmentPolicy { /// Target using this percentage of total cores, clamped by `min_threads` and `max_threads`. It is /// permitted to use 1.0 to try to use all remaining threads pub percent: f32, - /// Callback that is invoked once for every created thread as it starts. - /// This configuration will be ignored under wasm platform. - pub on_thread_spawn: Option>, - /// Callback that is invoked once for every created thread as it terminates - /// This configuration will be ignored under wasm platform. - pub on_thread_destroy: Option>, -} - -impl Debug for TaskPoolThreadAssignmentPolicy { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("TaskPoolThreadAssignmentPolicy") - .field("min_threads", &self.min_threads) - .field("max_threads", &self.max_threads) - .field("percent", &self.percent) - .finish() - } } impl TaskPoolThreadAssignmentPolicy { @@ -92,7 +76,7 @@ impl TaskPoolThreadAssignmentPolicy { /// Helper for configuring and creating the default task pools. For end-users who want full control, /// set up [`TaskPoolPlugin`] -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct TaskPoolOptions { /// If the number of physical cores is less than `min_total_threads`, force using /// `min_total_threads` @@ -101,47 +85,70 @@ pub struct TaskPoolOptions { /// `max_total_threads` pub max_total_threads: usize, - /// Used to determine number of IO threads to allocate - pub io: TaskPoolThreadAssignmentPolicy, - /// Used to determine number of async compute threads to allocate - pub async_compute: TaskPoolThreadAssignmentPolicy, - /// Used to determine number of compute threads to allocate - pub compute: TaskPoolThreadAssignmentPolicy, + /// Callback that is invoked once for every created thread as it starts. + /// This configuration will be ignored under wasm platform. + pub on_thread_spawn: Option>, + /// Callback that is invoked once for every created thread as it terminates + /// This configuration will be ignored under wasm platform. + pub on_thread_destroy: Option>, + + /// Used to determine number of threads to provide to each + pub priority_assignment_policies: HashMap, +} + +impl Debug for TaskPoolOptions { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TaskPoolOptions") + .field("min_total_threads", &self.min_total_threads) + .field("max_total_threads", &self.max_total_threads) + .field( + "priority_assignment_policies", + &self.priority_assignment_policies, + ) + .finish() + } } impl Default for TaskPoolOptions { fn default() -> Self { - TaskPoolOptions { - // By default, use however many cores are available on the system - min_total_threads: 1, - max_total_threads: usize::MAX, - - // Use 25% of cores for IO, at least 1, no more than 4 - io: TaskPoolThreadAssignmentPolicy { + let mut priority_assignment_policies = HashMap::new(); + // Use 25% of cores for IO, at least 1, no more than 4 + priority_assignment_policies.insert( + TaskPriority::BlockingIO, + TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, percent: 0.25, - on_thread_spawn: None, - on_thread_destroy: None, }, - - // Use 25% of cores for async compute, at least 1, no more than 4 - async_compute: TaskPoolThreadAssignmentPolicy { + ); + // Use 25% of cores for blocking compute, at least 1, no more than 4 + priority_assignment_policies.insert( + TaskPriority::BlockingCompute, + TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, percent: 0.25, - on_thread_spawn: None, - on_thread_destroy: None, }, - - // Use all remaining cores for compute (at least 1) - compute: TaskPoolThreadAssignmentPolicy { + ); + // Use 25% of cores for async IO, at least 1, no more than 4 + priority_assignment_policies.insert( + TaskPriority::AsyncIO, + TaskPoolThreadAssignmentPolicy { min_threads: 1, - max_threads: usize::MAX, - percent: 1.0, // This 1.0 here means "whatever is left over" - on_thread_spawn: None, - on_thread_destroy: None, + max_threads: 4, + percent: 0.25, }, + ); + + TaskPoolOptions { + // By default, use however many cores are available on the system + min_total_threads: 1, + max_total_threads: usize::MAX, + + on_thread_spawn: None, + on_thread_destroy: None, + + priority_assignment_policies, } } } @@ -164,133 +171,55 @@ impl TaskPoolOptions { let mut remaining_threads = total_threads; - { - // Determine the number of IO threads we will use - let io_threads = self - .io - .get_number_of_threads(remaining_threads, total_threads); - - trace!("IO Threads: {io_threads}"); - remaining_threads = remaining_threads.saturating_sub(io_threads); - - IoTaskPool::get_or_init(|| { - let builder = TaskPoolBuilder::default() - .num_threads(io_threads) - .thread_name("IO Task Pool".to_string()); - - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - let builder = { - let mut builder = builder; - if let Some(f) = self.io.on_thread_spawn.clone() { - builder = builder.on_thread_spawn(move || f()); - } - if let Some(f) = self.io.on_thread_destroy.clone() { - builder = builder.on_thread_destroy(move || f()); - } - builder - }; - - builder - }); - } + let mut builder = TaskPoolBuilder::default() + .num_threads(total_threads) + .thread_name("Task Pool".to_string()); - { - // Determine the number of async compute threads we will use - let async_compute_threads = self - .async_compute - .get_number_of_threads(remaining_threads, total_threads); - - trace!("Async Compute Threads: {async_compute_threads}"); - remaining_threads = remaining_threads.saturating_sub(async_compute_threads); - - AsyncComputeTaskPool::get_or_init(|| { - let builder = TaskPoolBuilder::default() - .num_threads(async_compute_threads) - .thread_name("Async Compute Task Pool".to_string()); - - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - let builder = { - let mut builder = builder; - if let Some(f) = self.async_compute.on_thread_spawn.clone() { - builder = builder.on_thread_spawn(move || f()); - } - if let Some(f) = self.async_compute.on_thread_destroy.clone() { - builder = builder.on_thread_destroy(move || f()); - } - builder - }; - - builder - }); - } + let mut ordered = self.priority_assignment_policies.iter().collect::>(); + ordered.sort_by_key(|(prio, _)| **prio); + for (priority, policy) in ordered { + let priority_threads = policy.get_number_of_threads(remaining_threads, total_threads); + builder = builder.priority_limit(*priority, Some(priority_threads)); - { - // Determine the number of compute threads we will use - // This is intentionally last so that an end user can specify 1.0 as the percent - let compute_threads = self - .compute - .get_number_of_threads(remaining_threads, total_threads); - - trace!("Compute Threads: {compute_threads}"); - - ComputeTaskPool::get_or_init(|| { - let builder = TaskPoolBuilder::default() - .num_threads(compute_threads) - .thread_name("Compute Task Pool".to_string()); - - #[cfg(not(all(target_arch = "wasm32", feature = "web")))] - let builder = { - let mut builder = builder; - if let Some(f) = self.compute.on_thread_spawn.clone() { - builder = builder.on_thread_spawn(move || f()); - } - if let Some(f) = self.compute.on_thread_destroy.clone() { - builder = builder.on_thread_destroy(move || f()); - } - builder - }; - - builder - }); + remaining_threads = remaining_threads.saturating_sub(priority_threads); + trace!("{:?} Threads: {priority_threads}", *priority); } + + #[cfg(not(all(target_arch = "wasm32", feature = "web")))] + let builder = { + let mut builder = builder; + if let Some(f) = self.on_thread_spawn.clone() { + builder = builder.on_thread_spawn(move || f()); + } + if let Some(f) = self.on_thread_destroy.clone() { + builder = builder.on_thread_destroy(move || f()); + } + builder + }; + + TaskPool::get_or_init(move || builder); } } #[cfg(test)] mod tests { use super::*; - use bevy_tasks::prelude::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; + use bevy_tasks::prelude::TaskPool; #[test] fn runs_spawn_local_tasks() { let mut app = App::new(); app.add_plugins(TaskPoolPlugin::default()); - let (async_tx, async_rx) = crossbeam_channel::unbounded(); - AsyncComputeTaskPool::get() - .spawn_local(async move { - async_tx.send(()).unwrap(); - }) - .detach(); - - let (compute_tx, compute_rx) = crossbeam_channel::unbounded(); - ComputeTaskPool::get() - .spawn_local(async move { - compute_tx.send(()).unwrap(); - }) - .detach(); - - let (io_tx, io_rx) = crossbeam_channel::unbounded(); - IoTaskPool::get() + let (tx, rx) = crossbeam_channel::unbounded(); + TaskPool::get() .spawn_local(async move { - io_tx.send(()).unwrap(); + tx.send(()).unwrap(); }) .detach(); app.run(); - async_rx.try_recv().unwrap(); - compute_rx.try_recv().unwrap(); - io_rx.try_recv().unwrap(); + rx.try_recv().unwrap(); } } diff --git a/crates/bevy_asset/src/processor/mod.rs b/crates/bevy_asset/src/processor/mod.rs index 7b3d36e686b02..258271808f9f7 100644 --- a/crates/bevy_asset/src/processor/mod.rs +++ b/crates/bevy_asset/src/processor/mod.rs @@ -59,7 +59,7 @@ use crate::{ use alloc::{borrow::ToOwned, boxed::Box, collections::VecDeque, sync::Arc, vec, vec::Vec}; use bevy_ecs::prelude::*; use bevy_platform::collections::{HashMap, HashSet}; -use bevy_tasks::IoTaskPool; +use bevy_tasks::{TaskPool, TaskPriority}; use futures_io::ErrorKind; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use parking_lot::RwLock; @@ -219,15 +219,18 @@ impl AssetProcessor { pub fn process_assets(&self) { let start_time = std::time::Instant::now(); debug!("Processing Assets"); - IoTaskPool::get().scope(|scope| { - scope.spawn(async move { - self.initialize().await.unwrap(); - for source in self.sources().iter_processed() { - self.process_assets_internal(scope, source, PathBuf::from("")) - .await - .unwrap(); - } - }); + TaskPool::get().scope(|scope| { + scope + .builder() + .with_priority(TaskPriority::BlockingCompute) + .spawn(async move { + self.initialize().await.unwrap(); + for source in self.sources().iter_processed() { + self.process_assets_internal(scope, source, PathBuf::from("")) + .await + .unwrap(); + } + }); }); // This must happen _after_ the scope resolves or it will happen "too early" // Don't move this into the async scope above! process_assets is a blocking/sync function this is fine @@ -421,12 +424,15 @@ impl AssetProcessor { #[cfg(any(target_arch = "wasm32", not(feature = "multi_threaded")))] error!("AddFolder event cannot be handled in single threaded mode (or Wasm) yet."); #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] - IoTaskPool::get().scope(|scope| { - scope.spawn(async move { - self.process_assets_internal(scope, source, path) - .await - .unwrap(); - }); + TaskPool::get().scope(|scope| { + scope + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + self.process_assets_internal(scope, source, path) + .await + .unwrap(); + }); }); } @@ -563,13 +569,16 @@ impl AssetProcessor { loop { let mut check_reprocess_queue = core::mem::take(&mut self.data.asset_infos.write().await.check_reprocess_queue); - IoTaskPool::get().scope(|scope| { + TaskPool::get().scope(|scope| { for path in check_reprocess_queue.drain(..) { let processor = self.clone(); let source = self.get_source(path.source()).unwrap(); - scope.spawn(async move { - processor.process_asset(source, path.into()).await; - }); + scope + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + processor.process_asset(source, path.into()).await; + }); } }); let infos = self.data.asset_infos.read().await; diff --git a/crates/bevy_asset/src/server/loaders.rs b/crates/bevy_asset/src/server/loaders.rs index 9c13c861bd986..0be6c3786c5b5 100644 --- a/crates/bevy_asset/src/server/loaders.rs +++ b/crates/bevy_asset/src/server/loaders.rs @@ -5,7 +5,7 @@ use crate::{ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use async_broadcast::RecvError; use bevy_platform::collections::HashMap; -use bevy_tasks::IoTaskPool; +use bevy_tasks::{TaskPool, TaskPriority}; use bevy_utils::TypeIdMap; use core::any::TypeId; use thiserror::Error; @@ -91,7 +91,9 @@ impl AssetLoaders { match maybe_loader { MaybeAssetLoader::Ready(_) => unreachable!(), MaybeAssetLoader::Pending { sender, .. } => { - IoTaskPool::get() + TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) .spawn(async move { let _ = sender.broadcast(loader).await; }) diff --git a/crates/bevy_asset/src/server/mod.rs b/crates/bevy_asset/src/server/mod.rs index 641952a67150e..9bfbb19890071 100644 --- a/crates/bevy_asset/src/server/mod.rs +++ b/crates/bevy_asset/src/server/mod.rs @@ -27,7 +27,7 @@ use alloc::{ use atomicow::CowArc; use bevy_ecs::prelude::*; use bevy_platform::collections::HashSet; -use bevy_tasks::IoTaskPool; +use bevy_tasks::{TaskPool, TaskPriority}; use core::{any::TypeId, future::Future, panic::AssertUnwindSafe, task::Poll}; use crossbeam_channel::{Receiver, Sender}; use either::Either; @@ -524,15 +524,18 @@ impl AssetServer { let owned_handle = handle.clone(); let server = self.clone(); - let task = IoTaskPool::get().spawn(async move { - if let Err(err) = server - .load_internal(Some(owned_handle), path, false, None) - .await - { - error!("{}", err); - } - drop(guard); - }); + let task = TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + if let Err(err) = server + .load_internal(Some(owned_handle), path, false, None) + .await + { + error!("{}", err); + } + drop(guard); + }); #[cfg(not(any(target_arch = "wasm32", not(feature = "multi_threaded"))))] { @@ -587,24 +590,29 @@ impl AssetServer { let id = handle.id().untyped(); let server = self.clone(); - let task = IoTaskPool::get().spawn(async move { - let path_clone = path.clone(); - match server.load_untyped_async(path).await { - Ok(handle) => server.send_asset_event(InternalAssetEvent::Loaded { - id, - loaded_asset: LoadedAsset::new_with_dependencies(LoadedUntypedAsset { handle }) - .into(), - }), - Err(err) => { - error!("{err}"); - server.send_asset_event(InternalAssetEvent::Failed { + let task = TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + let path_clone = path.clone(); + match server.load_untyped_async(path).await { + Ok(handle) => server.send_asset_event(InternalAssetEvent::Loaded { id, - path: path_clone, - error: err, - }); + loaded_asset: LoadedAsset::new_with_dependencies(LoadedUntypedAsset { + handle, + }) + .into(), + }), + Err(err) => { + error!("{err}"); + server.send_asset_event(InternalAssetEvent::Failed { + id, + path: path_clone, + error: err, + }); + } } - } - }); + }); #[cfg(not(any(target_arch = "wasm32", not(feature = "multi_threaded"))))] infos.pending_tasks.insert(handle.id().untyped(), task); @@ -827,7 +835,9 @@ impl AssetServer { pub fn reload<'a>(&self, path: impl Into>) { let server = self.clone(); let path = path.into().into_owned(); - IoTaskPool::get() + TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) .spawn(async move { let mut reloaded = false; @@ -922,29 +932,32 @@ impl AssetServer { let event_sender = self.data.asset_event_sender.clone(); - let task = IoTaskPool::get().spawn(async move { - match future.await { - Ok(asset) => { - let loaded_asset = LoadedAsset::new_with_dependencies(asset).into(); - event_sender - .send(InternalAssetEvent::Loaded { id, loaded_asset }) - .unwrap(); - } - Err(error) => { - let error = AddAsyncError { - error: Arc::new(error), - }; - error!("{error}"); - event_sender - .send(InternalAssetEvent::Failed { - id, - path: Default::default(), - error: AssetLoadError::AddAsyncError(error), - }) - .unwrap(); + let task = TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + match future.await { + Ok(asset) => { + let loaded_asset = LoadedAsset::new_with_dependencies(asset).into(); + event_sender + .send(InternalAssetEvent::Loaded { id, loaded_asset }) + .unwrap(); + } + Err(error) => { + let error = AddAsyncError { + error: Arc::new(error), + }; + error!("{error}"); + event_sender + .send(InternalAssetEvent::Failed { + id, + path: Default::default(), + error: AssetLoadError::AddAsyncError(error), + }) + .unwrap(); + } } - } - }); + }); #[cfg(not(any(target_arch = "wasm32", not(feature = "multi_threaded"))))] infos.pending_tasks.insert(id, task); @@ -1025,7 +1038,9 @@ impl AssetServer { let path = path.into_owned(); let server = self.clone(); - IoTaskPool::get() + TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingIO) .spawn(async move { let Ok(source) = server.get_source(path.source()) else { error!( diff --git a/crates/bevy_diagnostic/src/system_information_diagnostics_plugin.rs b/crates/bevy_diagnostic/src/system_information_diagnostics_plugin.rs index 83d3663895ca5..94297767b9e56 100644 --- a/crates/bevy_diagnostic/src/system_information_diagnostics_plugin.rs +++ b/crates/bevy_diagnostic/src/system_information_diagnostics_plugin.rs @@ -80,7 +80,7 @@ pub mod internal { use bevy_ecs::resource::Resource; use bevy_ecs::{prelude::ResMut, system::Local}; use bevy_platform::time::Instant; - use bevy_tasks::{available_parallelism, block_on, poll_once, AsyncComputeTaskPool, Task}; + use bevy_tasks::{available_parallelism, block_on, poll_once, Task, TaskPool, TaskPriority}; use log::info; use std::sync::Mutex; use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System}; @@ -143,7 +143,7 @@ pub mod internal { let last_refresh = last_refresh.get_or_insert_with(Instant::now); - let thread_pool = AsyncComputeTaskPool::get(); + let thread_pool = TaskPool::get(); // Only queue a new system refresh task when necessary // Queuing earlier than that will not give new data @@ -153,35 +153,38 @@ pub mod internal { && tasks.tasks.len() * 2 < available_parallelism() { let sys = Arc::clone(sysinfo); - let task = thread_pool.spawn(async move { - let mut sys = sys.lock().unwrap(); - let pid = sysinfo::get_current_pid().expect("Failed to get current process ID"); - sys.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true); + let task = thread_pool + .builder() + .with_priority(TaskPriority::BlockingCompute) + .spawn(async move { + let mut sys = sys.lock().unwrap(); + let pid = sysinfo::get_current_pid().expect("Failed to get current process ID"); + sys.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true); - sys.refresh_cpu_specifics(CpuRefreshKind::nothing().with_cpu_usage()); - sys.refresh_memory(); - let system_cpu_usage = sys.global_cpu_usage().into(); - let total_mem = sys.total_memory() as f64; - let used_mem = sys.used_memory() as f64; - let system_mem_usage = used_mem / total_mem * 100.0; + sys.refresh_cpu_specifics(CpuRefreshKind::nothing().with_cpu_usage()); + sys.refresh_memory(); + let system_cpu_usage = sys.global_cpu_usage().into(); + let total_mem = sys.total_memory() as f64; + let used_mem = sys.used_memory() as f64; + let system_mem_usage = used_mem / total_mem * 100.0; - let process_mem_usage = sys - .process(pid) - .map(|p| p.memory() as f64 * BYTES_TO_GIB) - .unwrap_or(0.0); + let process_mem_usage = sys + .process(pid) + .map(|p| p.memory() as f64 * BYTES_TO_GIB) + .unwrap_or(0.0); - let process_cpu_usage = sys - .process(pid) - .map(|p| p.cpu_usage() as f64 / sys.cpus().len() as f64) - .unwrap_or(0.0); + let process_cpu_usage = sys + .process(pid) + .map(|p| p.cpu_usage() as f64 / sys.cpus().len() as f64) + .unwrap_or(0.0); - SysinfoRefreshData { - system_cpu_usage, - system_mem_usage, - process_cpu_usage, - process_mem_usage, - } - }); + SysinfoRefreshData { + system_cpu_usage, + system_mem_usage, + process_cpu_usage, + process_mem_usage, + } + }); tasks.tasks.push(task); *last_refresh = Instant::now(); } diff --git a/crates/bevy_ecs/src/batching.rs b/crates/bevy_ecs/src/batching.rs index ab9f2d582c781..c4ee552eca53a 100644 --- a/crates/bevy_ecs/src/batching.rs +++ b/crates/bevy_ecs/src/batching.rs @@ -29,13 +29,13 @@ pub struct BatchingStrategy { /// /// Defaults to `[1, usize::MAX]`. pub batch_size_limits: Range, - /// The number of batches per thread in the [`ComputeTaskPool`]. + /// The number of batches per thread in the [`TaskPool`]. /// Increasing this value will decrease the batch size, which may /// increase the scheduling overhead for the iteration. /// /// Defaults to 1. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool pub batches_per_thread: usize, } diff --git a/crates/bevy_ecs/src/event/iterators.rs b/crates/bevy_ecs/src/event/iterators.rs index c90aed2a19d5e..ca4031f39ca29 100644 --- a/crates/bevy_ecs/src/event/iterators.rs +++ b/crates/bevy_ecs/src/event/iterators.rs @@ -189,10 +189,10 @@ impl<'a, E: BufferedEvent> EventParIter<'a, E> { /// Unlike normal iteration, the event order is not guaranteed in any form. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from an event reader that is being + /// If the [`TaskPool`] is not initialized. If using this from an event reader that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool pub fn for_each(self, func: FN) { self.for_each_with_id(move |e, _| func(e)); } @@ -203,10 +203,10 @@ impl<'a, E: BufferedEvent> EventParIter<'a, E> { /// Note that the order of iteration is not guaranteed, but `EventId`s are ordered by send order. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from an event reader that is being + /// If the [`TaskPool`] is not initialized. If using this from an event reader that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[cfg_attr( target_arch = "wasm32", expect(unused_mut, reason = "not mutated on this target") @@ -219,7 +219,7 @@ impl<'a, E: BufferedEvent> EventParIter<'a, E> { #[cfg(not(target_arch = "wasm32"))] { - let pool = bevy_tasks::ComputeTaskPool::get(); + let pool = bevy_tasks::TaskPool::get(); let thread_count = pool.thread_num(); if thread_count <= 1 { return self.into_iter().for_each(|(e, i)| func(e, i)); diff --git a/crates/bevy_ecs/src/event/mut_iterators.rs b/crates/bevy_ecs/src/event/mut_iterators.rs index 3fa8378f23c17..b90b4cff9f6ef 100644 --- a/crates/bevy_ecs/src/event/mut_iterators.rs +++ b/crates/bevy_ecs/src/event/mut_iterators.rs @@ -190,10 +190,10 @@ impl<'a, E: BufferedEvent> EventMutParIter<'a, E> { /// Unlike normal iteration, the event order is not guaranteed in any form. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from an event reader that is being + /// If the [`TaskPool`] is not initialized. If using this from an event reader that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool pub fn for_each(self, func: FN) { self.for_each_with_id(move |e, _| func(e)); } @@ -204,10 +204,10 @@ impl<'a, E: BufferedEvent> EventMutParIter<'a, E> { /// Note that the order of iteration is not guaranteed, but `EventId`s are ordered by send order. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from an event reader that is being + /// If the [`TaskPool`] is not initialized. If using this from an event reader that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[cfg_attr( target_arch = "wasm32", expect(unused_mut, reason = "not mutated on this target") @@ -223,7 +223,7 @@ impl<'a, E: BufferedEvent> EventMutParIter<'a, E> { #[cfg(not(target_arch = "wasm32"))] { - let pool = bevy_tasks::ComputeTaskPool::get(); + let pool = bevy_tasks::TaskPool::get(); let thread_count = pool.thread_num(); if thread_count <= 1 { return self.into_iter().for_each(|(e, i)| func(e, i)); diff --git a/crates/bevy_ecs/src/lib.rs b/crates/bevy_ecs/src/lib.rs index 8cddefae56bd2..15de1194d1c2b 100644 --- a/crates/bevy_ecs/src/lib.rs +++ b/crates/bevy_ecs/src/lib.rs @@ -170,7 +170,7 @@ mod tests { }; use alloc::{string::String, sync::Arc, vec, vec::Vec}; use bevy_platform::collections::HashSet; - use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; + use bevy_tasks::{TaskPool, TaskPoolBuilder}; use core::{ any::TypeId, marker::PhantomData, @@ -495,7 +495,7 @@ mod tests { #[test] fn par_for_each_dense() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); let e1 = world.spawn(A(1)).id(); let e2 = world.spawn(A(2)).id(); @@ -517,7 +517,7 @@ mod tests { #[test] fn par_for_each_sparse() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::new(); let e1 = world.spawn(SparseStored(1)).id(); let e2 = world.spawn(SparseStored(2)).id(); diff --git a/crates/bevy_ecs/src/query/par_iter.rs b/crates/bevy_ecs/src/query/par_iter.rs index b8d8618fa5bf1..a2559fa23972d 100644 --- a/crates/bevy_ecs/src/query/par_iter.rs +++ b/crates/bevy_ecs/src/query/par_iter.rs @@ -34,10 +34,10 @@ impl<'w, 's, D: QueryData, F: QueryFilter> QueryParIter<'w, 's, D, F> { /// Runs `func` on each query result in parallel. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each) + Send + Sync + Clone>(self, func: FN) { self.for_each_init(|| {}, |_, item| func(item)); @@ -69,10 +69,10 @@ impl<'w, 's, D: QueryData, F: QueryFilter> QueryParIter<'w, 's, D, F> { /// ``` /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each_init(self, init: INIT, func: FN) where @@ -101,7 +101,7 @@ impl<'w, 's, D: QueryData, F: QueryFilter> QueryParIter<'w, 's, D, F> { } #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { - let thread_count = bevy_tasks::ComputeTaskPool::get().thread_num(); + let thread_count = bevy_tasks::TaskPool::get().thread_num(); if thread_count <= 1 { let init = init(); // SAFETY: See the safety comment above. @@ -185,10 +185,10 @@ impl<'w, 's, D: ReadOnlyQueryData, F: QueryFilter, E: EntityEquivalent + Sync> /// Runs `func` on each query result in parallel. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each) + Send + Sync + Clone>(self, func: FN) { self.for_each_init(|| {}, |_, item| func(item)); @@ -240,10 +240,10 @@ impl<'w, 's, D: ReadOnlyQueryData, F: QueryFilter, E: EntityEquivalent + Sync> /// ``` /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each_init(self, init: INIT, func: FN) where @@ -272,7 +272,7 @@ impl<'w, 's, D: ReadOnlyQueryData, F: QueryFilter, E: EntityEquivalent + Sync> } #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { - let thread_count = bevy_tasks::ComputeTaskPool::get().thread_num(); + let thread_count = bevy_tasks::TaskPool::get().thread_num(); if thread_count <= 1 { let init = init(); // SAFETY: See the safety comment above. @@ -340,10 +340,10 @@ impl<'w, 's, D: QueryData, F: QueryFilter, E: EntityEquivalent + Sync> /// Runs `func` on each query result in parallel. /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each) + Send + Sync + Clone>(self, func: FN) { self.for_each_init(|| {}, |_, item| func(item)); @@ -395,10 +395,10 @@ impl<'w, 's, D: QueryData, F: QueryFilter, E: EntityEquivalent + Sync> /// ``` /// /// # Panics - /// If the [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// If the [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn for_each_init(self, init: INIT, func: FN) where @@ -427,7 +427,7 @@ impl<'w, 's, D: QueryData, F: QueryFilter, E: EntityEquivalent + Sync> } #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] { - let thread_count = bevy_tasks::ComputeTaskPool::get().thread_num(); + let thread_count = bevy_tasks::TaskPool::get().thread_num(); if thread_count <= 1 { let init = init(); // SAFETY: See the safety comment above. diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index 9e19417d6723d..f825530128dea 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -1345,7 +1345,7 @@ impl QueryState { /// #[derive(Component, PartialEq, Debug)] /// struct A(usize); /// - /// # bevy_tasks::ComputeTaskPool::get_or_init(|| bevy_tasks::TaskPoolBuilder::default()); + /// # bevy_tasks::TaskPool::get_or_init(|| bevy_tasks::TaskPoolBuilder::default()); /// /// let mut world = World::new(); /// @@ -1371,11 +1371,11 @@ impl QueryState { /// ``` /// /// # Panics - /// The [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// The [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// /// [`par_iter`]: Self::par_iter - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[inline] pub fn par_iter_mut<'w, 's>(&'s mut self, world: &'w mut World) -> QueryParIter<'w, 's, D, F> { self.query_mut(world).par_iter_inner() @@ -1386,7 +1386,7 @@ impl QueryState { /// `iter()` method, but cannot be chained like a normal [`Iterator`]. /// /// # Panics - /// The [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// The [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// /// # Safety @@ -1396,7 +1396,7 @@ impl QueryState { /// This does not validate that `world.id()` matches `self.world_id`. Calling this on a `world` /// with a mismatched [`WorldId`] is unsound. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] pub(crate) unsafe fn par_fold_init_unchecked_manual<'w, 's, T, FN, INIT>( &'s self, @@ -1415,7 +1415,7 @@ impl QueryState { // QueryState::par_many_fold_init_unchecked_manual, QueryState::par_many_unique_fold_init_unchecked_manual use arrayvec::ArrayVec; - bevy_tasks::ComputeTaskPool::get().scope(|scope| { + bevy_tasks::TaskPool::get().scope(|scope| { // SAFETY: We only access table data that has been registered in `self.component_access`. let tables = unsafe { &world.storages().tables }; let archetypes = world.archetypes(); @@ -1500,7 +1500,7 @@ impl QueryState { /// equivalent `iter_many_unique()` method, but cannot be chained like a normal [`Iterator`]. /// /// # Panics - /// The [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// The [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// /// # Safety @@ -1510,7 +1510,7 @@ impl QueryState { /// This does not validate that `world.id()` matches `self.world_id`. Calling this on a `world` /// with a mismatched [`WorldId`] is unsound. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] pub(crate) unsafe fn par_many_unique_fold_init_unchecked_manual<'w, 's, T, FN, INIT, E>( &'s self, @@ -1530,7 +1530,7 @@ impl QueryState { // QueryIter, QueryIterationCursor, QueryManyIter, QueryCombinationIter,QueryState::par_fold_init_unchecked_manual // QueryState::par_many_fold_init_unchecked_manual, QueryState::par_many_unique_fold_init_unchecked_manual - bevy_tasks::ComputeTaskPool::get().scope(|scope| { + bevy_tasks::TaskPool::get().scope(|scope| { let chunks = entity_list.chunks_exact(batch_size as usize); let remainder = chunks.remainder(); @@ -1563,7 +1563,7 @@ impl QueryState { /// `iter_many()` method, but cannot be chained like a normal [`Iterator`]. /// /// # Panics - /// The [`ComputeTaskPool`] is not initialized. If using this from a query that is being + /// The [`TaskPool`] is not initialized. If using this from a query that is being /// initialized and run from the ECS scheduler, this should never panic. /// /// # Safety @@ -1573,7 +1573,7 @@ impl QueryState { /// This does not validate that `world.id()` matches `self.world_id`. Calling this on a `world` /// with a mismatched [`WorldId`] is unsound. /// - /// [`ComputeTaskPool`]: bevy_tasks::ComputeTaskPool + /// [`TaskPool`]: bevy_tasks::TaskPool #[cfg(all(not(target_arch = "wasm32"), feature = "multi_threaded"))] pub(crate) unsafe fn par_many_fold_init_unchecked_manual<'w, 's, T, FN, INIT, E>( &'s self, @@ -1593,7 +1593,7 @@ impl QueryState { // QueryIter, QueryIterationCursor, QueryManyIter, QueryCombinationIter, QueryState::par_fold_init_unchecked_manual // QueryState::par_many_fold_init_unchecked_manual, QueryState::par_many_unique_fold_init_unchecked_manual - bevy_tasks::ComputeTaskPool::get().scope(|scope| { + bevy_tasks::TaskPool::get().scope(|scope| { let chunks = entity_list.chunks_exact(batch_size as usize); let remainder = chunks.remainder(); diff --git a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs index 9b06549860b2f..342033586a301 100644 --- a/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs +++ b/crates/bevy_ecs/src/schedule/executor/multi_threaded.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, vec::Vec}; use bevy_platform::cell::SyncUnsafeCell; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPoolBuilder, ThreadSpawner}; +use bevy_tasks::{Scope, ScopeTaskTarget, TaskPool, TaskPoolBuilder, ThreadSpawner}; use concurrent_queue::ConcurrentQueue; use core::{any::Any, panic::AssertUnwindSafe}; use fixedbitset::FixedBitSet; @@ -274,7 +274,7 @@ impl SystemExecutor for MultiThreadedExecutor { let environment = &Environment::new(self, schedule, world); - ComputeTaskPool::get_or_init(TaskPoolBuilder::default).scope_with_executor( + TaskPool::get_or_init(TaskPoolBuilder::default).scope_with_executor( thread_executor, |scope| { let context = Context { @@ -700,7 +700,11 @@ impl ExecutorState { context.scope.spawn(task); } else { self.local_thread_running = true; - context.scope.spawn_on_external(task); + context + .scope + .builder() + .with_target(ScopeTaskTarget::External) + .spawn(task); } } @@ -724,7 +728,11 @@ impl ExecutorState { context.system_completed(system_index, res, system); }; - context.scope.spawn_on_scope(task); + context + .scope + .builder() + .with_target(ScopeTaskTarget::Scope) + .spawn(task); } else { let task = async move { // SAFETY: `can_run` returned true for this system, which means @@ -746,7 +754,11 @@ impl ExecutorState { context.system_completed(system_index, res, system); }; - context.scope.spawn_on_scope(task); + context + .scope + .builder() + .with_target(ScopeTaskTarget::Scope) + .spawn(task); } self.exclusive_running = true; @@ -874,7 +886,7 @@ impl Default for MainThreadSpawner { impl MainThreadSpawner { /// Creates a new executor that can be used to run systems on the main thread. pub fn new() -> Self { - MainThreadSpawner(ComputeTaskPool::get().current_thread_spawner()) + MainThreadSpawner(TaskPool::get().current_thread_spawner()) } } diff --git a/crates/bevy_ecs/src/schedule/mod.rs b/crates/bevy_ecs/src/schedule/mod.rs index f19efb49dfa3b..f120817d58edd 100644 --- a/crates/bevy_ecs/src/schedule/mod.rs +++ b/crates/bevy_ecs/src/schedule/mod.rs @@ -110,12 +110,12 @@ mod tests { #[cfg(not(miri))] fn parallel_execution() { use alloc::sync::Arc; - use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; + use bevy_tasks::{TaskPool, TaskPoolBuilder}; use std::sync::Barrier; let mut world = World::default(); let mut schedule = Schedule::default(); - let thread_count = ComputeTaskPool::get_or_init(TaskPoolBuilder::default).thread_num(); + let thread_count = TaskPool::get_or_init(TaskPoolBuilder::default).thread_num(); let barrier = Arc::new(Barrier::new(thread_count)); diff --git a/crates/bevy_ecs/src/system/commands/parallel_scope.rs b/crates/bevy_ecs/src/system/commands/parallel_scope.rs index bee491017d500..76ba6163c71b6 100644 --- a/crates/bevy_ecs/src/system/commands/parallel_scope.rs +++ b/crates/bevy_ecs/src/system/commands/parallel_scope.rs @@ -29,7 +29,7 @@ struct ParallelCommandQueue { /// /// ``` /// # use bevy_ecs::prelude::*; -/// # use bevy_tasks::ComputeTaskPool; +/// # use bevy_tasks::TaskPool; /// # /// # #[derive(Component)] /// # struct Velocity; diff --git a/crates/bevy_gltf/src/loader/mod.rs b/crates/bevy_gltf/src/loader/mod.rs index 3eed903cca8ee..5ddd5e2b68eb8 100644 --- a/crates/bevy_gltf/src/loader/mod.rs +++ b/crates/bevy_gltf/src/loader/mod.rs @@ -44,7 +44,7 @@ use bevy_platform::collections::{HashMap, HashSet}; use bevy_render::render_resource::Face; use bevy_scene::Scene; #[cfg(not(target_arch = "wasm32"))] -use bevy_tasks::IoTaskPool; +use bevy_tasks::{TaskPool, TaskPriority}; use bevy_transform::components::Transform; use gltf::{ @@ -620,24 +620,27 @@ impl GltfLoader { } } else { #[cfg(not(target_arch = "wasm32"))] - IoTaskPool::get() + TaskPool::get() .scope(|scope| { gltf.textures().for_each(|gltf_texture| { let parent_path = load_context.path().parent().unwrap(); let linear_textures = &linear_textures; let buffer_data = &buffer_data; - scope.spawn(async move { - load_image( - gltf_texture, - buffer_data, - linear_textures, - parent_path, - loader.supported_compressed_formats, - default_sampler, - settings, - ) - .await - }); + scope + .builder() + .with_priority(TaskPriority::BlockingIO) + .spawn(async move { + load_image( + gltf_texture, + buffer_data, + linear_textures, + parent_path, + loader.supported_compressed_formats, + default_sampler, + settings, + ) + .await + }); }); }) .into_iter() diff --git a/crates/bevy_pbr/src/meshlet/from_mesh.rs b/crates/bevy_pbr/src/meshlet/from_mesh.rs index 141f4da0238ed..5c6b84098bd18 100644 --- a/crates/bevy_pbr/src/meshlet/from_mesh.rs +++ b/crates/bevy_pbr/src/meshlet/from_mesh.rs @@ -10,7 +10,7 @@ use bevy_math::{ use bevy_mesh::{Indices, Mesh}; use bevy_platform::collections::HashMap; use bevy_render::render_resource::PrimitiveTopology; -use bevy_tasks::{AsyncComputeTaskPool, ParallelSlice}; +use bevy_tasks::{ParallelSlice, TaskPool}; use bitvec::{order::Lsb0, vec::BitVec, view::BitView}; use core::{f32, ops::Range}; use itertools::Itertools; @@ -136,7 +136,7 @@ impl MeshletMesh { position_only_vertex_count, ); - let simplified = groups.par_chunk_map(AsyncComputeTaskPool::get(), 1, |_, groups| { + let simplified = groups.par_chunk_map(AsyncTaskPool::get(), 1, |_, groups| { let mut group = groups[0].clone(); // If the group only has a single meshlet we can't simplify it diff --git a/crates/bevy_remote/src/http.rs b/crates/bevy_remote/src/http.rs index 4e36e4a0bfe94..519af66bb317d 100644 --- a/crates/bevy_remote/src/http.rs +++ b/crates/bevy_remote/src/http.rs @@ -17,7 +17,7 @@ use async_io::Async; use bevy_app::{App, Plugin, Startup}; use bevy_ecs::resource::Resource; use bevy_ecs::system::Res; -use bevy_tasks::{futures_lite::StreamExt, IoTaskPool}; +use bevy_tasks::{futures_lite::StreamExt, TaskPool}; use core::{ convert::Infallible, net::{IpAddr, Ipv4Addr}, @@ -201,7 +201,9 @@ fn start_http_server( remote_port: Res, headers: Res, ) { - IoTaskPool::get() + TaskPool::get() + .builder() + .with_priority(bevy_tasks::TaskPriority::AsyncIO) .spawn(server_main( address.0, remote_port.0, @@ -236,7 +238,9 @@ async fn listen( let request_sender = request_sender.clone(); let headers = headers.clone(); - IoTaskPool::get() + TaskPool::get() + .builder() + .with_priority(bevy_tasks::TaskPriority::AsyncIO) .spawn(async move { let _ = handle_client(client, request_sender, headers).await; }) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 99666e2dff061..d3e1f1f3be325 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -353,7 +353,8 @@ impl Plugin for RenderPlugin { // In wasm, spawn a task and detach it for execution #[cfg(target_arch = "wasm32")] - bevy_tasks::IoTaskPool::get() + bevy_tasks::TaskPool::get() + .with_priority(bevy_tasks::TaskPriority::BlockingIO) .spawn_local(async_renderer) .detach(); // Otherwise, just block for it to complete diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 61f9a446a2150..e3f2a1826f44e 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -6,7 +6,7 @@ use bevy_ecs::{ schedule::MainThreadSpawner, world::{Mut, World}, }; -use bevy_tasks::ComputeTaskPool; +use bevy_tasks::TaskPool; use crate::RenderApp; @@ -150,7 +150,7 @@ impl Plugin for PipelinedRenderingPlugin { #[cfg(feature = "trace")] let _span = tracing::info_span!("render thread").entered(); - let compute_task_pool = ComputeTaskPool::get(); + let compute_task_pool = TaskPool::get(); loop { // run a scope here to allow main world to use this thread while it's waiting for the render app let sent_app = compute_task_pool @@ -185,7 +185,7 @@ fn renderer_extract(app_world: &mut World, _world: &mut World) { world.resource_scope(|world, mut render_channels: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. - if let Some(mut render_app) = ComputeTaskPool::get() + if let Some(mut render_app) = TaskPool::get() .scope_with_executor(Some(main_thread_executor.0.clone()), |s| { s.spawn(async { render_channels.recv().await }); }) diff --git a/crates/bevy_render/src/render_resource/pipeline_cache.rs b/crates/bevy_render/src/render_resource/pipeline_cache.rs index 1224e1998fb0d..56e416cdef22c 100644 --- a/crates/bevy_render/src/render_resource/pipeline_cache.rs +++ b/crates/bevy_render/src/render_resource/pipeline_cache.rs @@ -16,7 +16,7 @@ use bevy_shader::{ CachedPipelineId, PipelineCacheError, Shader, ShaderCache, ShaderCacheSource, ShaderDefVal, ValidateShader, }; -use bevy_tasks::Task; +use bevy_tasks::{Task, TaskPool, TaskPriority}; use bevy_utils::default; use core::{future::Future, hash::Hash, mem}; use std::sync::{Mutex, PoisonError}; @@ -807,7 +807,12 @@ fn create_pipeline_task( sync: bool, ) -> CachedPipelineState { if !sync { - return CachedPipelineState::Creating(bevy_tasks::AsyncComputeTaskPool::get().spawn(task)); + return CachedPipelineState::Creating( + TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingCompute) + .spawn(task), + ); } match bevy_tasks::block_on(task) { diff --git a/crates/bevy_render/src/renderer/mod.rs b/crates/bevy_render/src/renderer/mod.rs index 019d5f50e2422..69d05afcebd7e 100644 --- a/crates/bevy_render/src/renderer/mod.rs +++ b/crates/bevy_render/src/renderer/mod.rs @@ -579,24 +579,22 @@ impl<'w> RenderContext<'w> { #[cfg(not(all(target_arch = "wasm32", target_feature = "atomics")))] { - let mut task_based_command_buffers = - bevy_tasks::ComputeTaskPool::get().scope(|task_pool| { - for (i, queued_command_buffer) in - self.command_buffer_queue.into_iter().enumerate() - { - match queued_command_buffer { - QueuedCommandBuffer::Ready(command_buffer) => { - command_buffers.push((i, command_buffer)); - } - QueuedCommandBuffer::Task(command_buffer_generation_task) => { - let render_device = self.render_device.clone(); - task_pool.spawn(async move { - (i, command_buffer_generation_task(render_device)) - }); - } + let mut task_based_command_buffers = bevy_tasks::TaskPool::get().scope(|task_pool| { + for (i, queued_command_buffer) in self.command_buffer_queue.into_iter().enumerate() + { + match queued_command_buffer { + QueuedCommandBuffer::Ready(command_buffer) => { + command_buffers.push((i, command_buffer)); + } + QueuedCommandBuffer::Task(command_buffer_generation_task) => { + let render_device = self.render_device.clone(); + task_pool.spawn(async move { + (i, command_buffer_generation_task(render_device)) + }); } } - }); + } + }); command_buffers.append(&mut task_based_command_buffers); } diff --git a/crates/bevy_render/src/view/window/screenshot.rs b/crates/bevy_render/src/view/window/screenshot.rs index b87d76252c557..da2ca3a2f212f 100644 --- a/crates/bevy_render/src/view/window/screenshot.rs +++ b/crates/bevy_render/src/view/window/screenshot.rs @@ -25,7 +25,7 @@ use bevy_image::{Image, TextureFormatPixelInfo, ToExtents}; use bevy_platform::collections::HashSet; use bevy_reflect::Reflect; use bevy_shader::Shader; -use bevy_tasks::AsyncComputeTaskPool; +use bevy_tasks::{TaskPool, TaskPriority}; use bevy_utils::default; use bevy_window::{PrimaryWindow, WindowRef}; use core::ops::Deref; @@ -678,6 +678,10 @@ pub(crate) fn collect_screenshots(world: &mut World) { } }; - AsyncComputeTaskPool::get().spawn(finish).detach(); + TaskPool::get() + .builder() + .with_priority(TaskPriority::BlockingCompute) + .spawn(finish) + .detach(); } } diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 2af9bebd89a10..fb40c1c8ad563 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -57,7 +57,7 @@ async-io = { version = "2.0.0", optional = true } atomic-waker = { version = "1", default-features = false } concurrent-queue = { version = "2.5", default-features = false } crossbeam-utils = { version = "0.8", default-features = false, optional = true } -bitflags = "2.9" +log = "0.4" [target.'cfg(target_arch = "wasm32")'.dependencies] async-channel = "2.3.0" diff --git a/crates/bevy_tasks/README.md b/crates/bevy_tasks/README.md index 530489bfaa115..89d6960ced122 100644 --- a/crates/bevy_tasks/README.md +++ b/crates/bevy_tasks/README.md @@ -26,8 +26,8 @@ This currently applies to Wasm targets.) The determining factor for what kind of work should go in each pool is latency requirements: * For CPU-intensive work (tasks that generally spin until completion) we have a standard - [`ComputeTaskPool`] and an [`AsyncComputeTaskPool`]. Work that does not need to be completed to - present the next frame should go to the [`AsyncComputeTaskPool`]. + [`TaskPool`] and an [`AsyncTaskPool`]. Work that does not need to be completed to + present the next frame should go to the [`AsyncTaskPool`]. * For IO-intensive work (tasks that spend very little time in a "woken" state) we have an [`IoTaskPool`] whose tasks are expected to complete very quickly. Generally speaking, they should just diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index 599490bfab616..e211acf648b77 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -194,6 +194,18 @@ impl Executor { } } + /// # Safety + /// Must ensure that no other thread can call into the Executor from another + /// thread while this function is running. + pub unsafe fn set_priority_limits(&self, limits: [Option; TaskPriority::MAX]) { + for (i, limit) in limits.into_iter().enumerate() { + // SAFETY: The caller is required to ensure that no other thread can call into the Executor from another + // thread while this function is running. + unsafe { self.state.priority_limits[i].set_limit(limit) }; + log::info!("Priority {} set to {:?}", i, limit); + } + } + /// Spawns a 'static and Send task onto the executor. pub fn spawn(&'static self, future: impl Future + Send + 'static, metadata: Metadata) -> Task { // SAFETY: Both `T` and `future` are 'static. @@ -470,11 +482,11 @@ impl State { free_ids: Vec::new(), }), priority_limits: [ - CachePadded::new(AtomicSemaphore::new(None)), - CachePadded::new(AtomicSemaphore::new(None)), - CachePadded::new(AtomicSemaphore::new(None)), - CachePadded::new(AtomicSemaphore::new(None)), - CachePadded::new(AtomicSemaphore::new(None)) + CachePadded::new(AtomicSemaphore::new()), + CachePadded::new(AtomicSemaphore::new()), + CachePadded::new(AtomicSemaphore::new()), + CachePadded::new(AtomicSemaphore::new()), + CachePadded::new(AtomicSemaphore::new()) ], } } @@ -893,25 +905,32 @@ fn flush_to_local(src: &ConcurrentQueue, dst: &mut LocalQueue) { struct AtomicSemaphore { available: AtomicUsize, - limit: Option, + limit: UnsafeCell>, } +// SAFETY: The safety invaraints on `set_limit` ensure no aliasing occurs. +unsafe impl Send for AtomicSemaphore {} +// SAFETY: The safety invaraints on `set_limit` ensure no aliasing occurs. +unsafe impl Sync for AtomicSemaphore {} + impl AtomicSemaphore { - pub const fn new(limit: Option) -> Self { - match limit { - Some(limit) => Self { - available: AtomicUsize::new(limit.get()), - limit: Some(limit), - }, - None => Self { - available: AtomicUsize::new(0), - limit: None, - } + pub const fn new() -> Self { + Self { + available: AtomicUsize::new(0), + limit: UnsafeCell::new(None), } } + /// # Safety + /// Must not be called while another thread might call `acquire`. + pub unsafe fn set_limit(&self, limit: Option) { + unsafe { *self.limit.get() = limit; } + self.available.store(limit.map(NonZeroUsize::get).unwrap_or(0), Ordering::Relaxed); + } + pub fn acquire<'a>(&'a self) -> Permit<'a> { - if self.limit.is_none() { + // SAFETY: `set_limit` is not reentrant, and is required to not be called while + if unsafe { &*self.limit.get() }.is_none() { return Permit::Unrestricted; } let mut current = self.available.load(Ordering::Acquire); diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index ba90714a2b9fd..3ae202f9a5475 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -92,7 +92,6 @@ cfg::bevy_executor! { pub use iter::ParallelIterator; pub use slice::{ParallelSlice, ParallelSliceMut}; pub use task::Task; -pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; pub use futures_lite; pub use futures_lite::future::poll_once; @@ -156,7 +155,6 @@ pub mod prelude { block_on, iter::ParallelIterator, slice::{ParallelSlice, ParallelSliceMut}, - usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}, }; } @@ -179,13 +177,14 @@ pub fn available_parallelism() -> usize { }} } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(u8)] pub enum TaskPriority { BlockingIO, BlockingCompute, AsyncIO, - #[default] Compute, + #[default] + Compute, RunNow, } @@ -204,29 +203,85 @@ pub(crate) struct Metadata { pub is_send: bool, } -pub struct TaskBuilder { - priority: TaskPriority, - marker_: PhantomData<*const T> +pub struct TaskBuilder<'a, T> { + pub(crate) task_pool: &'a TaskPool, + pub(crate) priority: TaskPriority, + marker_: PhantomData<*const T>, } -impl TaskBuilder { +impl<'a, T> TaskBuilder<'a, T> { + pub(crate) fn new(task_pool: &'a TaskPool) -> Self { + Self { + task_pool, + priority: TaskPriority::default(), + marker_: PhantomData, + } + } + pub fn with_priority(mut self, priority: TaskPriority) -> Self { self.priority = priority; self } - pub fn build_metadata(self) -> Metadata { - Metadata { - priority: self.priority, + pub(crate) fn build_metadata(self) -> Metadata { + Metadata { + priority: self.priority, is_send: false, } } } -pub struct ScopeTaskBuilder<'a: 'scope, 'scope: 'env, 'env, T> { +#[derive(Clone, Copy, Default, Debug)] +pub enum ScopeTaskTarget { + #[default] + Any, + /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive + /// the provided future. The results of the future will be returned as a part of + /// [`TaskPool::scope`]'s return value. Users should generally prefer to use + /// [`Scope::spawn`] instead, unless the provided future needs to run on the scope's thread. + /// + /// For more information, see [`TaskPool::scope`]. + Scope, + + /// Spawns a scoped future onto the thread of the external thread executor. + /// This is typically the main thread. The scope *must* outlive + /// the provided future. The results of the future will be returned as a part of + /// [`TaskPool::scope`]'s return value. Users should generally prefer to use + /// [`Scope::spawn`] instead, unless the provided future needs to run on the external thread. + /// + /// For more information, see [`TaskPool::scope`]. + External, +} + +pub struct ScopeTaskBuilder<'a, 'scope, 'env: 'scope, T> { scope: &'a Scope<'scope, 'env, T>, + priority: TaskPriority, + target: ScopeTaskTarget, } -impl<'a, 'scope, 'env> ScopeTaskBuilder<'a, 'scope, 'env, T> { +impl<'a, 'scope, 'env, T> ScopeTaskBuilder<'a, 'scope, 'env, T> { + pub(crate) fn new(scope: &'a Scope<'scope, 'env, T>) -> Self { + Self { + scope, + priority: TaskPriority::default(), + target: ScopeTaskTarget::default(), + } + } -} \ No newline at end of file + pub fn with_priority(mut self, priority: TaskPriority) -> Self { + self.priority = priority; + self + } + + pub fn with_target(mut self, target: ScopeTaskTarget) -> Self { + self.target = target; + self + } + + pub(crate) fn build_metadata(self) -> Metadata { + Metadata { + priority: self.priority, + is_send: false, + } + } +} diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index f0fec26b338b3..e46a2c1ef2dde 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -1,8 +1,8 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; -use core::{future::Future, marker::PhantomData, mem, panic::AssertUnwindSafe}; +use core::{future::Future, marker::PhantomData, mem, num::NonZeroUsize, panic::AssertUnwindSafe}; use std::{sync::OnceLock, thread::{self, JoinHandle}}; -use crate::{bevy_executor::Executor, Metadata}; +use crate::{bevy_executor::Executor, Metadata, ScopeTaskBuilder, ScopeTaskTarget, TaskBuilder, TaskPriority}; use async_task::FallibleTask; use bevy_platform::sync::Arc; use concurrent_queue::ConcurrentQueue; @@ -40,6 +40,8 @@ pub struct TaskPoolBuilder { on_thread_spawn: Option>, on_thread_destroy: Option>, + + priority_limits: [Option; TaskPriority::MAX], } impl TaskPoolBuilder { @@ -61,6 +63,12 @@ impl TaskPoolBuilder { self } + /// Sets the limit of how many active threads for a given priority. + pub fn priority_limit(mut self, priority: TaskPriority, limit: Option) -> Self { + self.priority_limits[priority.to_index()] = limit.map(NonZeroUsize::new).flatten(); + self + } + /// Override the name of the threads created for the pool. If set, threads will /// be named ` ()`, i.e. `MyThreadPool (2)` pub fn thread_name(mut self, thread_name: String) -> Self { @@ -116,11 +124,12 @@ impl TaskPoolBuilder { /// Creates a new [`TaskPool`] based on the current options. pub fn build(self) -> TaskPool { - TaskPool::new_internal(self, Box::leak(Box::new(Executor::new()))) - } - - pub(crate) fn build_static(self, executor: &'static Executor) -> TaskPool { - TaskPool::new_internal(self, executor) + #[expect( + unsafe_code, + reason = "Required for priority limit initialization to be both performant and safe." + )] + // SAFETY: The box is unique and is otherwise never going to be called from any other place. + unsafe { TaskPool::new_internal(self, Box::leak(Box::new(Executor::new()))) } } } @@ -160,10 +169,27 @@ impl TaskPool { } pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static TaskPool { - TASK_POOL.get_or_init(|| f().build_static(&EXECUTOR)) + #[expect( + unsafe_code, + reason = "Required for priority limit initialization to be both performant and safe." + )] + // SAFETY: TASK_POOL is never reset and the OnceLock ensures it's only ever initialized + // once. + TASK_POOL.get_or_init(|| unsafe { Self::new_internal(f(), &EXECUTOR) }) } - fn new_internal(builder: TaskPoolBuilder, executor: &'static Executor) -> Self { + #[expect( + unsafe_code, + reason = "Required for priority limit initialization to be both performant and safe." + )] + /// # Safety + /// This should only be called once over the lifetime of the application. + unsafe fn new_internal(builder: TaskPoolBuilder, executor: &'static Executor) -> Self { + // SAFETY: The caller is required to ensure that this is only called once per application + // and no threads accessing the Executor are started until later in this very function. + // Thus it's impossible for there to be any aliasing access done here. + unsafe { executor.set_priority_limits(builder.priority_limits.clone()); } + let (shutdown_tx, shutdown_rx) = async_channel::unbounded::<()>(); let num_threads = builder @@ -393,6 +419,10 @@ impl TaskPool { } } + pub fn builder(&self) -> TaskBuilder<'_, T> { + TaskBuilder::new(self) + } + /// Spawns a static future onto the thread pool. The returned [`Task`] is a /// future that can be polled for the result. It can also be canceled and /// "detached", allowing the task to continue running even if dropped. In @@ -401,11 +431,13 @@ impl TaskPool { /// /// If the provided future is non-`Send`, [`TaskPool::spawn_local`] should /// be used instead. + /// + /// This is a shorthand for `self.builder().spawn(future)`. pub fn spawn(&self, future: impl Future + Send + 'static) -> Task where T: Send + 'static, { - Task::new(self.executor.spawn(future, Metadata::default())) + self.builder().spawn(future) } /// Spawns a static future on the thread-local async executor for the @@ -419,11 +451,13 @@ impl TaskPool { /// /// Users should generally prefer to use [`TaskPool::spawn`] instead, /// unless the provided future is not `Send`. + /// + /// This is a shorthand for `self.builder().spawn(future)`. pub fn spawn_local(&self, future: impl Future + 'static) -> Task where T: 'static, { - Task::new(self.executor.spawn_local(future, Metadata::default())) + self.builder().spawn_local(future) } pub(crate) fn try_tick_local() -> bool { @@ -462,10 +496,10 @@ pub struct Scope<'scope, 'env: 'scope, T> { } impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { - #[expect( - unsafe_code, - reason = "Executor::spawn otherwise requires 'static Futures" - )] + pub fn builder(&self) -> ScopeTaskBuilder<'_, 'scope, 'env, T> { + ScopeTaskBuilder::new(self) + } + /// Spawns a scoped future onto the thread pool. The scope *must* outlive /// the provided future. The results of the future will be returned as a part of /// [`TaskPool::scope`]'s return value. @@ -475,64 +509,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// /// For more information, see [`TaskPool::scope`]. pub fn spawn + 'scope + Send>(&self, f: Fut) { - // SAFETY: The scope call that generated this `Scope` ensures that the created - // Task does not outlive 'scope. - let task = unsafe { - self.executor - .spawn_scoped(AssertUnwindSafe(f).catch_unwind(), Metadata::default()) - .fallible() - }; - let result = self.spawned.push(task); - debug_assert!(result.is_ok()); - } - - #[expect( - unsafe_code, - reason = "ThreadSpawner::spawn otherwise requires 'static Futures" - )] - /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive - /// the provided future. The results of the future will be returned as a part of - /// [`TaskPool::scope`]'s return value. Users should generally prefer to use - /// [`Scope::spawn`] instead, unless the provided future needs to run on the scope's thread. - /// - /// For more information, see [`TaskPool::scope`]. - pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { - // SAFETY: The scope call that generated this `Scope` ensures that the created - // Task does not outlive 'scope. - let task = unsafe { - self.scope_spawner - .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) - .into_inner() - .fallible() - }; - let result = self.spawned.push(task); - debug_assert!(result.is_ok()); - } - - #[expect( - unsafe_code, - reason = "ThreadSpawner::spawn otherwise requires 'static Futures" - )] - /// Spawns a scoped future onto the thread of the external thread executor. - /// This is typically the main thread. The scope *must* outlive - /// the provided future. The results of the future will be returned as a part of - /// [`TaskPool::scope`]'s return value. Users should generally prefer to use - /// [`Scope::spawn`] instead, unless the provided future needs to run on the external thread. - /// - /// For more information, see [`TaskPool::scope`]. - pub fn spawn_on_external + 'scope + Send>(&self, f: Fut) { - // SAFETY: The scope call that generated this `Scope` ensures that the created - // Task does not outlive 'scope. - let task = unsafe { - self.external_spawner - .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) - .into_inner() - .fallible() - }; - // ConcurrentQueue only errors when closed or full, but we never - // close and use an unbounded queue, so pushing should always succeed. - let result = self.spawned.push(task); - debug_assert!(result.is_ok()); + self.builder().spawn(f); } } @@ -549,6 +526,88 @@ where } } +impl<'a, T> TaskBuilder<'a, T> { + /// Spawns a static future onto the thread pool. The returned [`Task`] is a + /// future that can be polled for the result. It can also be canceled and + /// "detached", allowing the task to continue running even if dropped. In + /// any case, the pool will execute the task even without polling by the + /// end-user. + /// + /// If the provided future is non-`Send`, [`TaskPool::spawn_local`] should + /// be used instead. + pub fn spawn(self, future: impl Future + Send + 'static) -> Task + where + T: Send + 'static, + { + Task::new(self.task_pool.executor.spawn(future, self.build_metadata())) + } + + /// Spawns a static future on the thread-local async executor for the + /// current thread. The task will run entirely on the thread the task was + /// spawned on. + /// + /// The returned [`Task`] is a future that can be polled for the + /// result. It can also be canceled and "detached", allowing the task to + /// continue running even if dropped. In any case, the pool will execute the + /// task even without polling by the end-user. + /// + /// Users should generally prefer to use [`TaskPool::spawn`] instead, + /// unless the provided future is not `Send`. + pub fn spawn_local(self, future: impl Future + 'static) -> Task + where + T: 'static, + { + Task::new(self.task_pool.executor.spawn_local(future, self.build_metadata())) + } +} + +impl<'a, 'scope, 'env, T: Send + 'scope> ScopeTaskBuilder<'a, 'scope, 'env, T> { + #[expect( + unsafe_code, + reason = "Executor::spawn and ThreadSpawner::spawn_scoped otherwise requires 'static Futures" + )] + /// Spawns a scoped future onto the thread pool. The scope *must* outlive + /// the provided future. The results of the future will be returned as a part of + /// [`TaskPool::scope`]'s return value. + /// + /// For futures that should run on the thread `scope` is called on [`Scope::spawn_on_scope`] should be used + /// instead. + /// + /// For more information, see [`TaskPool::scope`]. + pub fn spawn + 'scope + Send>(self, f: Fut) { + let task = match self.target { + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::Any => unsafe { + self.scope + .executor + .spawn_scoped(AssertUnwindSafe(f).catch_unwind(), Metadata::default()) + .fallible() + }, + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::Scope => unsafe { + self.scope + .scope_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() + .fallible() + }, + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::External => unsafe { + self.scope + .external_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() + .fallible() + }, + }; + let result = self.scope.spawned.push(task); + debug_assert!(result.is_ok()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -601,7 +660,7 @@ mod tests { start_counter.fetch_add(1, Ordering::Relaxed); barrier.clone().wait(); }) - .build_static(&EX); + .build(); last_barrier.wait(); assert_eq!(10, counter.load(Ordering::Relaxed)); } @@ -613,7 +672,7 @@ mod tests { .on_thread_destroy(move || { end_counter.fetch_sub(1, Ordering::Relaxed); }) - .build_static(&EX); + .build(); assert_eq!(10, counter.load(Ordering::Relaxed)); } assert_eq!(-10, counter.load(Ordering::Relaxed)); @@ -631,7 +690,7 @@ mod tests { .on_thread_destroy(move || { end_counter.fetch_sub(1, Ordering::Relaxed); }) - .build_static(&EX); + .build(); last_barrier.wait(); assert_eq!(-5, counter.load(Ordering::Relaxed)); } @@ -662,7 +721,10 @@ mod tests { }); } else { let count_clone = local_count.clone(); - scope.spawn_on_scope(async move { + scope + .builder() + .with_target(ScopeTaskTarget::Scope) + .spawn(async move { if *foo != 42 { panic!("not 42!?!?") } else { @@ -703,7 +765,9 @@ mod tests { }); let spawner = thread::current().id(); let inner_count_clone = count_clone.clone(); - scope.spawn_on_scope(async move { + scope.builder() + .with_target(ScopeTaskTarget::Scope) + .spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); if thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicking the @@ -778,7 +842,9 @@ mod tests { inner_count_clone.fetch_add(1, Ordering::Release); // spawning on the scope from another thread runs the futures on the scope's thread - scope.spawn_on_scope(async move { + scope.builder() + .with_target(ScopeTaskTarget::Scope) + .spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); if thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicking the diff --git a/crates/bevy_transform/src/systems.rs b/crates/bevy_transform/src/systems.rs index 2e9a113e66d8a..1781481225da1 100644 --- a/crates/bevy_transform/src/systems.rs +++ b/crates/bevy_transform/src/systems.rs @@ -252,7 +252,7 @@ mod parallel { // TODO: this implementation could be used in no_std if there are equivalents of these. use alloc::{sync::Arc, vec::Vec}; use bevy_ecs::{entity::UniqueEntityIter, prelude::*, system::lifetimeless::Read}; - use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; + use bevy_tasks::{TaskPool, TaskPoolBuilder}; use bevy_utils::Parallel; use core::sync::atomic::{AtomicI32, Ordering}; use std::sync::{ @@ -320,7 +320,7 @@ mod parallel { } // Spawn workers on the task pool to recursively propagate the hierarchy in parallel. - let task_pool = ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + let task_pool = TaskPool::get_or_init(TaskPoolBuilder::default); task_pool.scope(|s| { (1..task_pool.thread_num()) // First worker is run locally instead of the task pool. .for_each(|_| s.spawn(async { propagation_worker(&queue, &nodes) })); @@ -559,13 +559,13 @@ mod test { use bevy_app::prelude::*; use bevy_ecs::{prelude::*, world::CommandQueue}; use bevy_math::{vec3, Vec3}; - use bevy_tasks::{ComputeTaskPool, TaskPoolBuilder}; + use bevy_tasks::{TaskPool, TaskPoolBuilder}; use crate::systems::*; #[test] fn correct_parent_removed() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let offset_global_transform = |offset| GlobalTransform::from(Transform::from_xyz(offset, offset, offset)); @@ -626,7 +626,7 @@ mod test { #[test] fn did_propagate() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let mut schedule = Schedule::default(); @@ -702,7 +702,7 @@ mod test { #[test] fn correct_children() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); let mut world = World::default(); let mut schedule = Schedule::default(); @@ -783,7 +783,7 @@ mod test { #[test] fn correct_transforms_when_no_children() { let mut app = App::new(); - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); app.add_systems( Update, @@ -834,7 +834,7 @@ mod test { #[test] #[should_panic] fn panic_when_hierarchy_cycle() { - ComputeTaskPool::get_or_init(TaskPoolBuilder::default); + TaskPool::get_or_init(TaskPoolBuilder::default); // We cannot directly edit ChildOf and Children, so we use a temp world to break the // hierarchy's invariants. let mut temp = World::new(); diff --git a/examples/README.md b/examples/README.md index 51f59f0a9ebe3..aa7caa856ff56 100644 --- a/examples/README.md +++ b/examples/README.md @@ -35,59 +35,59 @@ git checkout v0.4.0 - [Examples](#examples) - [Table of Contents](#table-of-contents) -- [The Bare Minimum](#the-bare-minimum) - - [Hello, World!](#hello-world) -- [Cross-Platform Examples](#cross-platform-examples) - - [2D Rendering](#2d-rendering) - - [3D Rendering](#3d-rendering) - - [Animation](#animation) - - [Application](#application) - - [Assets](#assets) - - [Async Tasks](#async-tasks) - - [Audio](#audio) - - [Camera](#camera) - - [Dev tools](#dev-tools) - - [Diagnostics](#diagnostics) - - [ECS (Entity Component System)](#ecs-entity-component-system) - - [Embedded](#embedded) - - [Games](#games) - - [Gizmos](#gizmos) - - [Helpers](#helpers) - - [Input](#input) - - [Math](#math) - - [Movement](#movement) - - [Picking](#picking) - - [Reflection](#reflection) - - [Remote Protocol](#remote-protocol) - - [Scene](#scene) - - [Shaders](#shaders) - - [State](#state) - - [Stress Tests](#stress-tests) - - [Time](#time) - - [Tools](#tools) - - [Transforms](#transforms) - - [UI (User Interface)](#ui-user-interface) - - [Usage](#usage) - - [Window](#window) - -- [Tests](#tests) -- [Platform-Specific Examples](#platform-specific-examples) - - [Android](#android) - - [Setup](#setup) - - [Build & Run](#build--run) - - [About `libc++_shared.so`](#about-libc_sharedso) - - [Old phones](#old-phones) - - [About `cargo-apk`](#about-cargo-apk) - - [iOS](#ios) - - [Setup](#setup-1) - - [Build & Run](#build--run-1) - - [Wasm](#wasm) - - [Setup](#setup-2) - - [Build & Run](#build--run-2) - - [WebGL2 and WebGPU](#webgl2-and-webgpu) - - [Audio in the browsers](#audio-in-the-browsers) - - [Optimizing](#optimizing) - - [Loading Assets](#loading-assets) + - [The Bare Minimum](#the-bare-minimum) + - [Hello, World!](#hello-world) + - [Cross-Platform Examples](#cross-platform-examples) + - [2D Rendering](#2d-rendering) + - [3D Rendering](#3d-rendering) + - [Animation](#animation) + - [Application](#application) + - [Assets](#assets) + - [Async Tasks](#async-tasks) + - [Audio](#audio) + - [Camera](#camera) + - [Dev tools](#dev-tools) + - [Diagnostics](#diagnostics) + - [ECS (Entity Component System)](#ecs-entity-component-system) + - [Embedded](#embedded) + - [Games](#games) + - [Gizmos](#gizmos) + - [Helpers](#helpers) + - [Input](#input) + - [Math](#math) + - [Movement](#movement) + - [Picking](#picking) + - [Reflection](#reflection) + - [Remote Protocol](#remote-protocol) + - [Scene](#scene) + - [Shaders](#shaders) + - [State](#state) + - [Stress Tests](#stress-tests) + - [Time](#time) + - [Tools](#tools) + - [Transforms](#transforms) + - [UI (User Interface)](#ui-user-interface) + - [Usage](#usage) + - [Window](#window) + - [Tests](#tests) + - [Platform-Specific Examples](#platform-specific-examples) + - [Android](#android) + - [Setup](#setup) + - [Build \& Run](#build--run) + - [About `libc++_shared.so`](#about-libc_sharedso) + - [Debugging](#debugging) + - [Old phones](#old-phones) + - [About `cargo-apk`](#about-cargo-apk) + - [iOS](#ios) + - [Setup](#setup-1) + - [Build \& Run](#build--run-1) + - [Wasm](#wasm) + - [Setup](#setup-2) + - [Build \& Run](#build--run-2) + - [WebGL2 and WebGPU](#webgl2-and-webgpu) + - [Audio in the browsers](#audio-in-the-browsers) + - [Optimizing](#optimizing) + - [Loading Assets](#loading-assets) ## The Bare Minimum @@ -265,7 +265,7 @@ Example | Description Example | Description --- | --- -[Async Compute](../examples/async_tasks/async_compute.rs) | How to use `AsyncComputeTaskPool` to complete longer running tasks +[Blocking Compute](../examples/async_tasks/blocking_compute.rs) | How to use `TaskPool` to complete longer running tasks [External Source of Data on an External Thread](../examples/async_tasks/external_source_external_thread.rs) | How to use an external thread to run an infinite task and communicate with a channel ### Audio diff --git a/examples/animation/animation_graph.rs b/examples/animation/animation_graph.rs index e511a1bb7faa4..b5f4eb7762c6f 100644 --- a/examples/animation/animation_graph.rs +++ b/examples/animation/animation_graph.rs @@ -16,7 +16,10 @@ use argh::FromArgs; #[cfg(not(target_arch = "wasm32"))] use { - bevy::{asset::io::file::FileAssetReader, tasks::IoTaskPool}, + bevy::{ + asset::io::file::FileAssetReader, + tasks::{TaskPool, TaskPriority}, + }, ron::ser::PrettyConfig, std::{fs::File, path::Path}, }; @@ -176,9 +179,12 @@ fn setup_assets_programmatically( // If asked to save, do so. #[cfg(not(target_arch = "wasm32"))] if _save { + use bevy::tasks::TaskPriority; + let animation_graph = animation_graph.clone(); - IoTaskPool::get() + TaskPool::get() + .with_priority(TaskPriority::BlockingIO) .spawn(async move { use std::io::Write; diff --git a/examples/asset/multi_asset_sync.rs b/examples/asset/multi_asset_sync.rs index 83add4ba3c016..86d1f611fed42 100644 --- a/examples/asset/multi_asset_sync.rs +++ b/examples/asset/multi_asset_sync.rs @@ -9,7 +9,7 @@ use std::{ }, }; -use bevy::{gltf::Gltf, prelude::*, tasks::AsyncComputeTaskPool}; +use bevy::{gltf::Gltf, prelude::*, tasks::TaskPool}; use event_listener::Event; use futures_lite::Future; @@ -29,7 +29,7 @@ fn main() { // This approach polls a value in a system. .add_systems(Update, wait_on_load.run_if(assets_loaded)) // This showcases how to wait for assets using async - // by spawning a `Future` in `AsyncComputeTaskPool`. + // by spawning a `Future` in `TaskPool`. .add_systems( Update, get_async_loading_state.run_if(in_state(LoadingState::Loading)), @@ -158,7 +158,7 @@ fn setup_assets(mut commands: Commands, asset_server: Res) { commands.insert_resource(AsyncLoadingState(loading_state.clone())); // await the `AssetBarrierFuture`. - AsyncComputeTaskPool::get() + TaskPool::get() .spawn(async move { future.await; // Notify via `AsyncLoadingState` diff --git a/examples/async_tasks/async_compute.rs b/examples/async_tasks/async_compute.rs index 7e24525cb6230..7dd02c3262d93 100644 --- a/examples/async_tasks/async_compute.rs +++ b/examples/async_tasks/async_compute.rs @@ -1,10 +1,10 @@ -//! This example shows how to use the ECS and the [`AsyncComputeTaskPool`] +//! This example shows how to use the ECS and the [`TaskPool`] //! to spawn, poll, and complete tasks across systems and system ticks. use bevy::{ ecs::{system::SystemState, world::CommandQueue}, prelude::*, - tasks::{block_on, futures_lite::future, AsyncComputeTaskPool, Task}, + tasks::{block_on, futures_lite::future, TaskPool, Task}, }; use rand::Rng; use std::time::Duration; @@ -50,15 +50,17 @@ struct ComputeTransform(Task); /// system, [`handle_tasks`], will poll the spawned tasks on subsequent /// frames/ticks, and use the results to spawn cubes fn spawn_tasks(mut commands: Commands) { - let thread_pool = AsyncComputeTaskPool::get(); + let thread_pool = TaskPool::get(); for x in 0..NUM_CUBES { for y in 0..NUM_CUBES { for z in 0..NUM_CUBES { - // Spawn new task on the AsyncComputeTaskPool; the task will be + // Spawn new task on the TaskPool; the task will be // executed in the background, and the Task future returned by // spawn() can be used to poll for the result let entity = commands.spawn_empty().id(); - let task = thread_pool.spawn(async move { + let task = thread_pool + .with_priority(TaskPriority::BlockingCompute) + .spawn(async move { let duration = Duration::from_secs_f32(rand::rng().random_range(0.05..5.0)); // Pretend this is a time-intensive function. :) diff --git a/examples/ecs/parallel_query.rs b/examples/ecs/parallel_query.rs index 6ebd28ea5065e..245380935e64f 100644 --- a/examples/ecs/parallel_query.rs +++ b/examples/ecs/parallel_query.rs @@ -27,7 +27,7 @@ fn spawn_system(mut commands: Commands, asset_server: Res) { // Move sprites according to their velocity fn move_system(mut sprites: Query<(&mut Transform, &Velocity)>) { // Compute the new location of each sprite in parallel on the - // ComputeTaskPool + // TaskPool // // This example is only for demonstrative purposes. Using a // ParallelIterator for an inexpensive operation like addition on only 128 diff --git a/examples/scene/scene.rs b/examples/scene/scene.rs index 2fba727a82f9a..95317c64f20a1 100644 --- a/examples/scene/scene.rs +++ b/examples/scene/scene.rs @@ -20,11 +20,11 @@ //! # Note on working with files //! //! The saving behavior uses the standard filesystem APIs, which are blocking, so it -//! utilizes a thread pool (`IoTaskPool`) to avoid stalling the main thread. This +//! utilizes a thread pool (`TaskPool`) to avoid stalling the main thread. This //! won't work on WASM because WASM typically doesn't have direct filesystem access. //! -use bevy::{asset::LoadState, prelude::*, tasks::IoTaskPool}; +use bevy::{asset::LoadState, prelude::*, tasks::TaskPool}; use core::time::Duration; use std::{fs::File, io::Write}; @@ -195,7 +195,8 @@ fn save_scene_system(world: &mut World) { // // This can't work in Wasm as there is no filesystem access. #[cfg(not(target_arch = "wasm32"))] - IoTaskPool::get() + TaskPool::get() + .with_priority(TaskPriority::BlcokingIO) .spawn(async move { // Write the scene RON data to file File::create(format!("assets/{NEW_SCENE_FILE_PATH}")) From 47ad004caab71d176c55593cf8471b263bb5ed31 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 21:40:42 -0700 Subject: [PATCH 66/68] Docs and other assorted fixe --- crates/bevy_app/src/task_pool_plugin.rs | 2 +- crates/bevy_tasks/src/bevy_executor.rs | 21 ++++-- crates/bevy_tasks/src/lib.rs | 51 +++++++++++-- crates/bevy_tasks/src/slice.rs | 6 +- crates/bevy_tasks/src/task_pool.rs | 39 ++++++++-- examples/animation/animation_graph.rs | 1 + .../{async_compute.rs => blocking_compute.rs} | 73 ++++++++++--------- examples/scene/scene.rs | 1 + 8 files changed, 132 insertions(+), 62 deletions(-) rename examples/async_tasks/{async_compute.rs => blocking_compute.rs} (68%) diff --git a/crates/bevy_app/src/task_pool_plugin.rs b/crates/bevy_app/src/task_pool_plugin.rs index a65b8cfa321c0..d13e30ef9e077 100644 --- a/crates/bevy_app/src/task_pool_plugin.rs +++ b/crates/bevy_app/src/task_pool_plugin.rs @@ -212,7 +212,7 @@ mod tests { app.add_plugins(TaskPoolPlugin::default()); let (tx, rx) = crossbeam_channel::unbounded(); - TaskPool::get() + TaskPool::get_or_init(Default::default) .spawn_local(async move { tx.send(()).unwrap(); }) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index e211acf648b77..c363912e4c56f 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -198,11 +198,16 @@ impl Executor { /// Must ensure that no other thread can call into the Executor from another /// thread while this function is running. pub unsafe fn set_priority_limits(&self, limits: [Option; TaskPriority::MAX]) { - for (i, limit) in limits.into_iter().enumerate() { + let executor_limits = self.state.priority_limits.iter(); + for (i, (limit, executor_limit)) in limits.into_iter().zip(executor_limits).enumerate() { // SAFETY: The caller is required to ensure that no other thread can call into the Executor from another // thread while this function is running. - unsafe { self.state.priority_limits[i].set_limit(limit) }; - log::info!("Priority {} set to {:?}", i, limit); + unsafe { executor_limit.set_limit(limit) }; + if let Some(limit) = limit { + log::debug!("{:?} tasks now limited to {:?} simultaneous tasks.", TaskPriority::from_index(i).unwrap(), limit); + } else { + log::debug!("{:?} are now not limited.", TaskPriority::from_index(i).unwrap()); + } } } @@ -1094,7 +1099,7 @@ mod test { let s: String = "hello".into(); // SAFETY: We make sure that the task does not outlive the borrow on `s`. - let task: Task<&str> = unsafe { EX.spawn_scoped(async { &*s }) }; + let task: Task<&str, Metadata> = unsafe { EX.spawn_scoped(async { &*s }, Metadata::default()) }; future::block_on(EX.run(async { for _ in 0..10 { future::yield_now().await; @@ -1144,25 +1149,25 @@ mod test { #[test] fn smoke() { - do_run(|ex| async move { ex.spawn(async {}).await }); + do_run(|ex| async move { ex.spawn(async {}, Metadata::default()).await }); } #[test] fn yield_now() { - do_run(|ex| async move { ex.spawn(future::yield_now()).await }); + do_run(|ex| async move { ex.spawn(future::yield_now(), Metadata::default()).await }); } #[test] fn timer() { do_run(|ex| async move { - ex.spawn(async_io::Timer::after(Duration::from_millis(5))) + ex.spawn(async_io::Timer::after(Duration::from_millis(5)), Metadata::default()) .await; }); } #[test] fn test_panic_propagation() { - let task = EX.spawn(async { panic!("should be caught by the task") }); + let task = EX.spawn(async { panic!("should be caught by the task") }, Metadata::default()); // Running the executor should not panic. future::block_on(EX.run(async { diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 3ae202f9a5475..408ebd1bcb81e 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -155,6 +155,7 @@ pub mod prelude { block_on, iter::ParallelIterator, slice::{ParallelSlice, ParallelSliceMut}, + TaskPool, }; } @@ -177,14 +178,33 @@ pub fn available_parallelism() -> usize { }} } +/// The priority of a task scheduled onto the [`TaskPool`]. +/// +/// Using [`TaskPoolBuilder::priority_limit`], the `TaskPool` will limit how many tasks can +/// execute in parallel. This is *not* a limit on the number of tasks that can be scheduled +/// onto the task pool, but rather the number of them that can execute in parallel, and is +/// used to avoid starving out higher priority groups of parallelism. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(u8)] pub enum TaskPriority { + /// Intended for blocking IO operations (e.g. `File::read`). BlockingIO, + /// Intended for blocking CPU-bound tasks (e.g. shader compilation, building terrain) BlockingCompute, + /// Intended for non-blocking async IO (e.g. HTTP servers/clients, network IO, io-uring file IO). + /// These jobs generally should do very little compute bound work and then yield immeidately upon + /// there being no more work to do. AsyncIO, + /// Intended for shortlived CPU-bound jobs. These jobs are expected to do a small amount of work + /// and quickly terminate. This is the default. #[default] Compute, + /// Intended for shortlived CPU-bound jobs with tight realtime requirements. These jobs are expected + /// to do a small amount of work and quickly terminate or yield. + /// + /// Unlike the other priorities, this group forces tasks to immediately schedule onto the thread + /// where the task is awoken, and will start as soon as the currently executing task terminates + /// or yields. RunNow, } @@ -195,6 +215,18 @@ impl TaskPriority { fn to_index(self) -> usize { self as u8 as usize } + + #[inline] + fn from_index(index: usize) -> Option { + Some(match index { + 0 => Self::BlockingIO, + 1 => Self::BlockingCompute, + 2 => Self::AsyncIO, + 3 => Self::Compute, + 4 => Self::RunNow, + _ => return None, + }) + } } #[derive(Debug, Default)] @@ -203,6 +235,7 @@ pub(crate) struct Metadata { pub is_send: bool, } +/// A builder for a [`Task`] to be scheduled onto a [`TaskPool`]. pub struct TaskBuilder<'a, T> { pub(crate) task_pool: &'a TaskPool, pub(crate) priority: TaskPriority, @@ -218,6 +251,7 @@ impl<'a, T> TaskBuilder<'a, T> { } } + /// Sets the priority of the spawned task. See [`TaskPriority`] for more details. pub fn with_priority(mut self, priority: TaskPriority) -> Self { self.priority = priority; self @@ -231,28 +265,26 @@ impl<'a, T> TaskBuilder<'a, T> { } } +/// Configuration for which thread to schedule a [`Task`] within a [`Scope`] onto. #[derive(Clone, Copy, Default, Debug)] pub enum ScopeTaskTarget { + /// Spawns the future onto any thread intthe [`TaskPool`]. #[default] Any, - /// Spawns a scoped future onto the thread the scope is run on. The scope *must* outlive - /// the provided future. The results of the future will be returned as a part of - /// [`TaskPool::scope`]'s return value. Users should generally prefer to use - /// [`Scope::spawn`] instead, unless the provided future needs to run on the scope's thread. + + /// Spawns a scoped future onto the thread the scope is run on. /// /// For more information, see [`TaskPool::scope`]. Scope, /// Spawns a scoped future onto the thread of the external thread executor. - /// This is typically the main thread. The scope *must* outlive - /// the provided future. The results of the future will be returned as a part of - /// [`TaskPool::scope`]'s return value. Users should generally prefer to use - /// [`Scope::spawn`] instead, unless the provided future needs to run on the external thread. + /// This is typically the main thread. /// /// For more information, see [`TaskPool::scope`]. External, } +/// A builder for a [`Task`] within a [`Scope`]. pub struct ScopeTaskBuilder<'a, 'scope, 'env: 'scope, T> { scope: &'a Scope<'scope, 'env, T>, priority: TaskPriority, @@ -268,11 +300,14 @@ impl<'a, 'scope, 'env, T> ScopeTaskBuilder<'a, 'scope, 'env, T> { } } + /// Sets the priority of the spawned task. See [`TaskPriority`] for more details. pub fn with_priority(mut self, priority: TaskPriority) -> Self { self.priority = priority; self } + /// Sets the target for which thread to schedule the spawned task onto. + /// See [`ScopeTaskTarget`] for more details. pub fn with_target(mut self, target: ScopeTaskTarget) -> Self { self.target = target; self diff --git a/crates/bevy_tasks/src/slice.rs b/crates/bevy_tasks/src/slice.rs index a705314a34502..512bea0421b54 100644 --- a/crates/bevy_tasks/src/slice.rs +++ b/crates/bevy_tasks/src/slice.rs @@ -220,7 +220,7 @@ mod tests { #[test] fn test_par_chunks_map() { let v = vec![42; 1000]; - let task_pool = TaskPool::new(); + let task_pool = TaskPoolBuilder::new().build(); let outputs = v.par_splat_map(&task_pool, None, |_, numbers| -> i32 { numbers.iter().sum() }); @@ -236,7 +236,7 @@ mod tests { #[test] fn test_par_chunks_map_mut() { let mut v = vec![42; 1000]; - let task_pool = TaskPool::new(); + let task_pool = TaskPoolBuilder::new().build(); let outputs = v.par_splat_map_mut(&task_pool, None, |_, numbers| -> i32 { for number in numbers.iter_mut() { @@ -257,7 +257,7 @@ mod tests { #[test] fn test_par_chunks_map_index() { let v = vec![1; 1000]; - let task_pool = TaskPool::new(); + let task_pool = TaskPoolBuilder::new().build(); let outputs = v.par_chunk_map(&task_pool, 100, |index, numbers| -> i32 { numbers.iter().sum::() * index as i32 }); diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index e46a2c1ef2dde..31c1cd703577f 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -160,14 +160,22 @@ impl TaskPool { self.executor.current_thread_spawner() } + /// Attempts to get the global [`TaskPool`] instance, or returns `None` if it is not initialized. pub fn try_get() -> Option<&'static TaskPool> { TASK_POOL.get() } + /// Gets the global [`TaskPool`] instance. + /// + /// # Panics + /// + /// Panics if the global instance has not been initialized yet. pub fn get() -> &'static TaskPool { - Self::get_or_init(Default::default) + Self::try_get() + .expect("The TaskPool has not been initialized yet. Please call TaskPool::get_or_init beforehand.") } + /// Gets the global [`TaskPool`] instance, or initializes it with `f``. pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static TaskPool { #[expect( unsafe_code, @@ -419,6 +427,20 @@ impl TaskPool { } } + /// Creates a builder for a new [`Task`] to schedule onto the [`TaskPool`].k + /// + /// # Example + /// + /// ```norun + /// # async fn my_cool_task() {} + /// # use bevy_tasks::{TaskPool, TaskPriority}; + /// let task_pool = TaskPool::get() + /// let task = task_pool.builder() + /// .with_priority(TaskPriority::BlockingIO) + /// .spawn(async { + /// my_cool_task + /// }); + /// ``` pub fn builder(&self) -> TaskBuilder<'_, T> { TaskBuilder::new(self) } @@ -496,6 +518,9 @@ pub struct Scope<'scope, 'env: 'scope, T> { } impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { + /// Creates a builder to spawn a scoped future to schedule onto the [`TaskPool`]. + /// The scope *must* outlive the provided future. The results of the future will + /// be returned as a part of [`TaskPool::scope`]'s return value. pub fn builder(&self) -> ScopeTaskBuilder<'_, 'scope, 'env, T> { ScopeTaskBuilder::new(self) } @@ -504,10 +529,12 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// the provided future. The results of the future will be returned as a part of /// [`TaskPool::scope`]'s return value. /// - /// For futures that should run on the thread `scope` is called on [`Scope::spawn_on_scope`] should be used - /// instead. + /// For futures that should run on the thread `scope` is called on [`Scope::builder`] should + /// be used instead, with [`ScopeTaskBuilder::with_target``] to target specific thread. /// /// For more information, see [`TaskPool::scope`]. + /// + /// This is a shorthand for `scope.builder().spawn(f)`. pub fn spawn + 'scope + Send>(&self, f: Fut) { self.builder().spawn(f); } @@ -755,10 +782,9 @@ mod tests { for _ in 0..100 { let inner_barrier = barrier.clone(); let count_clone = count.clone(); - let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); thread::spawn(move || { - inner_pool.scope(|scope| { + pool.scope(|scope| { let inner_count_clone = count_clone.clone(); scope.spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); @@ -832,10 +858,9 @@ mod tests { for _ in 0..100 { let inner_barrier = barrier.clone(); let count_clone = count.clone(); - let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); thread::spawn(move || { - inner_pool.scope(|scope| { + pool.scope(|scope| { let spawner = thread::current().id(); let inner_count_clone = count_clone.clone(); scope.spawn(async move { diff --git a/examples/animation/animation_graph.rs b/examples/animation/animation_graph.rs index b5f4eb7762c6f..cd7bc001cccd4 100644 --- a/examples/animation/animation_graph.rs +++ b/examples/animation/animation_graph.rs @@ -184,6 +184,7 @@ fn setup_assets_programmatically( let animation_graph = animation_graph.clone(); TaskPool::get() + .builder() .with_priority(TaskPriority::BlockingIO) .spawn(async move { use std::io::Write; diff --git a/examples/async_tasks/async_compute.rs b/examples/async_tasks/blocking_compute.rs similarity index 68% rename from examples/async_tasks/async_compute.rs rename to examples/async_tasks/blocking_compute.rs index 7dd02c3262d93..bd1d5f88e3a03 100644 --- a/examples/async_tasks/async_compute.rs +++ b/examples/async_tasks/blocking_compute.rs @@ -4,7 +4,7 @@ use bevy::{ ecs::{system::SystemState, world::CommandQueue}, prelude::*, - tasks::{block_on, futures_lite::future, TaskPool, Task}, + tasks::{block_on, futures_lite::future, Task, TaskPool}, }; use rand::Rng; use std::time::Duration; @@ -59,44 +59,47 @@ fn spawn_tasks(mut commands: Commands) { // spawn() can be used to poll for the result let entity = commands.spawn_empty().id(); let task = thread_pool + .builder() .with_priority(TaskPriority::BlockingCompute) .spawn(async move { - let duration = Duration::from_secs_f32(rand::rng().random_range(0.05..5.0)); - - // Pretend this is a time-intensive function. :) - async_std::task::sleep(duration).await; - - // Such hard work, all done! - let transform = Transform::from_xyz(x as f32, y as f32, z as f32); - let mut command_queue = CommandQueue::default(); - - // we use a raw command queue to pass a FnOnce(&mut World) back to be - // applied in a deferred manner. - command_queue.push(move |world: &mut World| { - let (box_mesh_handle, box_material_handle) = { - let mut system_state = SystemState::<( - Res, - Res, - )>::new(world); - let (box_mesh_handle, box_material_handle) = - system_state.get_mut(world); - - (box_mesh_handle.clone(), box_material_handle.clone()) - }; - - world - .entity_mut(entity) - // Add our new `Mesh3d` and `MeshMaterial3d` to our tagged entity - .insert(( - Mesh3d(box_mesh_handle), - MeshMaterial3d(box_material_handle), - transform, - )); + let duration = Duration::from_secs_f32(rand::rng().random_range(0.05..5.0)); + + // Pretend this is a time-intensive function. :) + async_std::task::sleep(duration).await; + + // Such hard work, all done! + let transform = Transform::from_xyz(x as f32, y as f32, z as f32); + let mut command_queue = CommandQueue::default(); + + // we use a raw command queue to pass a FnOnce(&mut World) back to be + // applied in a deferred manner. + command_queue.push(move |world: &mut World| { + let (box_mesh_handle, box_material_handle) = { + let mut system_state = SystemState::<( + Res, + Res, + )>::new( + world + ); + let (box_mesh_handle, box_material_handle) = + system_state.get_mut(world); + + (box_mesh_handle.clone(), box_material_handle.clone()) + }; + + world + .entity_mut(entity) + // Add our new `Mesh3d` and `MeshMaterial3d` to our tagged entity + .insert(( + Mesh3d(box_mesh_handle), + MeshMaterial3d(box_material_handle), + transform, + )); + }); + + command_queue }); - command_queue - }); - // Add our new task as a component commands.entity(entity).insert(ComputeTransform(task)); } diff --git a/examples/scene/scene.rs b/examples/scene/scene.rs index 95317c64f20a1..1778da0956427 100644 --- a/examples/scene/scene.rs +++ b/examples/scene/scene.rs @@ -196,6 +196,7 @@ fn save_scene_system(world: &mut World) { // This can't work in Wasm as there is no filesystem access. #[cfg(not(target_arch = "wasm32"))] TaskPool::get() + .builder() .with_priority(TaskPriority::BlcokingIO) .spawn(async move { // Write the scene RON data to file From 88ed402404c2aa8e726255539bbb2e781a03794a Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 22:32:58 -0700 Subject: [PATCH 67/68] more CI fix attempts --- crates/bevy_tasks/src/bevy_executor.rs | 5 +- crates/bevy_tasks/src/task_pool.rs | 13 ++- examples/README.md | 106 ++++++++++++------------- 3 files changed, 65 insertions(+), 59 deletions(-) diff --git a/crates/bevy_tasks/src/bevy_executor.rs b/crates/bevy_tasks/src/bevy_executor.rs index c363912e4c56f..ce91e4e98a0b3 100644 --- a/crates/bevy_tasks/src/bevy_executor.rs +++ b/crates/bevy_tasks/src/bevy_executor.rs @@ -929,6 +929,7 @@ impl AtomicSemaphore { /// # Safety /// Must not be called while another thread might call `acquire`. pub unsafe fn set_limit(&self, limit: Option) { + // SAFETY: The caller must make sure that this does not alias. unsafe { *self.limit.get() = limit; } self.available.store(limit.map(NonZeroUsize::get).unwrap_or(0), Ordering::Relaxed); } @@ -960,8 +961,8 @@ enum Permit<'a> { impl<'a> Drop for Permit<'a> { fn drop(&mut self) { - if let Permit::Held(sempahore) = self { - sempahore.available.fetch_add(1, Ordering::AcqRel); + if let Permit::Held(semaphore) = self { + semaphore.available.fetch_add(1, Ordering::AcqRel); } } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 31c1cd703577f..b1b965e7b2451 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -65,7 +65,7 @@ impl TaskPoolBuilder { /// Sets the limit of how many active threads for a given priority. pub fn priority_limit(mut self, priority: TaskPriority, limit: Option) -> Self { - self.priority_limits[priority.to_index()] = limit.map(NonZeroUsize::new).flatten(); + self.priority_limits[priority.to_index()] = limit.and_then(NonZeroUsize::new); self } @@ -175,7 +175,7 @@ impl TaskPool { .expect("The TaskPool has not been initialized yet. Please call TaskPool::get_or_init beforehand.") } - /// Gets the global [`TaskPool`] instance, or initializes it with `f``. + /// Gets the global [`TaskPool`] instance, or initializes it with `f`. pub fn get_or_init(f: impl FnOnce() -> TaskPoolBuilder) -> &'static TaskPool { #[expect( unsafe_code, @@ -186,6 +186,11 @@ impl TaskPool { TASK_POOL.get_or_init(|| unsafe { Self::new_internal(f(), &EXECUTOR) }) } + /// Create a TaskPool with the default configuration. + pub fn new() -> Self { + TaskPoolBuilder::new().build() + } + #[expect( unsafe_code, reason = "Required for priority limit initialization to be both performant and safe." @@ -196,7 +201,7 @@ impl TaskPool { // SAFETY: The caller is required to ensure that this is only called once per application // and no threads accessing the Executor are started until later in this very function. // Thus it's impossible for there to be any aliasing access done here. - unsafe { executor.set_priority_limits(builder.priority_limits.clone()); } + unsafe { executor.set_priority_limits(builder.priority_limits); } let (shutdown_tx, shutdown_rx) = async_channel::unbounded::<()>(); @@ -530,7 +535,7 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// [`TaskPool::scope`]'s return value. /// /// For futures that should run on the thread `scope` is called on [`Scope::builder`] should - /// be used instead, with [`ScopeTaskBuilder::with_target``] to target specific thread. + /// be used instead, with [`ScopeTaskBuilder::with_target`] to target specific thread. /// /// For more information, see [`TaskPool::scope`]. /// diff --git a/examples/README.md b/examples/README.md index aa7caa856ff56..7da8f9af96489 100644 --- a/examples/README.md +++ b/examples/README.md @@ -35,59 +35,59 @@ git checkout v0.4.0 - [Examples](#examples) - [Table of Contents](#table-of-contents) - - [The Bare Minimum](#the-bare-minimum) - - [Hello, World!](#hello-world) - - [Cross-Platform Examples](#cross-platform-examples) - - [2D Rendering](#2d-rendering) - - [3D Rendering](#3d-rendering) - - [Animation](#animation) - - [Application](#application) - - [Assets](#assets) - - [Async Tasks](#async-tasks) - - [Audio](#audio) - - [Camera](#camera) - - [Dev tools](#dev-tools) - - [Diagnostics](#diagnostics) - - [ECS (Entity Component System)](#ecs-entity-component-system) - - [Embedded](#embedded) - - [Games](#games) - - [Gizmos](#gizmos) - - [Helpers](#helpers) - - [Input](#input) - - [Math](#math) - - [Movement](#movement) - - [Picking](#picking) - - [Reflection](#reflection) - - [Remote Protocol](#remote-protocol) - - [Scene](#scene) - - [Shaders](#shaders) - - [State](#state) - - [Stress Tests](#stress-tests) - - [Time](#time) - - [Tools](#tools) - - [Transforms](#transforms) - - [UI (User Interface)](#ui-user-interface) - - [Usage](#usage) - - [Window](#window) - - [Tests](#tests) - - [Platform-Specific Examples](#platform-specific-examples) - - [Android](#android) - - [Setup](#setup) - - [Build \& Run](#build--run) - - [About `libc++_shared.so`](#about-libc_sharedso) - - [Debugging](#debugging) - - [Old phones](#old-phones) - - [About `cargo-apk`](#about-cargo-apk) - - [iOS](#ios) - - [Setup](#setup-1) - - [Build \& Run](#build--run-1) - - [Wasm](#wasm) - - [Setup](#setup-2) - - [Build \& Run](#build--run-2) - - [WebGL2 and WebGPU](#webgl2-and-webgpu) - - [Audio in the browsers](#audio-in-the-browsers) - - [Optimizing](#optimizing) - - [Loading Assets](#loading-assets) +- [The Bare Minimum](#the-bare-minimum) + - [Hello, World!](#hello-world) +- [Cross-Platform Examples](#cross-platform-examples) + - [2D Rendering](#2d-rendering) + - [3D Rendering](#3d-rendering) + - [Animation](#animation) + - [Application](#application) + - [Assets](#assets) + - [Async Tasks](#async-tasks) + - [Audio](#audio) + - [Camera](#camera) + - [Dev tools](#dev-tools) + - [Diagnostics](#diagnostics) + - [ECS (Entity Component System)](#ecs-entity-component-system) + - [Embedded](#embedded) + - [Games](#games) + - [Gizmos](#gizmos) + - [Helpers](#helpers) + - [Input](#input) + - [Math](#math) + - [Movement](#movement) + - [Picking](#picking) + - [Reflection](#reflection) + - [Remote Protocol](#remote-protocol) + - [Scene](#scene) + - [Shaders](#shaders) + - [State](#state) + - [Stress Tests](#stress-tests) + - [Time](#time) + - [Tools](#tools) + - [Transforms](#transforms) + - [UI (User Interface)](#ui-user-interface) + - [Usage](#usage) + - [Window](#window) + +- [Tests](#tests) +- [Platform-Specific Examples](#platform-specific-examples) + - [Android](#android) + - [Setup](#setup) + - [Build & Run](#build--run) + - [About `libc++_shared.so`](#about-libc_sharedso) + - [Old phones](#old-phones) + - [About `cargo-apk`](#about-cargo-apk) + - [iOS](#ios) + - [Setup](#setup-1) + - [Build & Run](#build--run-1) + - [Wasm](#wasm) + - [Setup](#setup-2) + - [Build & Run](#build--run-2) + - [WebGL2 and WebGPU](#webgl2-and-webgpu) + - [Audio in the browsers](#audio-in-the-browsers) + - [Optimizing](#optimizing) + - [Loading Assets](#loading-assets) ## The Bare Minimum From 5fc308781b1f6ec2a2cdd7c5b64c6965e1961576 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 21 Aug 2025 23:12:27 -0700 Subject: [PATCH 68/68] Fix tests --- crates/bevy_app/src/task_pool_plugin.rs | 2 +- crates/bevy_render/src/pipelined_rendering.rs | 4 +- .../src/render_resource/pipeline_cache.rs | 4 +- crates/bevy_tasks/README.md | 12 +- crates/bevy_tasks/src/lib.rs | 2 +- .../src/single_threaded_task_pool.rs | 118 ++++++++++++++++-- crates/bevy_tasks/src/task_pool.rs | 18 +-- 7 files changed, 128 insertions(+), 32 deletions(-) diff --git a/crates/bevy_app/src/task_pool_plugin.rs b/crates/bevy_app/src/task_pool_plugin.rs index d13e30ef9e077..9b889719ba90c 100644 --- a/crates/bevy_app/src/task_pool_plugin.rs +++ b/crates/bevy_app/src/task_pool_plugin.rs @@ -21,7 +21,7 @@ cfg_if::cfg_if! { } } -/// Setup of default task pools: [`AsyncTaskPool`], [`TaskPool`], [`IoTaskPool`]. +/// Setup of the default task pool: [`TaskPool`]. #[derive(Default)] pub struct TaskPoolPlugin { /// Options for the [`TaskPool`](bevy_tasks::TaskPool) created at application start. diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index e3f2a1826f44e..f837d71121896 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -150,10 +150,10 @@ impl Plugin for PipelinedRenderingPlugin { #[cfg(feature = "trace")] let _span = tracing::info_span!("render thread").entered(); - let compute_task_pool = TaskPool::get(); + let task_pool = TaskPool::get(); loop { // run a scope here to allow main world to use this thread while it's waiting for the render app - let sent_app = compute_task_pool + let sent_app = task_pool .scope(|s| { s.spawn(async { app_to_render_receiver.recv().await }); }) diff --git a/crates/bevy_render/src/render_resource/pipeline_cache.rs b/crates/bevy_render/src/render_resource/pipeline_cache.rs index 56e416cdef22c..187430b5861e7 100644 --- a/crates/bevy_render/src/render_resource/pipeline_cache.rs +++ b/crates/bevy_render/src/render_resource/pipeline_cache.rs @@ -16,7 +16,7 @@ use bevy_shader::{ CachedPipelineId, PipelineCacheError, Shader, ShaderCache, ShaderCacheSource, ShaderDefVal, ValidateShader, }; -use bevy_tasks::{Task, TaskPool, TaskPriority}; +use bevy_tasks::Task; use bevy_utils::default; use core::{future::Future, hash::Hash, mem}; use std::sync::{Mutex, PoisonError}; @@ -806,6 +806,8 @@ fn create_pipeline_task( task: impl Future> + Send + 'static, sync: bool, ) -> CachedPipelineState { + use bevy_tasks::{TaskPool, TaskPriority}; + if !sync { return CachedPipelineState::Creating( TaskPool::get() diff --git a/crates/bevy_tasks/README.md b/crates/bevy_tasks/README.md index 89d6960ced122..2c18b8861623f 100644 --- a/crates/bevy_tasks/README.md +++ b/crates/bevy_tasks/README.md @@ -20,18 +20,18 @@ It is based on a fork of [`async-executor`][async-executor], a lightweight execu ## Usage In order to be able to optimize task execution in multi-threaded environments, -bevy provides three different thread pools via which tasks of different kinds can be spawned. +Bevy supports a thread pool via which tasks of different priorities can be spawned. (The same API is used in single-threaded environments, even if execution is limited to a single thread. This currently applies to Wasm targets.) -The determining factor for what kind of work should go in each pool is latency requirements: +The determining factor for how work is prioritized based on latency requirements: * For CPU-intensive work (tasks that generally spin until completion) we have a standard - [`TaskPool`] and an [`AsyncTaskPool`]. Work that does not need to be completed to - present the next frame should go to the [`AsyncTaskPool`]. + `Compute` priority, the default. Work that does not need to be completed to present the + next frame be set to the `BlockingCompute` priority. * For IO-intensive work (tasks that spend very little time in a "woken" state) we have an - [`IoTaskPool`] whose tasks are expected to complete very quickly. Generally speaking, they should just - await receiving data from somewhere (i.e. disk) and signal other systems when the data is ready + [`AsyncIO`] priority whose tasks are expected to complete very quickly. Generally speaking, they should just + await receiving data from somewhere (i.e. network) and signal other systems when the data is ready for consumption. (likely via channels) ## `no_std` Support diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 408ebd1bcb81e..ad8525e01dca7 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -192,7 +192,7 @@ pub enum TaskPriority { /// Intended for blocking CPU-bound tasks (e.g. shader compilation, building terrain) BlockingCompute, /// Intended for non-blocking async IO (e.g. HTTP servers/clients, network IO, io-uring file IO). - /// These jobs generally should do very little compute bound work and then yield immeidately upon + /// These jobs generally should do very little compute bound work and then yield immediately upon /// there being no more work to do. AsyncIO, /// Intended for shortlived CPU-bound jobs. These jobs are expected to do a small amount of work diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 9cf26de935a4f..23c1fe4fd6f26 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -167,6 +167,24 @@ impl TaskPool { .collect() } + /// Creates a builder for a new [`Task`] to schedule onto the [`TaskPool`].k + /// + /// # Example + /// + /// ```no_run + /// # async fn my_cool_task() {} + /// # use bevy_tasks::{TaskPool, TaskPriority}; + /// let task_pool = TaskPool::get(); + /// let task = task_pool.builder() + /// .with_priority(TaskPriority::BlockingIO) + /// .spawn(async { + /// my_cool_task + /// }); + /// ``` + pub fn builder(&self) -> TaskBuilder<'_, T> { + TaskBuilder::new(self) + } + /// Spawns a static future onto the thread pool. The returned Task is a future, which can be polled /// to retrieve the output of the original future. Dropping the task will attempt to cancel it. /// It can also be "detached", allowing it to continue running without having to be polled by the @@ -177,6 +195,48 @@ impl TaskPool { &self, future: impl Future + 'static + MaybeSend + MaybeSync, ) -> Task + where + T: 'static + MaybeSend + MaybeSync, + { + self.build().spawn(future) + } + + /// Spawns a static future on the JS event loop. This is exactly the same as [`TaskPool::spawn`]. + pub fn spawn_local( + &self, + future: impl Future + 'static + MaybeSend + MaybeSync, + ) -> Task + where + T: 'static + MaybeSend + MaybeSync, + { + } + + crate::cfg::web! { + if {} else { + pub(crate) fn try_tick_local() -> bool { + crate::cfg::bevy_executor! { + if { + Executor::try_tick_local() + } else { + EXECUTOR.try_tick() + } + } + } + } + } +} + +impl<'a, T> TaskBuilder<'a, T> { + /// Spawns a static future onto the thread pool. The returned Task is a future, which can be polled + /// to retrieve the output of the original future. Dropping the task will attempt to cancel it. + /// It can also be "detached", allowing it to continue running without having to be polled by the + /// end-user. + /// + /// If the provided future is non-`Send`, [`TaskPool::spawn_local`] should be used instead. + pub fn spawn( + &self, + future: impl Future + 'static + MaybeSend + MaybeSync, + ) -> Task where T: 'static + MaybeSend + MaybeSync, { @@ -204,22 +264,56 @@ impl TaskPool { { self.spawn(future) } +} - crate::cfg::web! { - if {} else { - pub(crate) fn try_tick_local() -> bool { - crate::cfg::bevy_executor! { - if { - Executor::try_tick_local() - } else { - EXECUTOR.try_tick() - } - } - } - } +impl<'a, 'scope, 'env, T: Send + 'scope> ScopeTaskBuilder<'a, 'scope, 'env, T> { + #[expect( + unsafe_code, + reason = "Executor::spawn and ThreadSpawner::spawn_scoped otherwise requires 'static Futures" + )] + /// Spawns a scoped future onto the thread pool. The scope *must* outlive + /// the provided future. The results of the future will be returned as a part of + /// [`TaskPool::scope`]'s return value. + /// + /// For futures that should run on the thread `scope` is called on [`Scope::spawn_on_scope`] should be used + /// instead. + /// + /// For more information, see [`TaskPool::scope`]. + pub fn spawn + 'scope + Send>(self, f: Fut) { + let task = match self.target { + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::Any => unsafe { + self.scope + .executor + .spawn_scoped(AssertUnwindSafe(f).catch_unwind(), Metadata::default()) + .fallible() + }, + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::Scope => unsafe { + self.scope + .scope_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() + .fallible() + }, + // SAFETY: The scope call that generated this `Scope` ensures that the created + // Task does not outlive 'scope. + ScopeTaskTarget::External => unsafe { + self.scope + .external_spawner + .spawn_scoped(AssertUnwindSafe(f).catch_unwind()) + .into_inner() + .fallible() + }, + }; + let result = self.scope.spawned.push(task); + debug_assert!(result.is_ok()); } } + /// A `TaskPool` scope for running one or more non-`'static` futures. /// /// For more information, see [`TaskPool::scope`]. diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index b1b965e7b2451..aff21ac782653 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -186,7 +186,7 @@ impl TaskPool { TASK_POOL.get_or_init(|| unsafe { Self::new_internal(f(), &EXECUTOR) }) } - /// Create a TaskPool with the default configuration. + /// Create a `TaskPool` with the default configuration. pub fn new() -> Self { TaskPoolBuilder::new().build() } @@ -436,10 +436,10 @@ impl TaskPool { /// /// # Example /// - /// ```norun + /// ```no_run /// # async fn my_cool_task() {} /// # use bevy_tasks::{TaskPool, TaskPriority}; - /// let task_pool = TaskPool::get() + /// let task_pool = TaskPool::get(); /// let task = task_pool.builder() /// .with_priority(TaskPriority::BlockingIO) /// .spawn(async { @@ -648,7 +648,7 @@ mod tests { #[test] fn test_spawn() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let foo = Box::new(42); let foo = &*foo; @@ -731,7 +731,7 @@ mod tests { #[test] fn test_mixed_spawn_on_scope_and_spawn() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let foo = Box::new(42); let foo = &*foo; @@ -779,7 +779,7 @@ mod tests { #[test] fn test_thread_locality() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let count = Arc::new(AtomicI32::new(0)); let barrier = Arc::new(Barrier::new(101)); let thread_check_failed = Arc::new(AtomicBool::new(false)); @@ -817,7 +817,7 @@ mod tests { #[test] fn test_nested_spawn() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let foo = Box::new(42); let foo = &*foo; @@ -855,7 +855,7 @@ mod tests { #[test] fn test_nested_locality() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let count = Arc::new(AtomicI32::new(0)); let barrier = Arc::new(Barrier::new(101)); let thread_check_failed = Arc::new(AtomicBool::new(false)); @@ -895,7 +895,7 @@ mod tests { // This test will often freeze on other executors. #[test] fn test_nested_scopes() { - let pool = TaskPool::get(); + let pool = TaskPool::get_or_init(TaskPoolBuilder::default); let count = Arc::new(AtomicI32::new(0)); pool.scope(|scope| {