tokio/runtime/scheduler/multi_thread/

worker.rs

//! A scheduler is initialized with a fixed number of workers. Each worker is
//! driven by a thread. Each worker has a "core" which contains data such as the
//! run queue and other state. When `block_in_place` is called, the worker's
//! "core" is handed off to a new thread allowing the scheduler to continue to
//! make progress while the originating thread blocks.
//!
//! # Shutdown
//!
//! Shutting down the runtime involves the following steps:
//!
//!  1. The Shared::close method is called. This closes the inject queue and
//!     `OwnedTasks` instance and wakes up all worker threads.
//!
//!  2. Each worker thread observes the close signal next time it runs
//!     Core::maintenance by checking whether the inject queue is closed.
//!     The `Core::is_shutdown` flag is set to true.
//!
//!  3. The worker thread calls `pre_shutdown` in parallel. Here, the worker
//!     will keep removing tasks from `OwnedTasks` until it is empty. No new
//!     tasks can be pushed to the `OwnedTasks` during or after this step as it
//!     was closed in step 1.
//!
//!  5. The workers call Shared::shutdown to enter the single-threaded phase of
//!     shutdown. These calls will push their core to `Shared::shutdown_cores`,
//!     and the last thread to push its core will finish the shutdown procedure.
//!
//!  6. The local run queue of each core is emptied, then the inject queue is
//!     emptied.
//!
//! At this point, shutdown has completed. It is not possible for any of the
//! collections to contain any tasks at this point, as each collection was
//! closed first, then emptied afterwards.
//!
//! ## Spawns during shutdown
//!
//! When spawning tasks during shutdown, there are two cases:
//!
//!  * The spawner observes the `OwnedTasks` being open, and the inject queue is
//!    closed.
//!  * The spawner observes the `OwnedTasks` being closed and doesn't check the
//!    inject queue.
//!
//! The first case can only happen if the `OwnedTasks::bind` call happens before
//! or during step 1 of shutdown. In this case, the runtime will clean up the
//! task in step 3 of shutdown.
//!
//! In the latter case, the task was not spawned and the task is immediately
//! cancelled by the spawner.
//!
//! The correctness of shutdown requires both the inject queue and `OwnedTasks`
//! collection to have a closed bit. With a close bit on only the inject queue,
//! spawning could run in to a situation where a task is successfully bound long
//! after the runtime has shut down. With a close bit on only the `OwnedTasks`,
//! the first spawning situation could result in the notification being pushed
//! to the inject queue after step 6 of shutdown, which would leave a task in
//! the inject queue indefinitely. This would be a ref-count cycle and a memory
//! leak.

use crate::loom::sync::{Arc, Mutex};
use crate::runtime;
use crate::runtime::scheduler::multi_thread::{
    idle, park, queue, Counters, Handle, Idle, Overflow, Parker, Stats, TraceStatus, Unparker,
};
use crate::runtime::scheduler::{inject::InjectQueue, Defer};
use crate::runtime::task::OwnedTasks;
use crate::runtime::{
    blocking, driver, scheduler, task, Config, SchedulerMetrics, TimerFlavor, WorkerMetrics,
};
use crate::runtime::{context, TaskHooks};
use crate::task::coop;
use crate::util::atomic_cell::AtomicCell;
use crate::util::cacheline::CachePadded;
use crate::util::rand::{FastRand, RngSeedGenerator};

use std::cell::RefCell;
use std::ops::ControlFlow;
use std::task::Waker;
use std::thread;
use std::time::{Duration, Instant};

mod metrics;

cfg_taskdump! {
    mod taskdump;
}

cfg_not_taskdump! {
    mod taskdump_mock;
}

#[cfg(all(tokio_unstable, feature = "time"))]
use crate::loom::sync::atomic::AtomicBool;

#[cfg(all(tokio_unstable, feature = "time"))]
use crate::runtime::time_alt;

use crate::runtime::metrics::ScheduleLatencyInstant;
#[cfg(all(tokio_unstable, feature = "time"))]
use crate::runtime::scheduler::util;

/// A scheduler worker
pub(super) struct Worker {
    /// Reference to scheduler's handle
    handle: Arc<Handle>,

    /// Index holding this worker's remote state
    index: usize,

    /// Used to hand-off a worker's core to another thread.
    core: AtomicCell<Core>,
}

/// Core data
struct Core {
    /// Used to schedule bookkeeping tasks every so often.
    tick: u32,

    /// When a task is scheduled from a worker, it is stored in this slot. The
    /// worker will check this slot for a task **before** checking the run
    /// queue. This effectively results in the **last** scheduled task to be run
    /// next (LIFO). This is an optimization for improving locality which
    /// benefits message passing patterns and helps to reduce latency.
    lifo_slot: Option<Notified>,

    /// When `true`, locally scheduled tasks go to the LIFO slot. When `false`,
    /// they go to the back of the `run_queue`.
    lifo_enabled: bool,

    /// The worker-local run queue.
    run_queue: queue::Local<Arc<Handle>>,

    #[cfg(all(tokio_unstable, feature = "time"))]
    time_context: time_alt::LocalContext,

    /// True if the worker is currently searching for more work. Searching
    /// involves attempting to steal from other workers.
    is_searching: bool,

    /// True if the scheduler is being shutdown
    is_shutdown: bool,

    /// True if the scheduler is being traced
    is_traced: bool,

    /// Whether or not the worker has just returned from a park in which we
    /// parked on the I/O driver.
    had_driver: park::HadDriver,

    /// If `true`, the worker should eagerly notify another worker when polling
    /// the first task after returning from a park in which it parked on the I/O
    /// or time driver.
    enable_eager_driver_handoff: bool,

    /// Parker
    ///
    /// Stored in an `Option` as the parker is added / removed to make the
    /// borrow checker happy.
    park: Option<Parker>,

    /// Per-worker runtime stats
    stats: Stats,

    /// How often to check the global queue
    global_queue_interval: u32,

    /// Fast random number generator.
    rand: FastRand,
}

/// State shared across all workers
pub(crate) struct Shared {
    /// Per-worker remote state. All other workers have access to this and is
    /// how they communicate between each other.
    remotes: Box<[Remote]>,

    /// Global task queue used for:
    ///  1. Submit work to the scheduler while **not** currently on a worker thread.
    ///  2. Submit work to the scheduler when a worker run queue is saturated
    pub(super) inject: CachePadded<InjectQueue<Arc<Handle>>>,

    /// Coordinates idle workers
    idle: CachePadded<Idle>,

    /// Collection of all active tasks spawned onto this executor.
    pub(crate) owned: OwnedTasks<Arc<Handle>>,

    /// Data synchronized by the scheduler mutex
    pub(super) synced: Mutex<Synced>,

    /// Cores that have observed the shutdown signal
    ///
    /// The core is **not** placed back in the worker to avoid it from being
    /// stolen by a thread that was spawned as part of `block_in_place`.
    #[allow(clippy::vec_box)] // we're moving an already-boxed value
    shutdown_cores: Mutex<Vec<Box<Core>>>,

    /// The number of cores that have observed the trace signal.
    pub(super) trace_status: TraceStatus,

    /// Scheduler configuration options
    config: Config,

    /// Collects metrics from the runtime.
    pub(super) scheduler_metrics: SchedulerMetrics,

    pub(super) worker_metrics: Box<[WorkerMetrics]>,

    /// Startup time of this scheduler.
    ///
    /// This instant is used as the basis of task `scheduled_at` measurements.
    schedule_latency_start: Option<Instant>,

    /// Only held to trigger some code on drop. This is used to get internal
    /// runtime metrics that can be useful when doing performance
    /// investigations. This does nothing (empty struct, no drop impl) unless
    /// the `tokio_internal_mt_counters` `cfg` flag is set.
    _counters: Counters,
}

/// Data synchronized by the scheduler mutex
pub(crate) struct Synced {
    /// Synchronized state for `Idle`.
    pub(super) idle: idle::Synced,

    #[cfg(all(tokio_unstable, feature = "time"))]
    /// Timers pending to be registered.
    /// This is used to register a timer but the [`Core`]
    /// is not available in the current thread.
    ///
    /// This must stay under the same mutex as `idle`: a parking worker drains
    /// it (via `try_lock`) only after publishing its parked state under this
    /// lock, and `notify_if_work_pending` does not check for pending timers,
    /// so sharing the lock with the parking transition is what prevents a
    /// timer push from being stranded while every worker sleeps.
    inject_timers: Vec<time_alt::EntryHandle>,
}

/// Used to communicate with a worker from other threads.
struct Remote {
    /// Steals tasks from this worker.
    pub(super) steal: queue::Steal<Arc<Handle>>,

    /// Unparks the associated worker thread
    unpark: Unparker,
}

/// Thread-local context
pub(crate) struct Context {
    /// Worker
    worker: Arc<Worker>,

    /// Core data
    core: RefCell<Option<Box<Core>>>,

    /// Tasks to wake after resource drivers are polled. This is mostly to
    /// handle yielded tasks.
    pub(crate) defer: Defer,
}

/// Starts the workers
pub(crate) struct Launch(Vec<Arc<Worker>>);

/// A notified task handle
type Notified = task::Notified<Arc<Handle>>;

/// Value picked out of thin-air. Running the LIFO slot a handful of times
/// seems sufficient to benefit from locality. More than 3 times probably is
/// over-weighting. The value can be tuned in the future with data that shows
/// improvements.
const MAX_LIFO_POLLS_PER_TICK: usize = 3;

#[allow(clippy::too_many_arguments)]
pub(super) fn create(
    size: usize,
    park: Parker,
    driver_handle: driver::Handle,
    blocking_spawner: blocking::Spawner,
    seed_generator: RngSeedGenerator,
    config: Config,
    timer_flavor: TimerFlavor,
    name: Option<String>,
) -> (Arc<Handle>, Launch) {
    let mut cores = Vec::with_capacity(size);
    let mut remotes = Vec::with_capacity(size);
    let mut worker_metrics = Vec::with_capacity(size);

    // Create the local queues
    for _ in 0..size {
        let (steal, run_queue) = queue::local();

        let park = park.clone();
        let unpark = park.unpark();
        let metrics = WorkerMetrics::from_config(&config);
        let stats = Stats::new(&metrics);

        cores.push(Box::new(Core {
            tick: 0,
            lifo_slot: None,
            lifo_enabled: !config.disable_lifo_slot,
            run_queue,
            #[cfg(all(tokio_unstable, feature = "time"))]
            time_context: time_alt::LocalContext::new(),
            is_searching: false,
            is_shutdown: false,
            is_traced: false,
            enable_eager_driver_handoff: config.enable_eager_driver_handoff,
            had_driver: park::HadDriver::No,
            park: Some(park),
            global_queue_interval: stats.tuned_global_queue_interval(&config),
            stats,
            rand: FastRand::from_seed(config.seed_generator.next_seed()),
        }));

        remotes.push(Remote { steal, unpark });
        worker_metrics.push(metrics);
    }

    let (idle, idle_synced) = Idle::new(size);
    let schedule_latency_start = config.track_task_schedule_latency.then(Instant::now);

    let remotes_len = remotes.len();
    let handle = Arc::new(Handle {
        name,
        task_hooks: TaskHooks::from_config(&config),
        shared: Shared {
            remotes: remotes.into_boxed_slice(),
            inject: CachePadded::new(InjectQueue::new()),
            idle: CachePadded::new(idle),
            owned: OwnedTasks::new(size),
            synced: Mutex::new(Synced {
                idle: idle_synced,
                #[cfg(all(tokio_unstable, feature = "time"))]
                inject_timers: Vec::new(),
            }),
            shutdown_cores: Mutex::new(vec![]),
            trace_status: TraceStatus::new(remotes_len),
            config,
            scheduler_metrics: SchedulerMetrics::new(),
            worker_metrics: worker_metrics.into_boxed_slice(),
            schedule_latency_start,
            _counters: Counters,
        },
        driver: driver_handle,
        blocking_spawner,
        seed_generator,
        timer_flavor,
        #[cfg(all(tokio_unstable, feature = "time"))]
        is_shutdown: AtomicBool::new(false),
    });

    let mut launch = Launch(vec![]);

    for (index, core) in cores.drain(..).enumerate() {
        launch.0.push(Arc::new(Worker {
            handle: handle.clone(),
            index,
            core: AtomicCell::new(Some(core)),
        }));
    }

    (handle, launch)
}

#[track_caller]
pub(crate) fn block_in_place<F, R>(f: F) -> R
where
    F: FnOnce() -> R,
{
    // Try to steal the worker core back
    struct Reset {
        take_core: bool,
        budget: coop::Budget,
    }

    impl Drop for Reset {
        fn drop(&mut self) {
            with_current(|maybe_cx| {
                if let Some(cx) = maybe_cx {
                    if self.take_core {
                        let core = cx.worker.core.take();

                        if core.is_some() {
                            cx.worker.handle.shared.worker_metrics[cx.worker.index]
                                .set_thread_id(thread::current().id());
                        }

                        let mut cx_core = cx.core.borrow_mut();
                        assert!(cx_core.is_none());
                        *cx_core = core;
                    }

                    // Reset the task budget as we are re-entering the
                    // runtime.
                    coop::set(self.budget);
                }
            });
        }
    }

    // Move the runtime to another thread, if there is one to move.
    // Do this in a non-generic function that doesn't depend on `F` or `R`.
    let (had_entered, take_core) = match maybe_move_runtime() {
        Ok(r) => r,
        Err(panic_message) => panic!("{}", panic_message),
    };

    if had_entered {
        // Unset the current task's budget. Blocking sections are not
        // constrained by task budgets.
        let _reset = Reset {
            take_core,
            budget: coop::stop(),
        };

        crate::runtime::context::exit_runtime(f)
    } else {
        f()
    }
}

fn maybe_move_runtime() -> Result<(bool, bool), &'static str> {
    let mut had_entered = false;
    let mut take_core = false;

    with_current(|maybe_cx| {
        match (
            crate::runtime::context::current_enter_context(),
            maybe_cx.is_some(),
        ) {
            (context::EnterRuntime::Entered { .. }, true) => {
                // We are on a thread pool runtime thread, so we just need to
                // set up blocking.
                had_entered = true;
            }
            (
                context::EnterRuntime::Entered {
                    allow_block_in_place,
                },
                false,
            ) => {
                // We are on an executor, but _not_ on the thread pool.  That is
                // _only_ okay if we are in a thread pool runtime's block_on
                // method:
                if allow_block_in_place {
                    had_entered = true;
                    return Ok(());
                } else {
                    // This probably means we are on the current_thread runtime or in a
                    // LocalSet, where it is _not_ okay to block.
                    return Err(
                        "can call blocking only when running on the multi-threaded runtime",
                    );
                }
            }
            (context::EnterRuntime::NotEntered, true) => {
                // This is a nested call to block_in_place (we already exited).
                // All the necessary setup has already been done.
                return Ok(());
            }
            (context::EnterRuntime::NotEntered, false) => {
                // We are outside of the tokio runtime, so blocking is fine.
                // We can also skip all of the thread pool blocking setup steps.
                return Ok(());
            }
        }

        let cx = maybe_cx.expect("no .is_some() == false cases above should lead here");

        // Since deferred tasks don't stay on `core`, make sure to wake them
        // before blocking.
        cx.defer.wake();

        // Get the worker core. If none is set, then blocking is fine!
        let mut core = match cx.core.borrow_mut().take() {
            Some(core) => core,
            None => return Ok(()),
        };

        // If we heavily call `spawn_blocking`, there might be no available thread to
        // run this core. Except for the task in the lifo_slot, all tasks can be
        // stolen, so we move the task out of the lifo_slot to the run_queue.
        if let Some(task) = core.lifo_slot.take() {
            core.run_queue
                .push_back_or_overflow(task, &*cx.worker.handle, &mut core.stats);
        }

        // We are taking the core from the context and sending it to another
        // thread.
        take_core = true;

        // The parker should be set here
        assert!(core.park.is_some());

        // In order to block, the core must be sent to another thread for
        // execution.
        //
        // First, move the core back into the worker's shared core slot.
        cx.worker.core.set(core);

        // Next, clone the worker handle and send it to a new thread for
        // processing.
        //
        // Once the blocking task is done executing, we will attempt to
        // steal the core back.
        let worker = cx.worker.clone();
        runtime::spawn_blocking(move || run(worker));
        Ok(())
    })?;
    Ok((had_entered, take_core))
}

impl Launch {
    pub(crate) fn launch(mut self) {
        for worker in self.0.drain(..) {
            runtime::spawn_blocking(move || run(worker));
        }
    }
}

fn run(worker: Arc<Worker>) {
    #[allow(dead_code)]
    struct AbortOnPanic;

    impl Drop for AbortOnPanic {
        fn drop(&mut self) {
            if std::thread::panicking() {
                eprintln!("worker thread panicking; aborting process");
                std::process::abort();
            }
        }
    }

    // Catching panics on worker threads in tests is quite tricky. Instead, when
    // debug assertions are enabled, we just abort the process.
    #[cfg(debug_assertions)]
    let _abort_on_panic = AbortOnPanic;

    // Acquire a core. If this fails, then another thread is running this
    // worker and there is nothing further to do.
    let core = match worker.core.take() {
        Some(core) => core,
        None => return,
    };

    worker.handle.shared.worker_metrics[worker.index].set_thread_id(thread::current().id());

    let handle = scheduler::Handle::MultiThread(worker.handle.clone());

    crate::runtime::context::enter_runtime(&handle, true, |_| {
        // Set the worker context.
        let cx = scheduler::Context::MultiThread(Context {
            worker,
            core: RefCell::new(None),
            defer: Defer::new(),
        });

        context::set_scheduler(&cx, || {
            let cx = cx.expect_multi_thread();

            cx.run(core);

            // Check if there are any deferred tasks to notify. This can happen when
            // the worker core is lost due to `block_in_place()` being called from
            // within the task.
            cx.defer.wake();
        });
    });
}

impl Context {
    fn run(&self, mut core: Box<Core>) {
        // Reset `lifo_enabled` here in case the core was previously stolen from
        // a task that had the LIFO slot disabled.
        self.reset_lifo_enabled(&mut core);

        // Start as "processing" tasks as polling tasks from the local queue
        // will be one of the first things we do.
        core.stats.start_processing_scheduled_tasks();

        while !core.is_shutdown {
            self.assert_lifo_enabled_is_correct(&core);

            if core.is_traced {
                core = self.worker.handle.trace_core(core);
            }

            // Increment the tick
            core.tick();

            // Run maintenance, if needed
            core = self.maintenance(core);

            // First, check work available to the current worker.
            if let Some(task) = core.next_task(&self.worker) {
                core = match self.run_task(task, core) {
                    ControlFlow::Continue(core) => core,
                    ControlFlow::Break(()) => return,
                };
                continue;
            }

            // We consumed all work in the queues and will start searching for work.
            core.stats.end_processing_scheduled_tasks();

            // There is no more **local** work to process, try to steal work
            // from other workers.
            if let Some(task) = core.steal_work(&self.worker) {
                // Found work, switch back to processing
                core.stats.start_processing_scheduled_tasks();
                core = match self.run_task(task, core) {
                    ControlFlow::Continue(core) => core,
                    ControlFlow::Break(()) => return,
                };
            } else {
                // Wait for work
                core = if !self.defer.is_empty() {
                    self.park_yield(core)
                } else {
                    self.park(core)
                };
                core.stats.start_processing_scheduled_tasks();
            }
        }

        #[cfg(all(tokio_unstable, feature = "time"))]
        {
            match self.worker.handle.timer_flavor {
                TimerFlavor::Traditional => {}
                TimerFlavor::Alternative => {
                    util::time_alt::shutdown_local_timers(
                        &mut core.time_context.wheel,
                        &mut core.time_context.canc_rx,
                        self.worker.handle.take_remote_timers(),
                        &self.worker.handle.driver,
                    );
                }
            }
        }

        core.pre_shutdown(&self.worker);
        // Signal shutdown
        self.worker.handle.shutdown_core(core);
    }

    /// Running a task may consume the core. If the core is still available when
    /// running the task completes, it is returned. Otherwise, the worker will need
    /// to stop processing.
    fn run_task(&self, task: Notified, mut core: Box<Core>) -> ControlFlow<(), Box<Core>> {
        let task = self.worker.handle.shared.owned.assert_owner(task);

        // Make sure the worker is not in the **searching** state. This enables
        // another idle worker to try to steal work.
        let notified_parked_worker = core.transition_from_searching(&self.worker);

        // If the setting to wake eagerly when releasing the I/O driver is
        // enabled, and this worker had the driver, wake a parked worker to come
        // grab it from us.
        //
        // Note that this is only done when we are *actually* about to poll a
        // task, rather than whenever the worker has unparked. When the worker
        // has been unparked, it may not actually have any tasks to poll, and if
        // it's still holding the I/O driver, it should just go back to polling
        // the driver again, rather than trying to wake someone else spuriously.
        //
        // Note that this explicitly checks `cfg!(tokio_unstable)` in addition,
        // as that should result in this whole expression being eliminated at
        // compile-time when unstable features are disabled.
        if cfg!(tokio_unstable)
            && core.enable_eager_driver_handoff
            && core.had_driver == park::HadDriver::Yes
            && !notified_parked_worker
        // don't do it a second time
        {
            core.had_driver = park::HadDriver::No;
            self.worker.handle.notify_parked_local();
        }

        self.assert_lifo_enabled_is_correct(&core);

        // Measure the poll start time. Note that we may end up polling other
        // tasks under this poll-time measurement. Tasks from the LIFO slot
        // inherit the "parent"'s limits, but their schedule latency is recorded
        // separately when each task is polled.
        let schedule_latency_context = task
            .get_scheduled_at()
            .prepare(self.worker.handle.shared.schedule_latency_start);
        let _task_schedule_latency = core.stats.start_poll(schedule_latency_context);

        #[cfg(tokio_unstable)]
        let task_meta = task.task_meta(_task_schedule_latency);

        // Make the core available to the runtime context
        *self.core.borrow_mut() = Some(core);

        // Run the task
        coop::budget(|| {
            // Unlike the poll time above, poll start callback is attached to the task id,
            // so it is tightly associated with the actual poll invocation.
            #[cfg(tokio_unstable)]
            self.worker
                .handle
                .task_hooks
                .poll_start_callback(&task_meta);

            task.run();

            #[cfg(tokio_unstable)]
            self.worker.handle.task_hooks.poll_stop_callback(&task_meta);

            let mut lifo_polls = 0;

            // As long as there is budget remaining and a task exists in the
            // `lifo_slot`, then keep running.
            loop {
                // Check if we still have the core. If not, the core was stolen
                // by another worker.
                let mut core = match self.core.borrow_mut().take() {
                    Some(core) => core,
                    None => {
                        // In this case, we cannot call `reset_lifo_enabled()`
                        // because the core was stolen. The stealer will handle
                        // that at the top of `Context::run`
                        return ControlFlow::Break(());
                    }
                };

                // Check for a task in the LIFO slot
                let task = match core.lifo_slot.take() {
                    Some(task) => task,
                    None => {
                        self.reset_lifo_enabled(&mut core);
                        core.stats.end_poll();
                        return ControlFlow::Continue(core);
                    }
                };

                if !coop::has_budget_remaining() {
                    core.stats.end_poll();

                    // Not enough budget left to run the LIFO task, push it to
                    // the back of the queue and return.
                    core.run_queue.push_back_or_overflow(
                        task,
                        &*self.worker.handle,
                        &mut core.stats,
                    );
                    // If we hit this point, the LIFO slot should be enabled.
                    // There is no need to reset it.
                    debug_assert!(core.lifo_enabled);
                    return ControlFlow::Continue(core);
                }

                // Track that we are about to run a task from the LIFO slot.
                lifo_polls += 1;
                super::counters::inc_lifo_schedules();

                // Disable the LIFO slot if we reach our limit
                //
                // In ping-ping style workloads where task A notifies task B,
                // which notifies task A again, continuously prioritizing the
                // LIFO slot can cause starvation as these two tasks will
                // repeatedly schedule the other. To mitigate this, we limit the
                // number of times the LIFO slot is prioritized.
                if lifo_polls >= MAX_LIFO_POLLS_PER_TICK {
                    core.lifo_enabled = false;
                    super::counters::inc_lifo_capped();
                }

                let task = self.worker.handle.shared.owned.assert_owner(task);

                // Record the LIFO task's schedule latency independently from
                // the outer task's poll-time sample. The returned value is the
                // same sample that is passed to the histogram recorder.
                let schedule_latency_context = task
                    .get_scheduled_at()
                    .prepare(self.worker.handle.shared.schedule_latency_start);
                let _task_schedule_latency =
                    core.stats.record_schedule_latency(schedule_latency_context);

                *self.core.borrow_mut() = Some(core);

                #[cfg(tokio_unstable)]
                let task_meta = task.task_meta(_task_schedule_latency);

                #[cfg(tokio_unstable)]
                self.worker
                    .handle
                    .task_hooks
                    .poll_start_callback(&task_meta);

                task.run();

                #[cfg(tokio_unstable)]
                self.worker.handle.task_hooks.poll_stop_callback(&task_meta);
            }
        })
    }

    fn reset_lifo_enabled(&self, core: &mut Core) {
        core.lifo_enabled = !self.worker.handle.shared.config.disable_lifo_slot;
    }

    fn assert_lifo_enabled_is_correct(&self, core: &Core) {
        debug_assert_eq!(
            core.lifo_enabled,
            !self.worker.handle.shared.config.disable_lifo_slot
        );
    }

    fn maintenance(&self, mut core: Box<Core>) -> Box<Core> {
        if core.tick % self.worker.handle.shared.config.event_interval == 0 {
            super::counters::inc_num_maintenance();

            core.stats.end_processing_scheduled_tasks();

            // Call `park` with a 0 timeout. This enables the I/O driver, timer, ...
            // to run without actually putting the thread to sleep.
            core = self.park_yield(core);

            // Run regularly scheduled maintenance
            core.maintenance(&self.worker);

            core.stats.start_processing_scheduled_tasks();
        }

        core
    }

    /// Parks the worker thread while waiting for tasks to execute.
    ///
    /// This function checks if indeed there's no more work left to be done before parking.
    /// Also important to notice that, before parking, the worker thread will try to take
    /// ownership of the Driver (IO/Time) and dispatch any events that might have fired.
    /// Whenever a worker thread executes the Driver loop, all waken tasks are scheduled
    /// in its own local queue until the queue saturates (ntasks > `LOCAL_QUEUE_CAPACITY`).
    /// When the local queue is saturated, the overflow tasks are added to the injection queue
    /// from where other workers can pick them up.
    /// Also, we rely on the workstealing algorithm to spread the tasks amongst workers
    /// after all the IOs get dispatched
    fn park(&self, mut core: Box<Core>) -> Box<Core> {
        if let Some(f) = &self.worker.handle.shared.config.before_park {
            f();
        }

        if core.transition_to_parked(&self.worker) {
            while !core.is_shutdown && !core.is_traced {
                core.stats.about_to_park();
                core.stats
                    .submit(&self.worker.handle.shared.worker_metrics[self.worker.index]);

                core = self.park_internal(core, None);

                core.stats.unparked();

                // Run regularly scheduled maintenance
                core.maintenance(&self.worker);

                if core.transition_from_parked(&self.worker) {
                    break;
                }
            }
        }

        if let Some(f) = &self.worker.handle.shared.config.after_unpark {
            f();
        }
        core
    }

    fn park_yield(&self, core: Box<Core>) -> Box<Core> {
        self.park_internal(core, Some(Duration::from_millis(0)))
    }

    fn park_internal(&self, mut core: Box<Core>, duration: Option<Duration>) -> Box<Core> {
        self.assert_lifo_enabled_is_correct(&core);

        // Take the parker out of core
        let mut park = core.park.take().expect("park missing");
        // Store `core` in context
        *self.core.borrow_mut() = Some(core);

        #[cfg(feature = "time")]
        let (duration, auto_advance_duration) = match self.worker.handle.timer_flavor {
            TimerFlavor::Traditional => (duration, None::<Duration>),
            #[cfg(tokio_unstable)]
            TimerFlavor::Alternative => {
                // Must happens after taking out the parker, as the `Handle::schedule_local`
                // will delay the notify if the parker taken out.
                //
                // See comments in `Handle::schedule_local` for more details.
                let MaintainLocalTimer {
                    park_duration: duration,
                    auto_advance_duration,
                } = self.maintain_local_timers_before_parking(duration);
                (duration, auto_advance_duration)
            }
        };

        // Park thread
        let had_driver = if let Some(timeout) = duration {
            park.park_timeout(&self.worker.handle.driver, timeout)
        } else {
            park.park(&self.worker.handle.driver)
        };

        self.defer.wake();

        #[cfg(feature = "time")]
        match self.worker.handle.timer_flavor {
            TimerFlavor::Traditional => {
                // suppress unused variable warning
                let _ = auto_advance_duration;
            }
            #[cfg(tokio_unstable)]
            TimerFlavor::Alternative => {
                // Must happens before placing back the parker, as the `Handle::schedule_local`
                // will delay the notify if the parker is still in `core`.
                //
                // See comments in `Handle::schedule_local` for more details.
                self.maintain_local_timers_after_parking(auto_advance_duration);
            }
        }

        // Remove `core` from context
        core = self.core.borrow_mut().take().expect("core missing");

        // Place `park` back in `core`
        core.park = Some(park);
        core.had_driver = had_driver;

        if core.should_notify_others() {
            self.worker.handle.notify_parked_local();
        }
        core
    }

    pub(crate) fn defer(&self, waker: &Waker) {
        if self.core.borrow().is_none() {
            // If there is no core, then the worker is currently in a block_in_place. In this case,
            // we cannot use the defer queue as we aren't really in the current runtime.
            waker.wake_by_ref();
        } else {
            self.defer.defer(waker);
        }
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    /// Maintain local timers before parking the resource driver.
    ///
    /// * Remove cancelled timers from the local timer wheel.
    /// * Register remote timers to the local timer wheel.
    /// * Adjust the park duration based on
    ///   * the next timer expiration time.
    ///   * whether auto-advancing is required (feature = "test-util").
    ///
    /// # Returns
    ///
    /// `(Box<Core>, park_duration, auto_advance_duration)`
    fn maintain_local_timers_before_parking(
        &self,
        park_duration: Option<Duration>,
    ) -> MaintainLocalTimer {
        let handle = &self.worker.handle;
        let mut wake_queue = time_alt::WakeQueue::new();

        let (should_yield, next_timer) = with_current(|maybe_cx| {
            let cx = maybe_cx.expect("function should be called when core is present");
            assert_eq!(
                Arc::as_ptr(&cx.worker.handle),
                Arc::as_ptr(&self.worker.handle),
                "function should be called on the exact same worker"
            );

            let mut maybe_core = cx.core.borrow_mut();
            let core = maybe_core.as_mut().expect("core missing");
            let time_cx = &mut core.time_context;

            util::time_alt::process_registration_queue(
                &mut time_cx.registration_queue,
                &mut time_cx.wheel,
                &time_cx.canc_tx,
                &mut wake_queue,
            );
            util::time_alt::insert_inject_timers(
                &mut time_cx.wheel,
                &time_cx.canc_tx,
                handle.take_remote_timers(),
                &mut wake_queue,
            );
            util::time_alt::remove_cancelled_timers(&mut time_cx.wheel, &mut time_cx.canc_rx);
            let should_yield = !wake_queue.is_empty();

            let next_timer = util::time_alt::next_expiration_time(&time_cx.wheel, &handle.driver);

            (should_yield, next_timer)
        });

        wake_queue.wake_all();

        if should_yield {
            MaintainLocalTimer {
                park_duration: Some(Duration::from_millis(0)),
                auto_advance_duration: None,
            }
        } else {
            // get the minimum duration
            let dur = util::time_alt::min_duration(park_duration, next_timer);
            if util::time_alt::pre_auto_advance(&handle.driver, dur) {
                MaintainLocalTimer {
                    park_duration: Some(Duration::ZERO),
                    auto_advance_duration: dur,
                }
            } else {
                MaintainLocalTimer {
                    park_duration: dur,
                    auto_advance_duration: None,
                }
            }
        }
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    /// Maintain local timers after unparking the resource driver.
    ///
    /// * Auto-advance time, if required (feature = "test-util").
    /// * Process expired timers.
    fn maintain_local_timers_after_parking(&self, auto_advance_duration: Option<Duration>) {
        let handle = &self.worker.handle;
        let mut wake_queue = time_alt::WakeQueue::new();

        with_current(|maybe_cx| {
            let cx = maybe_cx.expect("function should be called when core is present");
            assert_eq!(
                Arc::as_ptr(&cx.worker.handle),
                Arc::as_ptr(&self.worker.handle),
                "function should be called on the exact same worker"
            );

            let mut maybe_core = cx.core.borrow_mut();
            let core = maybe_core.as_mut().expect("core missing");
            let time_cx = &mut core.time_context;

            util::time_alt::post_auto_advance(&handle.driver, auto_advance_duration);
            util::time_alt::process_expired_timers(
                &mut time_cx.wheel,
                &handle.driver,
                &mut wake_queue,
            );
        });

        wake_queue.wake_all();
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    fn with_core<F, R>(&self, f: F) -> R
    where
        F: FnOnce(Option<&mut Core>) -> R,
    {
        match self.core.borrow_mut().as_mut() {
            Some(core) => f(Some(core)),
            None => f(None),
        }
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    pub(crate) fn with_time_temp_local_context<F, R>(&self, f: F) -> R
    where
        F: FnOnce(Option<time_alt::TempLocalContext<'_>>) -> R,
    {
        self.with_core(|maybe_core| match maybe_core {
            Some(core) if core.is_shutdown => f(Some(time_alt::TempLocalContext::new_shutdown())),
            Some(core) => f(Some(time_alt::TempLocalContext::new_running(
                &mut core.time_context,
            ))),
            None => f(None),
        })
    }

    pub(crate) fn worker_index(&self) -> usize {
        self.worker.index
    }
}

impl Core {
    /// Increment the tick
    fn tick(&mut self) {
        self.tick = self.tick.wrapping_add(1);
    }

    /// Return the next notified task available to this worker.
    fn next_task(&mut self, worker: &Worker) -> Option<Notified> {
        if self.tick % self.global_queue_interval == 0 {
            // Update the global queue interval, if needed
            self.tune_global_queue_interval(worker);

            worker
                .handle
                .next_remote_task()
                .or_else(|| self.next_local_task())
        } else {
            let maybe_task = self.next_local_task();

            if maybe_task.is_some() {
                return maybe_task;
            }

            if worker.inject().is_empty() {
                return None;
            }

            let cap = usize::min(
                // Other threads can only **remove** tasks from the current
                // worker's `run_queue`. So, we can be confident that by the
                // time we call `run_queue.push_back` below, there will be *at
                // least* `cap` available slots in the queue.
                //
                // Note that even though `next_local_task()` just returned
                // `None`, this may be different from `max_capacity()` if
                // another worker is currently stealing tasks from us.
                self.run_queue.remaining_slots(),
                // We want to make sure that all of the tasks we take end up in
                // the first half of the local queue. This ensures that the
                // tasks do not get pushed to the inject queue again if overflow
                // occurs, as overflow only affects tasks in the second half of
                // the local queue.
                //
                // Note that even if there are concurrent stealers, we do not
                // need to consider the value of `remaining_slots()` because a
                // future call to `push_overflow()` can only succeed once that
                // concurrent stealer has finished stealing, so at that point
                // the tasks we are adding now will be in the first half.
                self.run_queue.max_capacity() / 2,
            );

            // The worker is currently idle, pull a batch of work from the
            // injection queue. We don't want to pull *all* the work so other
            // workers can also get some.
            let n = usize::min(
                worker.inject().len() / worker.handle.shared.remotes.len() + 1,
                cap,
            );

            // Take at least one task since the first task is returned directly
            // and not pushed onto the local queue.
            let n = usize::max(1, n);

            worker.inject().pop_n(n, |mut tasks| {
                // Pop the first task to return immediately
                let ret = tasks.next();

                // Push the rest of the on the run queue
                self.run_queue.push_back(tasks);

                ret
            })
        }
    }

    fn next_local_task(&mut self) -> Option<Notified> {
        self.lifo_slot.take().or_else(|| self.run_queue.pop())
    }

    /// Function responsible for stealing tasks from another worker
    ///
    /// Note: Only if less than half the workers are searching for tasks to steal
    /// a new worker will actually try to steal. The idea is to make sure not all
    /// workers will be trying to steal at the same time.
    fn steal_work(&mut self, worker: &Worker) -> Option<Notified> {
        if !self.transition_to_searching(worker) {
            return None;
        }

        let num = worker.handle.shared.remotes.len();
        // Start from a random worker
        let start = self.rand.fastrand_n(num as u32) as usize;

        for i in 0..num {
            let i = (start + i) % num;

            // Don't steal from ourself! We know we don't have work.
            if i == worker.index {
                continue;
            }

            let target = &worker.handle.shared.remotes[i];
            if let Some(task) = target
                .steal
                .steal_into(&mut self.run_queue, &mut self.stats)
            {
                return Some(task);
            }
        }

        // Fallback on checking the global queue
        worker.handle.next_remote_task()
    }

    fn transition_to_searching(&mut self, worker: &Worker) -> bool {
        if !self.is_searching {
            self.is_searching = worker.handle.shared.idle.transition_worker_to_searching();
        }

        self.is_searching
    }

    fn transition_from_searching(&mut self, worker: &Worker) -> bool {
        if !self.is_searching {
            return false;
        }

        self.is_searching = false;
        worker.handle.transition_worker_from_searching()
    }

    fn has_tasks(&self) -> bool {
        self.lifo_slot.is_some() || self.run_queue.has_tasks()
    }

    fn should_notify_others(&self) -> bool {
        // If there are tasks available to steal, but this worker is not
        // looking for tasks to steal, notify another worker.
        if self.is_searching {
            return false;
        }
        self.lifo_slot.is_some() as usize + self.run_queue.len() > 1
    }

    /// Prepares the worker state for parking.
    ///
    /// Returns true if the transition happened, false if there is work to do first.
    fn transition_to_parked(&mut self, worker: &Worker) -> bool {
        // Workers should not park if they have work to do or are about to be traced
        if self.has_tasks() || self.is_traced {
            return false;
        }

        // When the final worker transitions **out** of searching to parked, it
        // must check all the queues one last time in case work materialized
        // between the last work scan and transitioning out of searching.
        let is_last_searcher = worker.handle.shared.idle.transition_worker_to_parked(
            &worker.handle.shared,
            worker.index,
            self.is_searching,
        );

        // The worker is no longer searching. Setting this is the local cache
        // only.
        self.is_searching = false;

        if is_last_searcher {
            worker.handle.notify_if_work_pending();
        }

        true
    }

    /// Returns `true` if the transition happened.
    fn transition_from_parked(&mut self, worker: &Worker) -> bool {
        // If a task is in the lifo slot/run queue, then we must unpark regardless of
        // being notified. Same when woken to be traced: a dump unparks worker
        // threads without going through `Idle`, so the worker does it here.
        if self.has_tasks() || self.is_traced {
            // When a worker wakes, it should only transition to the "searching"
            // state when the wake originates from another worker *or* a new task
            // is pushed. We do *not* want the worker to transition to "searching"
            // when it wakes when the I/O driver receives new events.
            self.is_searching = !worker
                .handle
                .shared
                .idle
                .unpark_worker_by_id(&worker.handle.shared, worker.index);
            return true;
        }

        if worker
            .handle
            .shared
            .idle
            .is_parked(&worker.handle.shared, worker.index)
        {
            return false;
        }

        // When unparked, the worker is in the searching state.
        self.is_searching = true;
        true
    }

    /// Runs maintenance work such as checking the pool's state.
    fn maintenance(&mut self, worker: &Worker) {
        self.stats
            .submit(&worker.handle.shared.worker_metrics[worker.index]);

        if !self.is_shutdown {
            // Check if the scheduler has been shutdown
            self.is_shutdown = worker.inject().is_closed();
        }

        if !self.is_traced {
            // Check if the worker should be tracing.
            self.is_traced = worker.handle.shared.trace_status.trace_requested();
        }
    }

    /// Signals all tasks to shut down, and waits for them to complete. Must run
    /// before we enter the single-threaded phase of shutdown processing.
    fn pre_shutdown(&mut self, worker: &Worker) {
        // Start from a random inner list
        let start = self
            .rand
            .fastrand_n(worker.handle.shared.owned.get_shard_size() as u32);
        // Signal to all tasks to shut down.
        worker
            .handle
            .shared
            .owned
            .close_and_shutdown_all(start as usize);

        self.stats
            .submit(&worker.handle.shared.worker_metrics[worker.index]);
    }

    /// Shuts down the core.
    fn shutdown(&mut self, handle: &Handle) {
        // Take the core
        let mut park = self.park.take().expect("park missing");

        // Drain the queue
        while self.next_local_task().is_some() {}

        park.shutdown(&handle.driver);
    }

    fn tune_global_queue_interval(&mut self, worker: &Worker) {
        let next = self
            .stats
            .tuned_global_queue_interval(&worker.handle.shared.config);

        // Smooth out jitter
        if u32::abs_diff(self.global_queue_interval, next) > 2 {
            self.global_queue_interval = next;
        }
    }
}

impl Worker {
    /// Returns a reference to the scheduler's injection queue.
    fn inject(&self) -> &InjectQueue<Arc<Handle>> {
        &self.handle.shared.inject
    }
}

impl Handle {
    pub(super) fn schedule_task(&self, task: Notified, is_yield: bool) {
        if self.shared.schedule_latency_start.is_some() {
            task.set_scheduled_at(ScheduleLatencyInstant::new(
                self.shared.schedule_latency_start,
            ));
        }

        with_current(|maybe_cx| {
            if let Some(cx) = maybe_cx {
                // Make sure the task is part of the **current** scheduler.
                if self.ptr_eq(&cx.worker.handle) {
                    // And the current thread still holds a core
                    if let Some(core) = cx.core.borrow_mut().as_mut() {
                        self.schedule_local(core, task, is_yield);
                        return;
                    }
                }
            }

            // Otherwise, use the inject queue.
            self.push_remote_task(task);
            self.notify_parked_remote();
        });
    }

    // Separated case to reduce LLVM codegen in `Handle::bind_new_task`.
    pub(super) fn schedule_option_task_without_yield(&self, task: Option<Notified>) {
        if let Some(task) = task {
            self.schedule_task(task, false);
        }
    }

    fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) {
        core.stats.inc_local_schedule_count();

        // Spawning from the worker thread. If scheduling a "yield" then the
        // task must always be pushed to the back of the queue, enabling other
        // tasks to be executed. If **not** a yield, then there is more
        // flexibility and the task may go to the front of the queue.
        let should_notify = if is_yield || !core.lifo_enabled {
            core.run_queue
                .push_back_or_overflow(task, self, &mut core.stats);
            true
        } else {
            // Push to the LIFO slot
            let prev = core.lifo_slot.take();
            let ret = prev.is_some();

            if let Some(prev) = prev {
                core.run_queue
                    .push_back_or_overflow(prev, self, &mut core.stats);
            }

            core.lifo_slot = Some(task);

            ret
        };

        // Only notify if not currently parked. If `park` is `None`, then the
        // scheduling is from a resource driver. As notifications often come in
        // batches, the notification is delayed until the park is complete.
        if should_notify && core.park.is_some() {
            self.notify_parked_local();
        }
    }

    fn next_remote_task(&self) -> Option<Notified> {
        self.shared.inject.pop()
    }

    fn push_remote_task(&self, task: Notified) {
        self.shared.scheduler_metrics.inc_remote_schedule_count();

        self.shared.inject.push(task);
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    pub(crate) fn push_remote_timer(&self, hdl: time_alt::EntryHandle) {
        assert_eq!(self.timer_flavor, TimerFlavor::Alternative);
        {
            let mut synced = self.shared.synced.lock();
            synced.inject_timers.push(hdl);
        }
        self.notify_parked_remote();
    }

    #[cfg(all(tokio_unstable, feature = "time"))]
    pub(crate) fn take_remote_timers(&self) -> Vec<time_alt::EntryHandle> {
        assert_eq!(self.timer_flavor, TimerFlavor::Alternative);
        // It's ok to lost the race, as another worker is
        // draining the inject_timers.
        match self.shared.synced.try_lock() {
            Some(mut synced) => std::mem::take(&mut synced.inject_timers),
            None => Vec::new(),
        }
    }

    pub(super) fn close(&self) {
        if self.shared.inject.close() {
            self.notify_all();
        }
    }

    /// Notify a parked worker.
    ///
    /// Returns `true` if a worker was notified, `false` otherwise.
    fn notify_parked_local(&self) -> bool {
        super::counters::inc_num_inc_notify_local();

        if let Some(index) = self.shared.idle.worker_to_notify(&self.shared) {
            super::counters::inc_num_unparks_local();
            self.shared.remotes[index].unpark.unpark(&self.driver);
            true
        } else {
            false
        }
    }

    fn notify_parked_remote(&self) {
        if let Some(index) = self.shared.idle.worker_to_notify(&self.shared) {
            self.shared.remotes[index].unpark.unpark(&self.driver);
        }
    }

    pub(super) fn notify_all(&self) {
        for remote in &self.shared.remotes[..] {
            remote.unpark.unpark(&self.driver);
        }
    }

    fn notify_if_work_pending(&self) {
        for remote in &self.shared.remotes[..] {
            if !remote.steal.is_empty() {
                self.notify_parked_local();
                return;
            }
        }

        if !self.shared.inject.is_empty() {
            self.notify_parked_local();
        }
    }

    /// Returns `true` if another parked worker was notified, `false` otherwise.
    fn transition_worker_from_searching(&self) -> bool {
        if self.shared.idle.transition_worker_from_searching() {
            // We are the final searching worker. Because work was found, we
            // need to notify another worker.
            self.notify_parked_local()
        } else {
            false
        }
    }

    /// Signals that a worker has observed the shutdown signal and has replaced
    /// its core back into its handle.
    ///
    /// If all workers have reached this point, the final cleanup is performed.
    fn shutdown_core(&self, core: Box<Core>) {
        let mut cores = self.shared.shutdown_cores.lock();
        cores.push(core);

        if cores.len() != self.shared.remotes.len() {
            return;
        }

        debug_assert!(self.shared.owned.is_empty());

        for mut core in cores.drain(..) {
            core.shutdown(self);
        }

        // Drain the injection queue
        //
        // We already shut down every task, so we can simply drop the tasks.
        while let Some(task) = self.next_remote_task() {
            drop(task);
        }
    }

    fn ptr_eq(&self, other: &Handle) -> bool {
        std::ptr::eq(self, other)
    }
}

impl Overflow<Arc<Handle>> for Handle {
    fn push(&self, task: task::Notified<Arc<Handle>>) {
        self.push_remote_task(task);
    }

    fn push_batch<I>(&self, iter: I)
    where
        I: Iterator<Item = task::Notified<Arc<Handle>>>,
    {
        self.shared.inject.push_batch(iter);
    }
}

#[cfg(all(tokio_unstable, feature = "time"))]
/// Returned by [`Context::maintain_local_timers_before_parking`].
struct MaintainLocalTimer {
    park_duration: Option<Duration>,
    auto_advance_duration: Option<Duration>,
}

#[track_caller]
fn with_current<R>(f: impl FnOnce(Option<&Context>) -> R) -> R {
    use scheduler::Context::MultiThread;

    context::with_scheduler(|ctx| match ctx {
        Some(MultiThread(ctx)) => f(Some(ctx)),
        _ => f(None),
    })
}