servo_media_audio/
buffer_source_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::sync::Arc;
6
7use malloc_size_of_derive::MallocSizeOf;
8
9use crate::audio_node::{
10    AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo,
11    OnEndedCallback, ShouldPlay,
12};
13use crate::block::{Block, Chunk, FRAMES_PER_BLOCK, Tick};
14use crate::param::{Param, ParamType};
15
16/// Control messages directed to AudioBufferSourceNodes.
17#[derive(Debug, Clone, MallocSizeOf)]
18pub enum AudioBufferSourceNodeMessage {
19    /// Set the data block holding the audio sample data to be played.
20    SetBuffer(#[conditional_malloc_size_of] Option<Arc<AudioBuffer>>),
21    /// Set loop parameter.
22    SetLoopEnabled(bool),
23    /// Set loop parameter.
24    SetLoopEnd(f64),
25    /// Set loop parameter.
26    SetLoopStart(f64),
27    /// Set start parameters (when, offset, duration).
28    SetStartParams(f64, Option<f64>, Option<f64>),
29}
30
31/// This specifies options for constructing an AudioBufferSourceNode.
32#[derive(Debug, Clone, MallocSizeOf)]
33pub struct AudioBufferSourceNodeOptions {
34    /// The audio asset to be played.
35    #[conditional_malloc_size_of]
36    pub buffer: Option<Arc<AudioBuffer>>,
37    /// The initial value for the detune AudioParam.
38    pub detune: f32,
39    /// The initial value for the loop_enabled attribute.
40    pub loop_enabled: bool,
41    /// The initial value for the loop_end attribute.
42    pub loop_end: Option<f64>,
43    /// The initial value for the loop_start attribute.
44    pub loop_start: Option<f64>,
45    /// The initial value for the playback_rate AudioParam.
46    pub playback_rate: f32,
47}
48
49impl Default for AudioBufferSourceNodeOptions {
50    fn default() -> Self {
51        AudioBufferSourceNodeOptions {
52            buffer: None,
53            detune: 0.,
54            loop_enabled: false,
55            loop_end: None,
56            loop_start: None,
57            playback_rate: 1.,
58        }
59    }
60}
61
62/// AudioBufferSourceNode engine.
63/// <https://webaudio.github.io/web-audio-api/#AudioBufferSourceNode>
64#[derive(AudioScheduledSourceNode, AudioNodeCommon)]
65#[allow(dead_code)]
66pub(crate) struct AudioBufferSourceNode {
67    channel_info: ChannelInfo,
68    /// A data block holding the audio sample data to be played.
69    buffer: Option<Arc<AudioBuffer>>,
70    /// How many more buffer-frames to output. See buffer_pos for clarification.
71    buffer_duration: f64,
72    /// "Index" of the next buffer frame to play. "Index" is in quotes because
73    /// this variable maps to a playhead position (the offset in seconds can be
74    /// obtained by dividing by self.buffer.sample_rate), and therefore has
75    /// subsample accuracy; a fractional "index" means interpolation is needed.
76    buffer_pos: f64,
77    /// AudioParam to modulate the speed at which is rendered the audio stream.
78    detune: Param,
79    /// Whether we need to compute offsets from scratch.
80    initialized_pos: bool,
81    /// Indicates if the region of audio data designated by loopStart and loopEnd
82    /// should be played continuously in a loop.
83    loop_enabled: bool,
84    /// An playhead position where looping should end if the loop_enabled
85    /// attribute is true.
86    loop_end: Option<f64>,
87    /// An playhead position where looping should begin if the loop_enabled
88    /// attribute is true.
89    loop_start: Option<f64>,
90    /// The speed at which to render the audio stream. Can be negative if the
91    /// audio is to be played backwards. With a negative playback_rate, looping
92    /// jumps from loop_start to loop_end instead of the other way around.
93    playback_rate: Param,
94    /// Time at which the source should start playing.
95    start_at: Option<Tick>,
96    /// Offset parameter passed to Start().
97    start_offset: Option<f64>,
98    /// Duration parameter passed to Start().
99    start_duration: Option<f64>,
100    /// The same as start_at, but with subsample accuracy.
101    /// FIXME: AudioScheduledSourceNode should use this as well.
102    start_when: f64,
103    /// Time at which the source should stop playing.
104    stop_at: Option<Tick>,
105    /// The ended event callback.
106    pub onended_callback: Option<OnEndedCallback>,
107}
108
109impl AudioBufferSourceNode {
110    pub fn new(options: AudioBufferSourceNodeOptions, channel_info: ChannelInfo) -> Self {
111        Self {
112            channel_info,
113            buffer: options.buffer,
114            buffer_pos: 0.,
115            detune: Param::new_krate(options.detune),
116            initialized_pos: false,
117            loop_enabled: options.loop_enabled,
118            loop_end: options.loop_end,
119            loop_start: options.loop_start,
120            playback_rate: Param::new_krate(options.playback_rate),
121            buffer_duration: f64::INFINITY,
122            start_at: None,
123            start_offset: None,
124            start_duration: None,
125            start_when: 0.,
126            stop_at: None,
127            onended_callback: None,
128        }
129    }
130
131    pub fn handle_message(&mut self, message: AudioBufferSourceNodeMessage, _: f32) {
132        match message {
133            AudioBufferSourceNodeMessage::SetBuffer(buffer) => {
134                self.buffer = buffer;
135            },
136            // XXX(collares): To fully support dynamically updating loop bounds,
137            // Must truncate self.buffer_pos if it is now outside the loop.
138            AudioBufferSourceNodeMessage::SetLoopEnabled(loop_enabled) => {
139                self.loop_enabled = loop_enabled
140            },
141            AudioBufferSourceNodeMessage::SetLoopEnd(loop_end) => self.loop_end = Some(loop_end),
142            AudioBufferSourceNodeMessage::SetLoopStart(loop_start) => {
143                self.loop_start = Some(loop_start)
144            },
145            AudioBufferSourceNodeMessage::SetStartParams(when, offset, duration) => {
146                self.start_when = when;
147                self.start_offset = offset;
148                self.start_duration = duration;
149            },
150        }
151    }
152}
153
154impl AudioNodeEngine for AudioBufferSourceNode {
155    fn node_type(&self) -> AudioNodeType {
156        AudioNodeType::AudioBufferSourceNode
157    }
158
159    fn input_count(&self) -> u32 {
160        0
161    }
162
163    fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk {
164        debug_assert!(inputs.is_empty());
165
166        if self.buffer.is_none() {
167            inputs.blocks.push(Default::default());
168            return inputs;
169        }
170
171        let (start_at, stop_at) = match self.should_play_at(info.frame) {
172            ShouldPlay::No => {
173                inputs.blocks.push(Default::default());
174                return inputs;
175            },
176            ShouldPlay::Between(start, end) => (start.0 as usize, end.0 as usize),
177        };
178
179        let buffer = self.buffer.as_ref().unwrap();
180
181        let (mut actual_loop_start, mut actual_loop_end) = (0., buffer.len() as f64);
182        if self.loop_enabled {
183            let loop_start = self.loop_start.unwrap_or(0.);
184            let loop_end = self.loop_end.unwrap_or(0.);
185
186            if loop_start >= 0. && loop_end > loop_start {
187                actual_loop_start = loop_start * (buffer.sample_rate as f64);
188                actual_loop_end = loop_end * (buffer.sample_rate as f64);
189            }
190        }
191
192        // https://webaudio.github.io/web-audio-api/#computedplaybackrate
193        self.playback_rate.update(info, Tick(0));
194        self.detune.update(info, Tick(0));
195        // computed_playback_rate can be negative or zero.
196        let computed_playback_rate =
197            self.playback_rate.value() as f64 * (2.0_f64).powf(self.detune.value() as f64 / 1200.);
198        let forward = computed_playback_rate >= 0.;
199
200        if !self.initialized_pos {
201            self.initialized_pos = true;
202
203            // Apply the offset and duration parameters passed to start. We handle
204            // this here because the buffer may be set after Start() gets called, so
205            // this might be the first time we know the buffer's sample rate.
206            if let Some(start_offset) = self.start_offset {
207                self.buffer_pos = start_offset * (buffer.sample_rate as f64);
208                if self.buffer_pos < 0. {
209                    self.buffer_pos = 0.
210                } else if self.buffer_pos > buffer.len() as f64 {
211                    self.buffer_pos = buffer.len() as f64;
212                }
213            }
214
215            if self.loop_enabled {
216                if forward && self.buffer_pos >= actual_loop_end {
217                    self.buffer_pos = actual_loop_start;
218                }
219                // https://github.com/WebAudio/web-audio-api/issues/2031
220                if !forward && self.buffer_pos < actual_loop_start {
221                    self.buffer_pos = actual_loop_end;
222                }
223            }
224
225            if let Some(start_duration) = self.start_duration {
226                self.buffer_duration = start_duration * (buffer.sample_rate as f64);
227            }
228
229            // start_when can be subsample accurate. Correct buffer_pos.
230            //
231            // XXX(collares): What happens to "start_when" if the buffer gets
232            // set after Start()?
233            // XXX(collares): Need a better way to distingush between Start()
234            // being called with "when" in the past (in which case "when" must
235            // be ignored) and Start() being called with "when" in the future.
236            // This can now make a difference if "when" shouldn't be ignored
237            // but falls after the last frame of the previous quantum.
238            if self.start_when > info.time - 1. / info.sample_rate as f64 {
239                let first_time = info.time + start_at as f64 / info.sample_rate as f64;
240                if self.start_when <= first_time {
241                    let subsample_offset = (first_time - self.start_when) *
242                        (buffer.sample_rate as f64) *
243                        computed_playback_rate;
244                    self.buffer_pos += subsample_offset;
245                    self.buffer_duration -= subsample_offset.abs();
246                }
247            }
248        }
249
250        let mut buffer_offset_per_tick =
251            computed_playback_rate * (buffer.sample_rate as f64 / info.sample_rate as f64);
252
253        // WebAudio §1.9.5: "Setting the loop attribute to true causes playback of
254        // the region of the buffer defined by the endpoints loopStart and loopEnd
255        // to continue indefinitely, once any part of the looped region has been
256        // played. While loop remains true, looped playback will continue until one
257        // of the following occurs:
258        //  * stop() is called,
259        //  * the scheduled stop time has been reached,
260        //  * the duration has been exceeded, if start() was called with a duration value."
261        // Even with extreme playback rates we must stay inside the loop body, so wrap
262        // the per-tick delta instead of bailing.
263        if self.loop_enabled && actual_loop_end > actual_loop_start {
264            let loop_length = actual_loop_end - actual_loop_start;
265            if loop_length > 0. {
266                let step = buffer_offset_per_tick.abs();
267                if step >= loop_length {
268                    let mut wrapped = step.rem_euclid(loop_length);
269                    if wrapped == 0. {
270                        wrapped = loop_length;
271                    }
272                    buffer_offset_per_tick = wrapped.copysign(buffer_offset_per_tick);
273                }
274            }
275        }
276
277        // We will output at most this many frames (fewer if we run out of data).
278        let frames_to_output = stop_at - start_at;
279
280        // Fast path for the case where we can just copy FRAMES_PER_BLOCK
281        // frames straight from the buffer.
282        if frames_to_output == FRAMES_PER_BLOCK.0 as usize &&
283            forward &&
284            buffer_offset_per_tick == 1. &&
285            self.buffer_pos.trunc() == self.buffer_pos &&
286            self.buffer_pos + (FRAMES_PER_BLOCK.0 as f64) <= actual_loop_end &&
287            FRAMES_PER_BLOCK.0 as f64 <= self.buffer_duration
288        {
289            let mut block = Block::empty();
290            let pos = self.buffer_pos as usize;
291
292            for chan in 0..buffer.chans() {
293                block.push_chan(&buffer.buffers[chan as usize][pos..(pos + frames_to_output)]);
294            }
295
296            inputs.blocks.push(block);
297            self.buffer_pos += FRAMES_PER_BLOCK.0 as f64;
298            self.buffer_duration -= FRAMES_PER_BLOCK.0 as f64;
299        } else {
300            // Slow path, with interpolation.
301            let mut block = Block::default();
302            block.repeat(buffer.chans());
303            block.explicit_repeat();
304
305            debug_assert!(buffer.chans() > 0);
306
307            for chan in 0..buffer.chans() {
308                let data = block.data_chan_mut(chan);
309                let (_, data) = data.split_at_mut(start_at);
310                let (data, _) = data.split_at_mut(frames_to_output);
311
312                let mut pos = self.buffer_pos;
313                let mut duration = self.buffer_duration;
314
315                for sample in data {
316                    if duration <= 0. {
317                        break;
318                    }
319
320                    if self.loop_enabled {
321                        if forward && pos >= actual_loop_end {
322                            pos -= actual_loop_end - actual_loop_start;
323                        } else if !forward && pos < actual_loop_start {
324                            pos += actual_loop_end - actual_loop_start;
325                        }
326                    } else if pos < 0. || pos >= buffer.len() as f64 {
327                        break;
328                    }
329
330                    *sample = buffer.interpolate(chan, pos);
331                    pos += buffer_offset_per_tick;
332                    duration -= buffer_offset_per_tick.abs();
333                }
334
335                // This is the last channel, update parameters.
336                if chan == buffer.chans() - 1 {
337                    self.buffer_pos = pos;
338                    self.buffer_duration = duration;
339                }
340            }
341
342            inputs.blocks.push(block);
343        }
344
345        if !self.loop_enabled && (self.buffer_pos < 0. || self.buffer_pos >= buffer.len() as f64) ||
346            self.buffer_duration <= 0.
347        {
348            self.maybe_trigger_onended_callback();
349        }
350
351        inputs
352    }
353
354    fn get_param(&mut self, id: ParamType) -> &mut Param {
355        match id {
356            ParamType::PlaybackRate => &mut self.playback_rate,
357            ParamType::Detune => &mut self.detune,
358            _ => panic!("Unknown param {:?} for AudioBufferSourceNode", id),
359        }
360    }
361
362    make_message_handler!(
363        AudioBufferSourceNode: handle_message,
364        AudioScheduledSourceNode: handle_source_node_message
365    );
366}
367
368#[derive(Debug, Clone, MallocSizeOf)]
369pub struct AudioBuffer {
370    /// Invariant: all buffers must be of the same length
371    pub buffers: Vec<Vec<f32>>,
372    pub sample_rate: f32,
373}
374
375impl AudioBuffer {
376    pub fn new(chan: u8, len: usize, sample_rate: f32) -> Self {
377        assert!(chan > 0);
378        let mut buffers = Vec::with_capacity(chan as usize);
379        let single = vec![0.; len];
380        buffers.resize(chan as usize, single);
381        AudioBuffer {
382            buffers,
383            sample_rate,
384        }
385    }
386
387    pub fn from_buffers(buffers: Vec<Vec<f32>>, sample_rate: f32) -> Self {
388        for buf in &buffers {
389            assert_eq!(buf.len(), buffers[0].len())
390        }
391
392        Self {
393            buffers,
394            sample_rate,
395        }
396    }
397
398    pub fn from_buffer(buffer: Vec<f32>, sample_rate: f32) -> Self {
399        AudioBuffer::from_buffers(vec![buffer], sample_rate)
400    }
401
402    pub fn len(&self) -> usize {
403        self.buffers[0].len()
404    }
405
406    pub fn is_empty(&self) -> bool {
407        self.len() == 0
408    }
409
410    pub fn chans(&self) -> u8 {
411        self.buffers.len() as u8
412    }
413
414    // XXX(collares): There are better fast interpolation algorithms.
415    // Firefox uses (via Speex's resampler) the algorithm described in
416    // https://ccrma.stanford.edu/~jos/resample/resample.pdf
417    // There are Rust bindings: https://github.com/rust-av/speexdsp-rs
418    pub fn interpolate(&self, chan: u8, pos: f64) -> f32 {
419        debug_assert!(pos >= 0. && pos < self.len() as f64);
420
421        let prev = pos.floor() as usize;
422        let offset = pos - pos.floor();
423        match self.buffers[chan as usize].get(prev + 1) {
424            Some(next_sample) => {
425                ((1. - offset) * (self.buffers[chan as usize][prev] as f64) +
426                    offset * (*next_sample as f64)) as f32
427            },
428            _ => {
429                // linear extrapolation of two prev samples if there are two
430                if prev > 0 {
431                    ((1. + offset) * (self.buffers[chan as usize][prev] as f64) -
432                        offset * (self.buffers[chan as usize][prev - 1] as f64))
433                        as f32
434                } else {
435                    self.buffers[chan as usize][prev]
436                }
437            },
438        }
439    }
440
441    pub fn scale(&mut self, scale: f64) {
442        for buffer in self.buffers.as_mut_slice() {
443            *buffer = buffer
444                .iter()
445                .map(|x| (*x as f64 * scale) as f32)
446                .collect::<Vec<_>>();
447        }
448    }
449
450    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
451        &mut self.buffers[chan as usize]
452    }
453}
servo_media_audio/buffer_source_node.rs

servo_media_audio/
buffer_source_node.rs
