servo_media_audio/
buffer_source_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::sync::Arc;
6
7use malloc_size_of_derive::MallocSizeOf;
8
9use crate::audio_node::{
10    AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo,
11    OnEndedCallback, ShouldPlay,
12};
13use crate::block::{Block, Chunk, FRAMES_PER_BLOCK, Tick};
14use crate::param::{Param, ParamType};
15
16/// Control messages directed to AudioBufferSourceNodes.
17#[derive(Debug, Clone, MallocSizeOf)]
18pub enum AudioBufferSourceNodeMessage {
19    /// Set the data block holding the audio sample data to be played.
20    SetBuffer(#[conditional_malloc_size_of] Option<Arc<AudioBuffer>>),
21    /// Set loop parameter.
22    SetLoopEnabled(bool),
23    /// Set loop parameter.
24    SetLoopEnd(f64),
25    /// Set loop parameter.
26    SetLoopStart(f64),
27    /// Set start parameters (when, offset, duration).
28    SetStartParams(f64, Option<f64>, Option<f64>),
29}
30
31/// This specifies options for constructing an AudioBufferSourceNode.
32#[derive(Debug, Clone, MallocSizeOf)]
33pub struct AudioBufferSourceNodeOptions {
34    /// The audio asset to be played.
35    pub buffer: Option<AudioBuffer>,
36    /// The initial value for the detune AudioParam.
37    pub detune: f32,
38    /// The initial value for the loop_enabled attribute.
39    pub loop_enabled: bool,
40    /// The initial value for the loop_end attribute.
41    pub loop_end: Option<f64>,
42    /// The initial value for the loop_start attribute.
43    pub loop_start: Option<f64>,
44    /// The initial value for the playback_rate AudioParam.
45    pub playback_rate: f32,
46}
47
48impl Default for AudioBufferSourceNodeOptions {
49    fn default() -> Self {
50        AudioBufferSourceNodeOptions {
51            buffer: None,
52            detune: 0.,
53            loop_enabled: false,
54            loop_end: None,
55            loop_start: None,
56            playback_rate: 1.,
57        }
58    }
59}
60
61/// AudioBufferSourceNode engine.
62/// <https://webaudio.github.io/web-audio-api/#AudioBufferSourceNode>
63#[derive(AudioScheduledSourceNode, AudioNodeCommon)]
64#[allow(dead_code)]
65pub(crate) struct AudioBufferSourceNode {
66    channel_info: ChannelInfo,
67    /// A data block holding the audio sample data to be played.
68    buffer: Option<Arc<AudioBuffer>>,
69    /// How many more buffer-frames to output. See buffer_pos for clarification.
70    buffer_duration: f64,
71    /// "Index" of the next buffer frame to play. "Index" is in quotes because
72    /// this variable maps to a playhead position (the offset in seconds can be
73    /// obtained by dividing by self.buffer.sample_rate), and therefore has
74    /// subsample accuracy; a fractional "index" means interpolation is needed.
75    buffer_pos: f64,
76    /// AudioParam to modulate the speed at which is rendered the audio stream.
77    detune: Param,
78    /// Whether we need to compute offsets from scratch.
79    initialized_pos: bool,
80    /// Indicates if the region of audio data designated by loopStart and loopEnd
81    /// should be played continuously in a loop.
82    loop_enabled: bool,
83    /// An playhead position where looping should end if the loop_enabled
84    /// attribute is true.
85    loop_end: Option<f64>,
86    /// An playhead position where looping should begin if the loop_enabled
87    /// attribute is true.
88    loop_start: Option<f64>,
89    /// The speed at which to render the audio stream. Can be negative if the
90    /// audio is to be played backwards. With a negative playback_rate, looping
91    /// jumps from loop_start to loop_end instead of the other way around.
92    playback_rate: Param,
93    /// Time at which the source should start playing.
94    start_at: Option<Tick>,
95    /// Offset parameter passed to Start().
96    start_offset: Option<f64>,
97    /// Duration parameter passed to Start().
98    start_duration: Option<f64>,
99    /// The same as start_at, but with subsample accuracy.
100    /// FIXME: AudioScheduledSourceNode should use this as well.
101    start_when: f64,
102    /// Time at which the source should stop playing.
103    stop_at: Option<Tick>,
104    /// The ended event callback.
105    pub onended_callback: Option<OnEndedCallback>,
106}
107
108impl AudioBufferSourceNode {
109    pub fn new(options: AudioBufferSourceNodeOptions, channel_info: ChannelInfo) -> Self {
110        Self {
111            channel_info,
112            buffer: options.buffer.map(Arc::new),
113            buffer_pos: 0.,
114            detune: Param::new_krate(options.detune),
115            initialized_pos: false,
116            loop_enabled: options.loop_enabled,
117            loop_end: options.loop_end,
118            loop_start: options.loop_start,
119            playback_rate: Param::new_krate(options.playback_rate),
120            buffer_duration: f64::INFINITY,
121            start_at: None,
122            start_offset: None,
123            start_duration: None,
124            start_when: 0.,
125            stop_at: None,
126            onended_callback: None,
127        }
128    }
129
130    pub fn handle_message(&mut self, message: AudioBufferSourceNodeMessage, _: f32) {
131        match message {
132            AudioBufferSourceNodeMessage::SetBuffer(buffer) => {
133                self.buffer = buffer;
134            },
135            // XXX(collares): To fully support dynamically updating loop bounds,
136            // Must truncate self.buffer_pos if it is now outside the loop.
137            AudioBufferSourceNodeMessage::SetLoopEnabled(loop_enabled) => {
138                self.loop_enabled = loop_enabled
139            },
140            AudioBufferSourceNodeMessage::SetLoopEnd(loop_end) => self.loop_end = Some(loop_end),
141            AudioBufferSourceNodeMessage::SetLoopStart(loop_start) => {
142                self.loop_start = Some(loop_start)
143            },
144            AudioBufferSourceNodeMessage::SetStartParams(when, offset, duration) => {
145                self.start_when = when;
146                self.start_offset = offset;
147                self.start_duration = duration;
148            },
149        }
150    }
151}
152
153impl AudioNodeEngine for AudioBufferSourceNode {
154    fn node_type(&self) -> AudioNodeType {
155        AudioNodeType::AudioBufferSourceNode
156    }
157
158    fn input_count(&self) -> u32 {
159        0
160    }
161
162    fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk {
163        debug_assert!(inputs.is_empty());
164
165        if self.buffer.is_none() {
166            inputs.blocks.push(Default::default());
167            return inputs;
168        }
169
170        let (start_at, stop_at) = match self.should_play_at(info.frame) {
171            ShouldPlay::No => {
172                inputs.blocks.push(Default::default());
173                return inputs;
174            },
175            ShouldPlay::Between(start, end) => (start.0 as usize, end.0 as usize),
176        };
177
178        let buffer = self.buffer.as_ref().unwrap();
179
180        let (mut actual_loop_start, mut actual_loop_end) = (0., buffer.len() as f64);
181        if self.loop_enabled {
182            let loop_start = self.loop_start.unwrap_or(0.);
183            let loop_end = self.loop_end.unwrap_or(0.);
184
185            if loop_start >= 0. && loop_end > loop_start {
186                actual_loop_start = loop_start * (buffer.sample_rate as f64);
187                actual_loop_end = loop_end * (buffer.sample_rate as f64);
188            }
189        }
190
191        // https://webaudio.github.io/web-audio-api/#computedplaybackrate
192        self.playback_rate.update(info, Tick(0));
193        self.detune.update(info, Tick(0));
194        // computed_playback_rate can be negative or zero.
195        let computed_playback_rate =
196            self.playback_rate.value() as f64 * (2.0_f64).powf(self.detune.value() as f64 / 1200.);
197        let forward = computed_playback_rate >= 0.;
198
199        if !self.initialized_pos {
200            self.initialized_pos = true;
201
202            // Apply the offset and duration parameters passed to start. We handle
203            // this here because the buffer may be set after Start() gets called, so
204            // this might be the first time we know the buffer's sample rate.
205            if let Some(start_offset) = self.start_offset {
206                self.buffer_pos = start_offset * (buffer.sample_rate as f64);
207                if self.buffer_pos < 0. {
208                    self.buffer_pos = 0.
209                } else if self.buffer_pos > buffer.len() as f64 {
210                    self.buffer_pos = buffer.len() as f64;
211                }
212            }
213
214            if self.loop_enabled {
215                if forward && self.buffer_pos >= actual_loop_end {
216                    self.buffer_pos = actual_loop_start;
217                }
218                // https://github.com/WebAudio/web-audio-api/issues/2031
219                if !forward && self.buffer_pos < actual_loop_start {
220                    self.buffer_pos = actual_loop_end;
221                }
222            }
223
224            if let Some(start_duration) = self.start_duration {
225                self.buffer_duration = start_duration * (buffer.sample_rate as f64);
226            }
227
228            // start_when can be subsample accurate. Correct buffer_pos.
229            //
230            // XXX(collares): What happens to "start_when" if the buffer gets
231            // set after Start()?
232            // XXX(collares): Need a better way to distingush between Start()
233            // being called with "when" in the past (in which case "when" must
234            // be ignored) and Start() being called with "when" in the future.
235            // This can now make a difference if "when" shouldn't be ignored
236            // but falls after the last frame of the previous quantum.
237            if self.start_when > info.time - 1. / info.sample_rate as f64 {
238                let first_time = info.time + start_at as f64 / info.sample_rate as f64;
239                if self.start_when <= first_time {
240                    let subsample_offset = (first_time - self.start_when) *
241                        (buffer.sample_rate as f64) *
242                        computed_playback_rate;
243                    self.buffer_pos += subsample_offset;
244                    self.buffer_duration -= subsample_offset.abs();
245                }
246            }
247        }
248
249        let mut buffer_offset_per_tick =
250            computed_playback_rate * (buffer.sample_rate as f64 / info.sample_rate as f64);
251
252        // WebAudio §1.9.5: "Setting the loop attribute to true causes playback of
253        // the region of the buffer defined by the endpoints loopStart and loopEnd
254        // to continue indefinitely, once any part of the looped region has been
255        // played. While loop remains true, looped playback will continue until one
256        // of the following occurs:
257        //  * stop() is called,
258        //  * the scheduled stop time has been reached,
259        //  * the duration has been exceeded, if start() was called with a duration value."
260        // Even with extreme playback rates we must stay inside the loop body, so wrap
261        // the per-tick delta instead of bailing.
262        if self.loop_enabled && actual_loop_end > actual_loop_start {
263            let loop_length = actual_loop_end - actual_loop_start;
264            if loop_length > 0. {
265                let step = buffer_offset_per_tick.abs();
266                if step >= loop_length {
267                    let mut wrapped = step.rem_euclid(loop_length);
268                    if wrapped == 0. {
269                        wrapped = loop_length;
270                    }
271                    buffer_offset_per_tick = wrapped.copysign(buffer_offset_per_tick);
272                }
273            }
274        }
275
276        // We will output at most this many frames (fewer if we run out of data).
277        let frames_to_output = stop_at - start_at;
278
279        // Fast path for the case where we can just copy FRAMES_PER_BLOCK
280        // frames straight from the buffer.
281        if frames_to_output == FRAMES_PER_BLOCK.0 as usize &&
282            forward &&
283            buffer_offset_per_tick == 1. &&
284            self.buffer_pos.trunc() == self.buffer_pos &&
285            self.buffer_pos + (FRAMES_PER_BLOCK.0 as f64) <= actual_loop_end &&
286            FRAMES_PER_BLOCK.0 as f64 <= self.buffer_duration
287        {
288            let mut block = Block::empty();
289            let pos = self.buffer_pos as usize;
290
291            for chan in 0..buffer.chans() {
292                block.push_chan(&buffer.buffers[chan as usize][pos..(pos + frames_to_output)]);
293            }
294
295            inputs.blocks.push(block);
296            self.buffer_pos += FRAMES_PER_BLOCK.0 as f64;
297            self.buffer_duration -= FRAMES_PER_BLOCK.0 as f64;
298        } else {
299            // Slow path, with interpolation.
300            let mut block = Block::default();
301            block.repeat(buffer.chans());
302            block.explicit_repeat();
303
304            debug_assert!(buffer.chans() > 0);
305
306            for chan in 0..buffer.chans() {
307                let data = block.data_chan_mut(chan);
308                let (_, data) = data.split_at_mut(start_at);
309                let (data, _) = data.split_at_mut(frames_to_output);
310
311                let mut pos = self.buffer_pos;
312                let mut duration = self.buffer_duration;
313
314                for sample in data {
315                    if duration <= 0. {
316                        break;
317                    }
318
319                    if self.loop_enabled {
320                        if forward && pos >= actual_loop_end {
321                            pos -= actual_loop_end - actual_loop_start;
322                        } else if !forward && pos < actual_loop_start {
323                            pos += actual_loop_end - actual_loop_start;
324                        }
325                    } else if pos < 0. || pos >= buffer.len() as f64 {
326                        break;
327                    }
328
329                    *sample = buffer.interpolate(chan, pos);
330                    pos += buffer_offset_per_tick;
331                    duration -= buffer_offset_per_tick.abs();
332                }
333
334                // This is the last channel, update parameters.
335                if chan == buffer.chans() - 1 {
336                    self.buffer_pos = pos;
337                    self.buffer_duration = duration;
338                }
339            }
340
341            inputs.blocks.push(block);
342        }
343
344        if !self.loop_enabled && (self.buffer_pos < 0. || self.buffer_pos >= buffer.len() as f64) ||
345            self.buffer_duration <= 0.
346        {
347            self.maybe_trigger_onended_callback();
348        }
349
350        inputs
351    }
352
353    fn get_param(&mut self, id: ParamType) -> &mut Param {
354        match id {
355            ParamType::PlaybackRate => &mut self.playback_rate,
356            ParamType::Detune => &mut self.detune,
357            _ => panic!("Unknown param {:?} for AudioBufferSourceNode", id),
358        }
359    }
360
361    make_message_handler!(
362        AudioBufferSourceNode: handle_message,
363        AudioScheduledSourceNode: handle_source_node_message
364    );
365}
366
367#[derive(Debug, Clone, MallocSizeOf)]
368pub struct AudioBuffer {
369    /// Invariant: all buffers must be of the same length
370    pub buffers: Vec<Vec<f32>>,
371    pub sample_rate: f32,
372}
373
374impl AudioBuffer {
375    pub fn new(chan: u8, len: usize, sample_rate: f32) -> Self {
376        assert!(chan > 0);
377        let mut buffers = Vec::with_capacity(chan as usize);
378        let single = vec![0.; len];
379        buffers.resize(chan as usize, single);
380        AudioBuffer {
381            buffers,
382            sample_rate,
383        }
384    }
385
386    pub fn from_buffers(buffers: Vec<Vec<f32>>, sample_rate: f32) -> Self {
387        for buf in &buffers {
388            assert_eq!(buf.len(), buffers[0].len())
389        }
390
391        Self {
392            buffers,
393            sample_rate,
394        }
395    }
396
397    pub fn from_buffer(buffer: Vec<f32>, sample_rate: f32) -> Self {
398        AudioBuffer::from_buffers(vec![buffer], sample_rate)
399    }
400
401    pub fn len(&self) -> usize {
402        self.buffers[0].len()
403    }
404
405    pub fn is_empty(&self) -> bool {
406        self.len() == 0
407    }
408
409    pub fn chans(&self) -> u8 {
410        self.buffers.len() as u8
411    }
412
413    // XXX(collares): There are better fast interpolation algorithms.
414    // Firefox uses (via Speex's resampler) the algorithm described in
415    // https://ccrma.stanford.edu/~jos/resample/resample.pdf
416    // There are Rust bindings: https://github.com/rust-av/speexdsp-rs
417    pub fn interpolate(&self, chan: u8, pos: f64) -> f32 {
418        debug_assert!(pos >= 0. && pos < self.len() as f64);
419
420        let prev = pos.floor() as usize;
421        let offset = pos - pos.floor();
422        match self.buffers[chan as usize].get(prev + 1) {
423            Some(next_sample) => {
424                ((1. - offset) * (self.buffers[chan as usize][prev] as f64) +
425                    offset * (*next_sample as f64)) as f32
426            },
427            _ => {
428                // linear extrapolation of two prev samples if there are two
429                if prev > 0 {
430                    ((1. + offset) * (self.buffers[chan as usize][prev] as f64) -
431                        offset * (self.buffers[chan as usize][prev - 1] as f64))
432                        as f32
433                } else {
434                    self.buffers[chan as usize][prev]
435                }
436            },
437        }
438    }
439
440    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
441        &mut self.buffers[chan as usize]
442    }
443}
servo_media_audio/buffer_source_node.rs

servo_media_audio/
buffer_source_node.rs
