servo_media_audio/
buffer_source_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use malloc_size_of_derive::MallocSizeOf;
6
7use crate::block::{Block, Chunk, FRAMES_PER_BLOCK, Tick};
8use crate::node::{
9    AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo,
10    OnEndedCallback, ShouldPlay,
11};
12use crate::param::{Param, ParamType};
13
14/// Control messages directed to AudioBufferSourceNodes.
15#[derive(Debug, Clone, MallocSizeOf)]
16pub enum AudioBufferSourceNodeMessage {
17    /// Set the data block holding the audio sample data to be played.
18    SetBuffer(Option<AudioBuffer>),
19    /// Set loop parameter.
20    SetLoopEnabled(bool),
21    /// Set loop parameter.
22    SetLoopEnd(f64),
23    /// Set loop parameter.
24    SetLoopStart(f64),
25    /// Set start parameters (when, offset, duration).
26    SetStartParams(f64, Option<f64>, Option<f64>),
27}
28
29/// This specifies options for constructing an AudioBufferSourceNode.
30#[derive(Debug, Clone, MallocSizeOf)]
31pub struct AudioBufferSourceNodeOptions {
32    /// The audio asset to be played.
33    pub buffer: Option<AudioBuffer>,
34    /// The initial value for the detune AudioParam.
35    pub detune: f32,
36    /// The initial value for the loop_enabled attribute.
37    pub loop_enabled: bool,
38    /// The initial value for the loop_end attribute.
39    pub loop_end: Option<f64>,
40    /// The initial value for the loop_start attribute.
41    pub loop_start: Option<f64>,
42    /// The initial value for the playback_rate AudioParam.
43    pub playback_rate: f32,
44}
45
46impl Default for AudioBufferSourceNodeOptions {
47    fn default() -> Self {
48        AudioBufferSourceNodeOptions {
49            buffer: None,
50            detune: 0.,
51            loop_enabled: false,
52            loop_end: None,
53            loop_start: None,
54            playback_rate: 1.,
55        }
56    }
57}
58
59/// AudioBufferSourceNode engine.
60/// <https://webaudio.github.io/web-audio-api/#AudioBufferSourceNode>
61#[derive(AudioScheduledSourceNode, AudioNodeCommon)]
62#[allow(dead_code)]
63pub(crate) struct AudioBufferSourceNode {
64    channel_info: ChannelInfo,
65    /// A data block holding the audio sample data to be played.
66    buffer: Option<AudioBuffer>,
67    /// How many more buffer-frames to output. See buffer_pos for clarification.
68    buffer_duration: f64,
69    /// "Index" of the next buffer frame to play. "Index" is in quotes because
70    /// this variable maps to a playhead position (the offset in seconds can be
71    /// obtained by dividing by self.buffer.sample_rate), and therefore has
72    /// subsample accuracy; a fractional "index" means interpolation is needed.
73    buffer_pos: f64,
74    /// AudioParam to modulate the speed at which is rendered the audio stream.
75    detune: Param,
76    /// Whether we need to compute offsets from scratch.
77    initialized_pos: bool,
78    /// Indicates if the region of audio data designated by loopStart and loopEnd
79    /// should be played continuously in a loop.
80    loop_enabled: bool,
81    /// An playhead position where looping should end if the loop_enabled
82    /// attribute is true.
83    loop_end: Option<f64>,
84    /// An playhead position where looping should begin if the loop_enabled
85    /// attribute is true.
86    loop_start: Option<f64>,
87    /// The speed at which to render the audio stream. Can be negative if the
88    /// audio is to be played backwards. With a negative playback_rate, looping
89    /// jumps from loop_start to loop_end instead of the other way around.
90    playback_rate: Param,
91    /// Time at which the source should start playing.
92    start_at: Option<Tick>,
93    /// Offset parameter passed to Start().
94    start_offset: Option<f64>,
95    /// Duration parameter passed to Start().
96    start_duration: Option<f64>,
97    /// The same as start_at, but with subsample accuracy.
98    /// FIXME: AudioScheduledSourceNode should use this as well.
99    start_when: f64,
100    /// Time at which the source should stop playing.
101    stop_at: Option<Tick>,
102    /// The ended event callback.
103    pub onended_callback: Option<OnEndedCallback>,
104}
105
106impl AudioBufferSourceNode {
107    pub fn new(options: AudioBufferSourceNodeOptions, channel_info: ChannelInfo) -> Self {
108        Self {
109            channel_info,
110            buffer: options.buffer,
111            buffer_pos: 0.,
112            detune: Param::new_krate(options.detune),
113            initialized_pos: false,
114            loop_enabled: options.loop_enabled,
115            loop_end: options.loop_end,
116            loop_start: options.loop_start,
117            playback_rate: Param::new_krate(options.playback_rate),
118            buffer_duration: f64::INFINITY,
119            start_at: None,
120            start_offset: None,
121            start_duration: None,
122            start_when: 0.,
123            stop_at: None,
124            onended_callback: None,
125        }
126    }
127
128    pub fn handle_message(&mut self, message: AudioBufferSourceNodeMessage, _: f32) {
129        match message {
130            AudioBufferSourceNodeMessage::SetBuffer(buffer) => {
131                self.buffer = buffer;
132            },
133            // XXX(collares): To fully support dynamically updating loop bounds,
134            // Must truncate self.buffer_pos if it is now outside the loop.
135            AudioBufferSourceNodeMessage::SetLoopEnabled(loop_enabled) => {
136                self.loop_enabled = loop_enabled
137            },
138            AudioBufferSourceNodeMessage::SetLoopEnd(loop_end) => self.loop_end = Some(loop_end),
139            AudioBufferSourceNodeMessage::SetLoopStart(loop_start) => {
140                self.loop_start = Some(loop_start)
141            },
142            AudioBufferSourceNodeMessage::SetStartParams(when, offset, duration) => {
143                self.start_when = when;
144                self.start_offset = offset;
145                self.start_duration = duration;
146            },
147        }
148    }
149}
150
151impl AudioNodeEngine for AudioBufferSourceNode {
152    fn node_type(&self) -> AudioNodeType {
153        AudioNodeType::AudioBufferSourceNode
154    }
155
156    fn input_count(&self) -> u32 {
157        0
158    }
159
160    fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk {
161        debug_assert!(inputs.is_empty());
162
163        if self.buffer.is_none() {
164            inputs.blocks.push(Default::default());
165            return inputs;
166        }
167
168        let (start_at, stop_at) = match self.should_play_at(info.frame) {
169            ShouldPlay::No => {
170                inputs.blocks.push(Default::default());
171                return inputs;
172            },
173            ShouldPlay::Between(start, end) => (start.0 as usize, end.0 as usize),
174        };
175
176        let buffer = self.buffer.as_ref().unwrap();
177
178        let (mut actual_loop_start, mut actual_loop_end) = (0., buffer.len() as f64);
179        if self.loop_enabled {
180            let loop_start = self.loop_start.unwrap_or(0.);
181            let loop_end = self.loop_end.unwrap_or(0.);
182
183            if loop_start >= 0. && loop_end > loop_start {
184                actual_loop_start = loop_start * (buffer.sample_rate as f64);
185                actual_loop_end = loop_end * (buffer.sample_rate as f64);
186            }
187        }
188
189        // https://webaudio.github.io/web-audio-api/#computedplaybackrate
190        self.playback_rate.update(info, Tick(0));
191        self.detune.update(info, Tick(0));
192        // computed_playback_rate can be negative or zero.
193        let computed_playback_rate =
194            self.playback_rate.value() as f64 * (2.0_f64).powf(self.detune.value() as f64 / 1200.);
195        let forward = computed_playback_rate >= 0.;
196
197        if !self.initialized_pos {
198            self.initialized_pos = true;
199
200            // Apply the offset and duration parameters passed to start. We handle
201            // this here because the buffer may be set after Start() gets called, so
202            // this might be the first time we know the buffer's sample rate.
203            if let Some(start_offset) = self.start_offset {
204                self.buffer_pos = start_offset * (buffer.sample_rate as f64);
205                if self.buffer_pos < 0. {
206                    self.buffer_pos = 0.
207                } else if self.buffer_pos > buffer.len() as f64 {
208                    self.buffer_pos = buffer.len() as f64;
209                }
210            }
211
212            if self.loop_enabled {
213                if forward && self.buffer_pos >= actual_loop_end {
214                    self.buffer_pos = actual_loop_start;
215                }
216                // https://github.com/WebAudio/web-audio-api/issues/2031
217                if !forward && self.buffer_pos < actual_loop_start {
218                    self.buffer_pos = actual_loop_end;
219                }
220            }
221
222            if let Some(start_duration) = self.start_duration {
223                self.buffer_duration = start_duration * (buffer.sample_rate as f64);
224            }
225
226            // start_when can be subsample accurate. Correct buffer_pos.
227            //
228            // XXX(collares): What happens to "start_when" if the buffer gets
229            // set after Start()?
230            // XXX(collares): Need a better way to distingush between Start()
231            // being called with "when" in the past (in which case "when" must
232            // be ignored) and Start() being called with "when" in the future.
233            // This can now make a difference if "when" shouldn't be ignored
234            // but falls after the last frame of the previous quantum.
235            if self.start_when > info.time - 1. / info.sample_rate as f64 {
236                let first_time = info.time + start_at as f64 / info.sample_rate as f64;
237                if self.start_when <= first_time {
238                    let subsample_offset = (first_time - self.start_when) *
239                        (buffer.sample_rate as f64) *
240                        computed_playback_rate;
241                    self.buffer_pos += subsample_offset;
242                    self.buffer_duration -= subsample_offset.abs();
243                }
244            }
245        }
246
247        let mut buffer_offset_per_tick =
248            computed_playback_rate * (buffer.sample_rate as f64 / info.sample_rate as f64);
249
250        // WebAudio §1.9.5: "Setting the loop attribute to true causes playback of
251        // the region of the buffer defined by the endpoints loopStart and loopEnd
252        // to continue indefinitely, once any part of the looped region has been
253        // played. While loop remains true, looped playback will continue until one
254        // of the following occurs:
255        //  * stop() is called,
256        //  * the scheduled stop time has been reached,
257        //  * the duration has been exceeded, if start() was called with a duration value."
258        // Even with extreme playback rates we must stay inside the loop body, so wrap
259        // the per-tick delta instead of bailing.
260        if self.loop_enabled && actual_loop_end > actual_loop_start {
261            let loop_length = actual_loop_end - actual_loop_start;
262            if loop_length > 0. {
263                let step = buffer_offset_per_tick.abs();
264                if step >= loop_length {
265                    let mut wrapped = step.rem_euclid(loop_length);
266                    if wrapped == 0. {
267                        wrapped = loop_length;
268                    }
269                    buffer_offset_per_tick = wrapped.copysign(buffer_offset_per_tick);
270                }
271            }
272        }
273
274        // We will output at most this many frames (fewer if we run out of data).
275        let frames_to_output = stop_at - start_at;
276
277        // Fast path for the case where we can just copy FRAMES_PER_BLOCK
278        // frames straight from the buffer.
279        if frames_to_output == FRAMES_PER_BLOCK.0 as usize &&
280            forward &&
281            buffer_offset_per_tick == 1. &&
282            self.buffer_pos.trunc() == self.buffer_pos &&
283            self.buffer_pos + (FRAMES_PER_BLOCK.0 as f64) <= actual_loop_end &&
284            FRAMES_PER_BLOCK.0 as f64 <= self.buffer_duration
285        {
286            let mut block = Block::empty();
287            let pos = self.buffer_pos as usize;
288
289            for chan in 0..buffer.chans() {
290                block.push_chan(&buffer.buffers[chan as usize][pos..(pos + frames_to_output)]);
291            }
292
293            inputs.blocks.push(block);
294            self.buffer_pos += FRAMES_PER_BLOCK.0 as f64;
295            self.buffer_duration -= FRAMES_PER_BLOCK.0 as f64;
296        } else {
297            // Slow path, with interpolation.
298            let mut block = Block::default();
299            block.repeat(buffer.chans());
300            block.explicit_repeat();
301
302            debug_assert!(buffer.chans() > 0);
303
304            for chan in 0..buffer.chans() {
305                let data = block.data_chan_mut(chan);
306                let (_, data) = data.split_at_mut(start_at);
307                let (data, _) = data.split_at_mut(frames_to_output);
308
309                let mut pos = self.buffer_pos;
310                let mut duration = self.buffer_duration;
311
312                for sample in data {
313                    if duration <= 0. {
314                        break;
315                    }
316
317                    if self.loop_enabled {
318                        if forward && pos >= actual_loop_end {
319                            pos -= actual_loop_end - actual_loop_start;
320                        } else if !forward && pos < actual_loop_start {
321                            pos += actual_loop_end - actual_loop_start;
322                        }
323                    } else if pos < 0. || pos >= buffer.len() as f64 {
324                        break;
325                    }
326
327                    *sample = buffer.interpolate(chan, pos);
328                    pos += buffer_offset_per_tick;
329                    duration -= buffer_offset_per_tick.abs();
330                }
331
332                // This is the last channel, update parameters.
333                if chan == buffer.chans() - 1 {
334                    self.buffer_pos = pos;
335                    self.buffer_duration = duration;
336                }
337            }
338
339            inputs.blocks.push(block);
340        }
341
342        if !self.loop_enabled && (self.buffer_pos < 0. || self.buffer_pos >= buffer.len() as f64) ||
343            self.buffer_duration <= 0.
344        {
345            self.maybe_trigger_onended_callback();
346        }
347
348        inputs
349    }
350
351    fn get_param(&mut self, id: ParamType) -> &mut Param {
352        match id {
353            ParamType::PlaybackRate => &mut self.playback_rate,
354            ParamType::Detune => &mut self.detune,
355            _ => panic!("Unknown param {:?} for AudioBufferSourceNode", id),
356        }
357    }
358
359    make_message_handler!(
360        AudioBufferSourceNode: handle_message,
361        AudioScheduledSourceNode: handle_source_node_message
362    );
363}
364
365#[derive(Debug, Clone, MallocSizeOf)]
366pub struct AudioBuffer {
367    /// Invariant: all buffers must be of the same length
368    pub buffers: Vec<Vec<f32>>,
369    pub sample_rate: f32,
370}
371
372impl AudioBuffer {
373    pub fn new(chan: u8, len: usize, sample_rate: f32) -> Self {
374        assert!(chan > 0);
375        let mut buffers = Vec::with_capacity(chan as usize);
376        let single = vec![0.; len];
377        buffers.resize(chan as usize, single);
378        AudioBuffer {
379            buffers,
380            sample_rate,
381        }
382    }
383
384    pub fn from_buffers(buffers: Vec<Vec<f32>>, sample_rate: f32) -> Self {
385        for buf in &buffers {
386            assert_eq!(buf.len(), buffers[0].len())
387        }
388
389        Self {
390            buffers,
391            sample_rate,
392        }
393    }
394
395    pub fn from_buffer(buffer: Vec<f32>, sample_rate: f32) -> Self {
396        AudioBuffer::from_buffers(vec![buffer], sample_rate)
397    }
398
399    pub fn len(&self) -> usize {
400        self.buffers[0].len()
401    }
402
403    pub fn is_empty(&self) -> bool {
404        self.len() == 0
405    }
406
407    pub fn chans(&self) -> u8 {
408        self.buffers.len() as u8
409    }
410
411    // XXX(collares): There are better fast interpolation algorithms.
412    // Firefox uses (via Speex's resampler) the algorithm described in
413    // https://ccrma.stanford.edu/~jos/resample/resample.pdf
414    // There are Rust bindings: https://github.com/rust-av/speexdsp-rs
415    pub fn interpolate(&self, chan: u8, pos: f64) -> f32 {
416        debug_assert!(pos >= 0. && pos < self.len() as f64);
417
418        let prev = pos.floor() as usize;
419        let offset = pos - pos.floor();
420        match self.buffers[chan as usize].get(prev + 1) {
421            Some(next_sample) => {
422                ((1. - offset) * (self.buffers[chan as usize][prev] as f64) +
423                    offset * (*next_sample as f64)) as f32
424            },
425            _ => {
426                // linear extrapolation of two prev samples if there are two
427                if prev > 0 {
428                    ((1. + offset) * (self.buffers[chan as usize][prev] as f64) -
429                        offset * (self.buffers[chan as usize][prev - 1] as f64))
430                        as f32
431                } else {
432                    self.buffers[chan as usize][prev]
433                }
434            },
435        }
436    }
437
438    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
439        &mut self.buffers[chan as usize]
440    }
441}
servo_media_audio/buffer_source_node.rs

servo_media_audio/
buffer_source_node.rs
