servo_media_audio/
block.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::f32::consts::SQRT_2;
6use std::mem;
7use std::ops::*;
8
9use byte_slice_cast::*;
10use euclid::default::Vector3D;
11use serde::{Deserialize, Serialize};
12use smallvec::{SmallVec, smallvec};
13
14use crate::graph::{PortIndex, PortKind};
15use crate::node::ChannelInterpretation;
16
17// defined by spec
18// https://webaudio.github.io/web-audio-api/#render-quantum
19pub const FRAMES_PER_BLOCK: Tick = Tick(128);
20pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize;
21
22/// A tick, i.e. the time taken for a single frame
23#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
24pub struct Tick(pub u64);
25
26/// A collection of blocks received as input by a node
27/// or outputted by a node.
28///
29/// This will usually be a single block.
30///
31/// Some nodes have multiple inputs or outputs, which is
32/// where this becomes useful. Source nodes have an input
33/// of an empty chunk.
34pub struct Chunk {
35    pub blocks: SmallVec<[Block; 1]>,
36}
37
38impl Default for Chunk {
39    fn default() -> Self {
40        Chunk {
41            blocks: SmallVec::new(),
42        }
43    }
44}
45
46impl Chunk {
47    pub fn is_empty(&self) -> bool {
48        self.blocks.is_empty()
49    }
50
51    pub fn len(&self) -> usize {
52        self.blocks.len()
53    }
54
55    pub fn explicit_silence() -> Self {
56        let mut block = Block::default();
57        block.explicit_silence();
58        let blocks = smallvec![block];
59        Self { blocks }
60    }
61}
62
63/// We render audio in blocks of size FRAMES_PER_BLOCK
64///
65/// A single block may contain multiple channels
66#[derive(Clone, Serialize, Deserialize, Debug)]
67pub struct Block {
68    /// The number of channels in this block
69    channels: u8,
70    /// This is an optimization which means that the buffer is representing multiple channels with the
71    /// same content at once. Happens when audio is upmixed or when a source like
72    /// an oscillator node has multiple channel outputs
73    repeat: bool,
74    /// If this vector is empty, it is a shorthand for "silence"
75    /// It is possible to obtain an explicitly silent buffer via .explicit_silence()
76    ///
77    /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true,
78    /// in which case it will be of length FRAMES_PER_BLOCK
79    buffer: Vec<f32>,
80}
81
82impl Default for Block {
83    fn default() -> Self {
84        Block {
85            channels: 1,
86            repeat: false,
87            buffer: Vec::new(),
88        }
89    }
90}
91
92impl Block {
93    /// Empty block with no channels, for pushing
94    /// new channels to.
95    ///
96    /// Must be used with push_chan
97    pub fn empty() -> Self {
98        Block {
99            channels: 0,
100            ..Default::default()
101        }
102    }
103
104    pub fn for_channels_explicit(channels: u8) -> Self {
105        Block {
106            channels,
107            repeat: false,
108            buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize],
109        }
110    }
111
112    /// This provides the entire buffer as a mutable slice of u8
113    pub fn as_mut_byte_slice(&mut self) -> &mut [u8] {
114        self.data_mut().as_mut_byte_slice()
115    }
116
117    pub fn for_vec(buffer: Vec<f32>) -> Self {
118        assert!(buffer.len() % FRAMES_PER_BLOCK_USIZE == 0);
119        Block {
120            channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8,
121            repeat: false,
122            buffer,
123        }
124    }
125
126    /// Zero-gain sum with another buffer
127    ///
128    /// Used after mixing multiple inputs to a single port
129    pub fn sum(mut self, mut other: Self) -> Self {
130        if self.is_silence() {
131            other
132        } else if other.is_silence() {
133            self
134        } else {
135            debug_assert_eq!(self.channels, other.channels);
136            if self.repeat ^ other.repeat {
137                self.explicit_repeat();
138                other.explicit_repeat();
139            }
140            debug_assert_eq!(self.buffer.len(), other.buffer.len());
141            for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) {
142                *a += b
143            }
144            self
145        }
146    }
147
148    /// If this is in "silence" mode without a buffer, allocate a silent buffer
149    pub fn explicit_silence(&mut self) {
150        if self.buffer.is_empty() {
151            self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.);
152            self.repeat = true;
153        }
154    }
155
156    /// This provides the entire buffer as a mutable slice of f32
157    pub fn data_mut(&mut self) -> &mut [f32] {
158        self.explicit_silence();
159        &mut self.buffer
160    }
161
162    pub fn explicit_repeat(&mut self) {
163        if self.repeat {
164            debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE);
165            if self.channels > 1 {
166                let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize);
167                for _ in 0..self.channels {
168                    new.extend(&self.buffer)
169                }
170
171                self.buffer = new;
172            }
173            self.repeat = false;
174        } else if self.is_silence() {
175            self.buffer
176                .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.);
177        }
178    }
179
180    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
181        self.explicit_repeat();
182        let start = chan as usize * FRAMES_PER_BLOCK_USIZE;
183        &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE]
184    }
185
186    #[inline]
187    pub fn data_chan(&self, chan: u8) -> &[f32] {
188        debug_assert!(
189            !self.is_silence(),
190            "data_chan doesn't work with silent buffers"
191        );
192        let offset = if self.repeat {
193            0
194        } else {
195            chan as usize * FRAMES_PER_BLOCK_USIZE
196        };
197        &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE]
198    }
199
200    pub fn take(&mut self) -> Block {
201        let new = Block {
202            channels: self.channels,
203            ..Default::default()
204        };
205        mem::replace(self, new)
206    }
207
208    pub fn chan_count(&self) -> u8 {
209        self.channels
210    }
211
212    pub fn iter(&mut self) -> FrameIterator<'_> {
213        FrameIterator::new(self)
214    }
215
216    pub fn is_silence(&self) -> bool {
217        self.buffer.is_empty()
218    }
219
220    pub fn is_repeat(&self) -> bool {
221        self.repeat
222    }
223
224    pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 {
225        if self.is_silence() {
226            0.
227        } else {
228            self.data_chan(chan)[frame]
229        }
230    }
231
232    pub fn push_chan(&mut self, data: &[f32]) {
233        assert!(!self.repeat);
234        assert!(!self.is_silence() || self.channels == 0);
235        assert!(data.len() == FRAMES_PER_BLOCK_USIZE);
236        self.buffer.extend(data);
237        self.channels += 1;
238    }
239
240    /// upmix/downmix the channels if necessary
241    ///
242    /// Currently only supports upmixing from 1
243    pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) {
244        // If we're not changing the number of channels, we
245        // don't actually need to mix
246        if self.channels == channels {
247            return;
248        }
249
250        // Silent buffers stay silent
251        if self.is_silence() {
252            self.channels = channels;
253            return;
254        }
255
256        if interpretation == ChannelInterpretation::Discrete {
257            // discrete downmixes by truncation, upmixes by adding
258            // silent channels
259
260            // If we're discrete, have a repeat, and are downmixing,
261            // just truncate by changing the channel value
262            if self.repeat && self.channels > channels {
263                self.channels = channels;
264            } else {
265                // otherwise resize the buffer, silent-filling when necessary
266                self.resize_silence(channels);
267            }
268        } else {
269            // For speakers, we have to do special things based on the
270            // interpretation of the channels for each kind of speakers
271
272            // The layout of each speaker kind is:
273            //
274            // - Mono: [The mono channel]
275            // - Stereo: [L, R]
276            // - Quad: [L, R, SL, SR]
277            // - 5.1: [L, R, C, LFE, SL, SR]
278
279            match (self.channels, channels) {
280                // Upmixing
281                // https://webaudio.github.io/web-audio-api/#UpMix-sub
282
283                // mono
284                (1, 2) => {
285                    // output.{L, R} = input
286                    self.repeat(2);
287                },
288                (1, 4) => {
289                    // output.{L, R} = input
290                    self.repeat(2);
291                    // output.{SL, SR} = 0
292                    self.resize_silence(4);
293                },
294                (1, 6) => {
295                    let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE);
296                    // output.{L, R} = 0
297                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
298                    // output.C = input
299                    v.extend(&self.buffer);
300                    self.buffer = v;
301                    // output.{LFE, SL, SR} = 0
302                    self.resize_silence(6);
303                },
304
305                // stereo
306                (2, 4) | (2, 6) => {
307                    // output.{L, R} = input.{L, R}
308                    // (5.1) output.{C, LFE} = 0
309                    // output.{SL, SR} = 0
310                    self.resize_silence(channels);
311                },
312
313                // quad
314                (4, 6) => {
315                    // we can avoid this and instead calculate offsets
316                    // based off whether or not this is `repeat`, but
317                    // a `repeat` quad block should be rare
318                    self.explicit_repeat();
319
320                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
321                    // output.{L, R} = input.{L, R}
322                    v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]);
323                    // output.{C, LFE} = 0
324                    v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.);
325                    // output.{SL, R} = input.{SL, SR}
326                    v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]);
327                    self.buffer = v;
328                    self.channels = channels;
329                },
330
331                // Downmixing
332                // https://webaudio.github.io/web-audio-api/#down-mix
333
334                // mono
335                (2, 1) => {
336                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
337                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
338                        // output = 0.5 * (input.L + input.R);
339                        let o =
340                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1));
341                        v.push(o);
342                    }
343                    self.buffer = v;
344                    self.channels = 1;
345                    self.repeat = false;
346                },
347                (4, 1) => {
348                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
349                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
350                        // output = 0.5 * (input.L + input.R + input.SL + input.SR);
351                        let o = 0.25 *
352                            (self.data_chan_frame(frame, 0) +
353                                self.data_chan_frame(frame, 1) +
354                                self.data_chan_frame(frame, 2) +
355                                self.data_chan_frame(frame, 3));
356                        v.push(o);
357                    }
358                    self.buffer = v;
359                    self.channels = 1;
360                    self.repeat = false;
361                },
362                (6, 1) => {
363                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
364                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
365                        // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR)
366                        let o =
367                            // sqrt(0.5) * (input.L + input.R)
368                            SQRT_2 * (self.data_chan_frame(frame, 0) +
369                                      self.data_chan_frame(frame, 1)) +
370                            // input.C
371                            self.data_chan_frame(frame, 2) +
372                            // (ignore LFE)
373                            // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE]
374                            // 0.5 * (input.SL + input.SR)
375                            0.5 * (self.data_chan_frame(frame, 4) +
376                                   self.data_chan_frame(frame, 5));
377                        v.push(o);
378                    }
379                    self.buffer = v;
380                    self.channels = 1;
381                    self.repeat = false;
382                },
383
384                // stereo
385                (4, 2) => {
386                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
387                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
388                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
389                        // output.L = 0.5 * (input.L + input.SL)
390                        v[frame] =
391                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2));
392                        // output.R = 0.5 * (input.R + input.SR)
393                        v[frame + FRAMES_PER_BLOCK_USIZE] =
394                            0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3));
395                    }
396                    self.buffer = v;
397                    self.channels = 2;
398                    self.repeat = false;
399                },
400                (6, 2) => {
401                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
402                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
403                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
404                        // output.L = L + sqrt(0.5) * (input.C + input.SL)
405                        v[frame] = self.data_chan_frame(frame, 0) +
406                            SQRT_2 *
407                                (self.data_chan_frame(frame, 2) +
408                                    self.data_chan_frame(frame, 4));
409                        // output.R = R + sqrt(0.5) * (input.C + input.SR)
410                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
411                            SQRT_2 *
412                                (self.data_chan_frame(frame, 2) +
413                                    self.data_chan_frame(frame, 5));
414                    }
415                    self.buffer = v;
416                    self.channels = 2;
417                    self.repeat = false;
418                },
419
420                // quad
421                (6, 4) => {
422                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
423                    v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.);
424                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
425                        // output.L = L + sqrt(0.5) * input.C
426                        v[frame] = self.data_chan_frame(frame, 0) +
427                            SQRT_2 * self.data_chan_frame(frame, 2);
428                        // output.R = R + sqrt(0.5) * input.C
429                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
430                            SQRT_2 * self.data_chan_frame(frame, 2);
431                        // output.SL = input.SL
432                        v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4);
433                        // output.SR = input.SR
434                        v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5);
435                    }
436                    self.buffer = v;
437                    self.channels = 4;
438                    self.repeat = false;
439                },
440
441                // If it's not a known kind of speaker configuration, treat as
442                // discrete
443                _ => {
444                    self.mix(channels, ChannelInterpretation::Discrete);
445                },
446            }
447            debug_assert!(self.channels == channels);
448        }
449    }
450
451    /// Resize to add or remove channels, fill extra channels with silence
452    pub fn resize_silence(&mut self, channels: u8) {
453        self.explicit_repeat();
454        self.buffer
455            .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.);
456        self.channels = channels;
457    }
458
459    /// Take a single-channel block and repeat the
460    /// channel
461    pub fn repeat(&mut self, channels: u8) {
462        debug_assert!(self.channels == 1);
463        self.channels = channels;
464        if !self.is_silence() {
465            self.repeat = true;
466        }
467    }
468
469    pub fn interleave(&mut self) -> Vec<f32> {
470        self.explicit_repeat();
471        let mut vec = Vec::with_capacity(self.buffer.len());
472        // FIXME this isn't too efficient
473        vec.resize(self.buffer.len(), 0.);
474        for frame in 0..FRAMES_PER_BLOCK_USIZE {
475            let channels = self.channels as usize;
476            for chan in 0..channels {
477                vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame]
478            }
479        }
480        vec
481    }
482
483    pub fn is_empty(&self) -> bool {
484        self.buffer.is_empty()
485    }
486
487    /// Get the position, forward, and up vectors for a given
488    /// AudioListener-produced block
489    pub fn listener_data(&self, frame: Tick) -> (Vector3D<f32>, Vector3D<f32>, Vector3D<f32>) {
490        let frame = frame.0 as usize;
491        (
492            Vector3D::new(
493                self.data_chan_frame(frame, 0),
494                self.data_chan_frame(frame, 1),
495                self.data_chan_frame(frame, 2),
496            ),
497            Vector3D::new(
498                self.data_chan_frame(frame, 3),
499                self.data_chan_frame(frame, 4),
500                self.data_chan_frame(frame, 5),
501            ),
502            Vector3D::new(
503                self.data_chan_frame(frame, 6),
504                self.data_chan_frame(frame, 7),
505                self.data_chan_frame(frame, 8),
506            ),
507        )
508    }
509}
510
511/// An iterator over frames in a block
512pub struct FrameIterator<'a> {
513    frame: Tick,
514    block: &'a mut Block,
515}
516
517impl<'a> FrameIterator<'a> {
518    #[inline]
519    pub fn new(block: &'a mut Block) -> Self {
520        FrameIterator {
521            frame: Tick(0),
522            block,
523        }
524    }
525
526    /// Advance the iterator
527    ///
528    /// We can't implement Iterator since it doesn't support
529    /// streaming iterators, but we can call `while let Some(frame) = iter.next()`
530    /// here
531    #[inline]
532    pub fn next<'b>(&'b mut self) -> Option<FrameRef<'b>> {
533        let curr = self.frame;
534        if curr < FRAMES_PER_BLOCK {
535            self.frame.advance();
536            Some(FrameRef {
537                frame: curr,
538                block: self.block,
539            })
540        } else {
541            None
542        }
543    }
544}
545
546/// A reference to a frame
547pub struct FrameRef<'a> {
548    frame: Tick,
549    block: &'a mut Block,
550}
551
552impl<'a> FrameRef<'a> {
553    #[inline]
554    pub fn tick(&self) -> Tick {
555        self.frame
556    }
557
558    /// Given a block and a function `f`, mutate the frame through all channels with `f`
559    ///
560    /// Use this when you plan to do the same operation for each channel.
561    /// (Helpers for the other cases will eventually exist)
562    ///
563    /// Block must not be silence
564    ///
565    /// The second parameter to f is the channel number, 0 in case of a repeat()
566    #[inline]
567    pub fn mutate_with<F>(&mut self, mut f: F)
568    where
569        F: FnMut(&mut f32, u8),
570    {
571        debug_assert!(
572            !self.block.is_silence(),
573            "mutate_frame_with should not be called with a silenced block, \
574             call .explicit_silence() if you wish to use this"
575        );
576        if self.block.repeat {
577            f(&mut self.block.buffer[self.frame.0 as usize], 0)
578        } else {
579            for chan in 0..self.block.channels {
580                f(
581                    &mut self.block.buffer
582                        [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
583                    chan,
584                )
585            }
586        }
587    }
588}
589
590// operator impls
591
592impl<T: PortKind> IndexMut<PortIndex<T>> for Chunk {
593    fn index_mut(&mut self, i: PortIndex<T>) -> &mut Block {
594        if let PortIndex::Port(i) = i {
595            &mut self.blocks[i as usize]
596        } else {
597            panic!("attempted to index chunk with param")
598        }
599    }
600}
601
602impl<T: PortKind> Index<PortIndex<T>> for Chunk {
603    type Output = Block;
604    fn index(&self, i: PortIndex<T>) -> &Block {
605        if let PortIndex::Port(i) = i {
606            &self.blocks[i as usize]
607        } else {
608            panic!("attempted to index chunk with param")
609        }
610    }
611}
612
613impl Add<Tick> for Tick {
614    type Output = Tick;
615    fn add(self, other: Tick) -> Self {
616        self + other.0
617    }
618}
619
620impl AddAssign for Tick {
621    fn add_assign(&mut self, other: Tick) {
622        *self = *self + other
623    }
624}
625
626impl Sub<Tick> for Tick {
627    type Output = Tick;
628    fn sub(self, other: Tick) -> Self {
629        self - other.0
630    }
631}
632
633impl Add<u64> for Tick {
634    type Output = Tick;
635    fn add(self, other: u64) -> Self {
636        Tick(self.0 + other)
637    }
638}
639
640impl Sub<u64> for Tick {
641    type Output = Tick;
642    fn sub(self, other: u64) -> Self {
643        Tick(self.0 - other)
644    }
645}
646
647impl Div<f64> for Tick {
648    type Output = f64;
649    fn div(self, other: f64) -> f64 {
650        self.0 as f64 / other
651    }
652}
653
654impl Tick {
655    pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK;
656    const EPSILON: f64 = 1e-7;
657    pub fn from_time(time: f64, rate: f32) -> Tick {
658        Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64)
659    }
660
661    pub fn advance(&mut self) {
662        self.0 += 1;
663    }
664}
servo_media_audio/block.rs

servo_media_audio/
block.rs
