servo_media_audio/
block.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::f32::consts::SQRT_2;
6use std::mem;
7use std::ops::*;
8
9use byte_slice_cast::*;
10use euclid::default::Vector3D;
11use serde::{Deserialize, Serialize};
12use smallvec::{SmallVec, smallvec};
13
14use crate::audio_node::ChannelInterpretation;
15use crate::graph::{PortIndex, PortKind};
16
17// defined by spec
18// https://webaudio.github.io/web-audio-api/#render-quantum
19pub const FRAMES_PER_BLOCK: Tick = Tick(128);
20pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize;
21
22/// A tick, i.e. the time taken for a single frame
23#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
24pub struct Tick(pub u64);
25
26/// A collection of blocks received as input by a node
27/// or outputted by a node.
28///
29/// This will usually be a single block.
30///
31/// Some nodes have multiple inputs or outputs, which is
32/// where this becomes useful. Source nodes have an input
33/// of an empty chunk.
34pub struct Chunk {
35    pub blocks: SmallVec<[Block; 1]>,
36}
37
38impl Default for Chunk {
39    fn default() -> Self {
40        Chunk {
41            blocks: SmallVec::new(),
42        }
43    }
44}
45
46impl Chunk {
47    pub fn new(block: Block) -> Self {
48        Chunk {
49            blocks: smallvec![block; 1],
50        }
51    }
52
53    pub fn is_empty(&self) -> bool {
54        self.blocks.is_empty()
55    }
56
57    pub fn len(&self) -> usize {
58        self.blocks.len()
59    }
60
61    pub fn explicit_silence() -> Self {
62        let mut block = Block::default();
63        block.explicit_silence();
64        let blocks = smallvec![block];
65        Self { blocks }
66    }
67}
68
69/// We render audio in blocks of size FRAMES_PER_BLOCK
70///
71/// A single block may contain multiple channels
72#[derive(Clone, Serialize, Deserialize, Debug)]
73pub struct Block {
74    /// The number of channels in this block
75    channels: u8,
76    /// This is an optimization which means that the buffer is representing multiple channels with the
77    /// same content at once. Happens when audio is upmixed or when a source like
78    /// an oscillator node has multiple channel outputs
79    repeat: bool,
80    /// If this vector is empty, it is a shorthand for "silence"
81    /// It is possible to obtain an explicitly silent buffer via .explicit_silence()
82    ///
83    /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true,
84    /// in which case it will be of length FRAMES_PER_BLOCK
85    buffer: Vec<f32>,
86}
87
88impl Default for Block {
89    fn default() -> Self {
90        Block {
91            channels: 1,
92            repeat: false,
93            buffer: Vec::new(),
94        }
95    }
96}
97
98impl Block {
99    /// Empty block with no channels, for pushing
100    /// new channels to.
101    ///
102    /// Must be used with push_chan
103    pub fn empty() -> Self {
104        Block {
105            channels: 0,
106            ..Default::default()
107        }
108    }
109
110    pub fn for_channels_explicit(channels: u8) -> Self {
111        Block {
112            channels,
113            repeat: false,
114            buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize],
115        }
116    }
117
118    /// This provides the entire buffer as a mutable slice of u8
119    pub fn as_mut_byte_slice(&mut self) -> &mut [u8] {
120        self.data_mut().as_mut_byte_slice()
121    }
122
123    pub fn for_vec(buffer: Vec<f32>) -> Self {
124        assert!(buffer.len().is_multiple_of(FRAMES_PER_BLOCK_USIZE));
125        Block {
126            channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8,
127            repeat: false,
128            buffer,
129        }
130    }
131
132    /// Zero-gain sum with another buffer
133    ///
134    /// Used after mixing multiple inputs to a single port
135    pub fn sum(mut self, mut other: Self) -> Self {
136        if self.is_silence() {
137            other
138        } else if other.is_silence() {
139            self
140        } else {
141            debug_assert_eq!(self.channels, other.channels);
142            if self.repeat ^ other.repeat {
143                self.explicit_repeat();
144                other.explicit_repeat();
145            }
146            debug_assert_eq!(self.buffer.len(), other.buffer.len());
147            for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) {
148                *a += b
149            }
150            self
151        }
152    }
153
154    /// If this is in "silence" mode without a buffer, allocate a silent buffer
155    pub fn explicit_silence(&mut self) {
156        if self.buffer.is_empty() {
157            self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.);
158            self.repeat = true;
159        }
160    }
161
162    /// This provides the entire buffer as a mutable slice of f32
163    pub fn data_mut(&mut self) -> &mut [f32] {
164        self.explicit_silence();
165        &mut self.buffer
166    }
167
168    pub fn explicit_repeat(&mut self) {
169        if self.repeat {
170            debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE);
171            if self.channels > 1 {
172                let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize);
173                for _ in 0..self.channels {
174                    new.extend(&self.buffer)
175                }
176
177                self.buffer = new;
178            }
179            self.repeat = false;
180        } else if self.is_silence() {
181            self.buffer
182                .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.);
183        }
184    }
185
186    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
187        self.explicit_repeat();
188        let start = chan as usize * FRAMES_PER_BLOCK_USIZE;
189        &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE]
190    }
191
192    #[inline]
193    pub fn data_chan(&self, chan: u8) -> &[f32] {
194        debug_assert!(
195            !self.is_silence(),
196            "data_chan doesn't work with silent buffers"
197        );
198        let offset = if self.repeat {
199            0
200        } else {
201            chan as usize * FRAMES_PER_BLOCK_USIZE
202        };
203        &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE]
204    }
205
206    pub fn take(&mut self) -> Block {
207        let new = Block {
208            channels: self.channels,
209            ..Default::default()
210        };
211        mem::replace(self, new)
212    }
213
214    pub fn chan_count(&self) -> u8 {
215        self.channels
216    }
217
218    pub fn iter(&mut self) -> FrameIterator<'_> {
219        FrameIterator::new(self)
220    }
221
222    pub fn is_silence(&self) -> bool {
223        self.buffer.is_empty()
224    }
225
226    pub fn is_repeat(&self) -> bool {
227        self.repeat
228    }
229
230    pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 {
231        if self.is_silence() {
232            0.
233        } else {
234            self.data_chan(chan)[frame]
235        }
236    }
237
238    pub fn push_chan(&mut self, data: &[f32]) {
239        assert!(!self.repeat);
240        assert!(!self.is_silence() || self.channels == 0);
241        assert!(data.len() == FRAMES_PER_BLOCK_USIZE);
242        self.buffer.extend(data);
243        self.channels += 1;
244    }
245
246    /// upmix/downmix the channels if necessary
247    ///
248    /// Currently only supports upmixing from 1
249    pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) {
250        // If we're not changing the number of channels, we
251        // don't actually need to mix
252        if self.channels == channels {
253            return;
254        }
255
256        // Silent buffers stay silent
257        if self.is_silence() {
258            self.channels = channels;
259            return;
260        }
261
262        if interpretation == ChannelInterpretation::Discrete {
263            // discrete downmixes by truncation, upmixes by adding
264            // silent channels
265
266            // If we're discrete, have a repeat, and are downmixing,
267            // just truncate by changing the channel value
268            if self.repeat && self.channels > channels {
269                self.channels = channels;
270            } else {
271                // otherwise resize the buffer, silent-filling when necessary
272                self.resize_silence(channels);
273            }
274        } else {
275            // For speakers, we have to do special things based on the
276            // interpretation of the channels for each kind of speakers
277
278            // The layout of each speaker kind is:
279            //
280            // - Mono: [The mono channel]
281            // - Stereo: [L, R]
282            // - Quad: [L, R, SL, SR]
283            // - 5.1: [L, R, C, LFE, SL, SR]
284
285            match (self.channels, channels) {
286                // Upmixing
287                // https://webaudio.github.io/web-audio-api/#UpMix-sub
288
289                // mono
290                (1, 2) => {
291                    // output.{L, R} = input
292                    self.repeat(2);
293                },
294                (1, 4) => {
295                    // output.{L, R} = input
296                    self.repeat(2);
297                    // output.{SL, SR} = 0
298                    self.resize_silence(4);
299                },
300                (1, 6) => {
301                    let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE);
302                    // output.{L, R} = 0
303                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
304                    // output.C = input
305                    v.extend(&self.buffer);
306                    self.buffer = v;
307                    // output.{LFE, SL, SR} = 0
308                    self.resize_silence(6);
309                },
310
311                // stereo
312                (2, 4) | (2, 6) => {
313                    // output.{L, R} = input.{L, R}
314                    // (5.1) output.{C, LFE} = 0
315                    // output.{SL, SR} = 0
316                    self.resize_silence(channels);
317                },
318
319                // quad
320                (4, 6) => {
321                    // we can avoid this and instead calculate offsets
322                    // based off whether or not this is `repeat`, but
323                    // a `repeat` quad block should be rare
324                    self.explicit_repeat();
325
326                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
327                    // output.{L, R} = input.{L, R}
328                    v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]);
329                    // output.{C, LFE} = 0
330                    v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.);
331                    // output.{SL, R} = input.{SL, SR}
332                    v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]);
333                    self.buffer = v;
334                    self.channels = channels;
335                },
336
337                // Downmixing
338                // https://webaudio.github.io/web-audio-api/#down-mix
339
340                // mono
341                (2, 1) => {
342                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
343                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
344                        // output = 0.5 * (input.L + input.R);
345                        let o =
346                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1));
347                        v.push(o);
348                    }
349                    self.buffer = v;
350                    self.channels = 1;
351                    self.repeat = false;
352                },
353                (4, 1) => {
354                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
355                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
356                        // output = 0.5 * (input.L + input.R + input.SL + input.SR);
357                        let o = 0.25 *
358                            (self.data_chan_frame(frame, 0) +
359                                self.data_chan_frame(frame, 1) +
360                                self.data_chan_frame(frame, 2) +
361                                self.data_chan_frame(frame, 3));
362                        v.push(o);
363                    }
364                    self.buffer = v;
365                    self.channels = 1;
366                    self.repeat = false;
367                },
368                (6, 1) => {
369                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
370                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
371                        // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR)
372                        let o =
373                            // sqrt(0.5) * (input.L + input.R)
374                            SQRT_2 * (self.data_chan_frame(frame, 0) +
375                                      self.data_chan_frame(frame, 1)) +
376                            // input.C
377                            self.data_chan_frame(frame, 2) +
378                            // (ignore LFE)
379                            // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE]
380                            // 0.5 * (input.SL + input.SR)
381                            0.5 * (self.data_chan_frame(frame, 4) +
382                                   self.data_chan_frame(frame, 5));
383                        v.push(o);
384                    }
385                    self.buffer = v;
386                    self.channels = 1;
387                    self.repeat = false;
388                },
389
390                // stereo
391                (4, 2) => {
392                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
393                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
394                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
395                        // output.L = 0.5 * (input.L + input.SL)
396                        v[frame] =
397                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2));
398                        // output.R = 0.5 * (input.R + input.SR)
399                        v[frame + FRAMES_PER_BLOCK_USIZE] =
400                            0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3));
401                    }
402                    self.buffer = v;
403                    self.channels = 2;
404                    self.repeat = false;
405                },
406                (6, 2) => {
407                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
408                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
409                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
410                        // output.L = L + sqrt(0.5) * (input.C + input.SL)
411                        v[frame] = self.data_chan_frame(frame, 0) +
412                            SQRT_2 *
413                                (self.data_chan_frame(frame, 2) +
414                                    self.data_chan_frame(frame, 4));
415                        // output.R = R + sqrt(0.5) * (input.C + input.SR)
416                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
417                            SQRT_2 *
418                                (self.data_chan_frame(frame, 2) +
419                                    self.data_chan_frame(frame, 5));
420                    }
421                    self.buffer = v;
422                    self.channels = 2;
423                    self.repeat = false;
424                },
425
426                // quad
427                (6, 4) => {
428                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
429                    v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.);
430                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
431                        // output.L = L + sqrt(0.5) * input.C
432                        v[frame] = self.data_chan_frame(frame, 0) +
433                            SQRT_2 * self.data_chan_frame(frame, 2);
434                        // output.R = R + sqrt(0.5) * input.C
435                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
436                            SQRT_2 * self.data_chan_frame(frame, 2);
437                        // output.SL = input.SL
438                        v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4);
439                        // output.SR = input.SR
440                        v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5);
441                    }
442                    self.buffer = v;
443                    self.channels = 4;
444                    self.repeat = false;
445                },
446
447                // If it's not a known kind of speaker configuration, treat as
448                // discrete
449                _ => {
450                    self.mix(channels, ChannelInterpretation::Discrete);
451                },
452            }
453            debug_assert!(self.channels == channels);
454        }
455    }
456
457    /// Resize to add or remove channels, fill extra channels with silence
458    pub fn resize_silence(&mut self, channels: u8) {
459        self.explicit_repeat();
460        self.buffer
461            .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.);
462        self.channels = channels;
463    }
464
465    /// Take a single-channel block and repeat the
466    /// channel
467    pub fn repeat(&mut self, channels: u8) {
468        debug_assert!(self.channels == 1);
469        self.channels = channels;
470        if !self.is_silence() {
471            self.repeat = true;
472        }
473    }
474
475    pub fn interleave(&mut self) -> Vec<f32> {
476        self.explicit_repeat();
477        let mut vec = Vec::with_capacity(self.buffer.len());
478        // FIXME this isn't too efficient
479        vec.resize(self.buffer.len(), 0.);
480        for frame in 0..FRAMES_PER_BLOCK_USIZE {
481            let channels = self.channels as usize;
482            for chan in 0..channels {
483                vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame]
484            }
485        }
486        vec
487    }
488
489    pub fn is_empty(&self) -> bool {
490        self.buffer.is_empty()
491    }
492
493    /// Get the position, forward, and up vectors for a given
494    /// AudioListener-produced block
495    pub fn listener_data(&self, frame: Tick) -> (Vector3D<f32>, Vector3D<f32>, Vector3D<f32>) {
496        let frame = frame.0 as usize;
497        (
498            Vector3D::new(
499                self.data_chan_frame(frame, 0),
500                self.data_chan_frame(frame, 1),
501                self.data_chan_frame(frame, 2),
502            ),
503            Vector3D::new(
504                self.data_chan_frame(frame, 3),
505                self.data_chan_frame(frame, 4),
506                self.data_chan_frame(frame, 5),
507            ),
508            Vector3D::new(
509                self.data_chan_frame(frame, 6),
510                self.data_chan_frame(frame, 7),
511                self.data_chan_frame(frame, 8),
512            ),
513        )
514    }
515}
516
517/// An iterator over frames in a block
518pub struct FrameIterator<'a> {
519    frame: Tick,
520    block: &'a mut Block,
521}
522
523impl<'a> FrameIterator<'a> {
524    #[inline]
525    pub fn new(block: &'a mut Block) -> Self {
526        FrameIterator {
527            frame: Tick(0),
528            block,
529        }
530    }
531
532    /// Advance the iterator
533    ///
534    /// We can't implement Iterator since it doesn't support
535    /// streaming iterators, but we can call `while let Some(frame) = iter.next()`
536    /// here
537    #[inline]
538    pub fn next<'b>(&'b mut self) -> Option<FrameRef<'b>> {
539        let curr = self.frame;
540        if curr < FRAMES_PER_BLOCK {
541            self.frame.advance();
542            Some(FrameRef {
543                frame: curr,
544                block: self.block,
545            })
546        } else {
547            None
548        }
549    }
550}
551
552/// A reference to a frame
553pub struct FrameRef<'a> {
554    frame: Tick,
555    block: &'a mut Block,
556}
557
558impl<'a> FrameRef<'a> {
559    #[inline]
560    pub fn tick(&self) -> Tick {
561        self.frame
562    }
563
564    /// Given a block and a function `f`, mutate the frame through all channels with `f`
565    ///
566    /// Use this when you plan to do the same operation for each channel.
567    /// (Helpers for the other cases will eventually exist)
568    ///
569    /// Block must not be silence
570    ///
571    /// The second parameter to f is the channel number, 0 in case of a repeat()
572    #[inline]
573    pub fn mutate_with<F>(&mut self, mut f: F)
574    where
575        F: FnMut(&mut f32, u8),
576    {
577        debug_assert!(
578            !self.block.is_silence(),
579            "mutate_frame_with should not be called with a silenced block, \
580             call .explicit_silence() if you wish to use this"
581        );
582        if self.block.repeat {
583            f(&mut self.block.buffer[self.frame.0 as usize], 0)
584        } else {
585            for chan in 0..self.block.channels {
586                f(
587                    &mut self.block.buffer
588                        [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
589                    chan,
590                )
591            }
592        }
593    }
594
595    #[inline]
596    /// Gets the data in the current frame of the block
597    /// Returned slice with one value per channel
598    pub fn get_frame(&self) -> Vec<f32> {
599        if self.block.repeat {
600            vec![self.block.buffer[self.frame.0 as usize]; self.block.channels as usize]
601        } else {
602            let mut frames = Vec::with_capacity(self.block.channels as usize);
603            for channel in 0..self.block.channels {
604                frames.push(
605                    self.block.buffer
606                        [channel as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
607                );
608            }
609            frames
610        }
611    }
612}
613
614// operator impls
615
616impl<T: PortKind> IndexMut<PortIndex<T>> for Chunk {
617    fn index_mut(&mut self, i: PortIndex<T>) -> &mut Block {
618        if let PortIndex::Port(i) = i {
619            &mut self.blocks[i as usize]
620        } else {
621            panic!("attempted to index chunk with param")
622        }
623    }
624}
625
626impl<T: PortKind> Index<PortIndex<T>> for Chunk {
627    type Output = Block;
628    fn index(&self, i: PortIndex<T>) -> &Block {
629        if let PortIndex::Port(i) = i {
630            &self.blocks[i as usize]
631        } else {
632            panic!("attempted to index chunk with param")
633        }
634    }
635}
636
637impl Add<Tick> for Tick {
638    type Output = Tick;
639    fn add(self, other: Tick) -> Self {
640        self + other.0
641    }
642}
643
644impl AddAssign for Tick {
645    fn add_assign(&mut self, other: Tick) {
646        *self = *self + other
647    }
648}
649
650impl Sub<Tick> for Tick {
651    type Output = Tick;
652    fn sub(self, other: Tick) -> Self {
653        self - other.0
654    }
655}
656
657impl Add<u64> for Tick {
658    type Output = Tick;
659    fn add(self, other: u64) -> Self {
660        Tick(self.0 + other)
661    }
662}
663
664impl Sub<u64> for Tick {
665    type Output = Tick;
666    fn sub(self, other: u64) -> Self {
667        Tick(self.0 - other)
668    }
669}
670
671impl Div<f64> for Tick {
672    type Output = f64;
673    fn div(self, other: f64) -> f64 {
674        self.0 as f64 / other
675    }
676}
677
678impl Tick {
679    pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK;
680    const EPSILON: f64 = 1e-7;
681    pub fn from_time(time: f64, rate: f32) -> Tick {
682        Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64)
683    }
684
685    pub fn advance(&mut self) {
686        self.0 += 1;
687    }
688}
servo_media_audio/block.rs

servo_media_audio/
block.rs
