servo_media_audio/
block.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::f32::consts::SQRT_2;
6use std::mem;
7use std::ops::*;
8
9use byte_slice_cast::*;
10use euclid::default::Vector3D;
11use smallvec::{SmallVec, smallvec};
12
13use crate::graph::{PortIndex, PortKind};
14use crate::node::ChannelInterpretation;
15
16// defined by spec
17// https://webaudio.github.io/web-audio-api/#render-quantum
18pub const FRAMES_PER_BLOCK: Tick = Tick(128);
19pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize;
20
21/// A tick, i.e. the time taken for a single frame
22#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
23pub struct Tick(pub u64);
24
25/// A collection of blocks received as input by a node
26/// or outputted by a node.
27///
28/// This will usually be a single block.
29///
30/// Some nodes have multiple inputs or outputs, which is
31/// where this becomes useful. Source nodes have an input
32/// of an empty chunk.
33pub struct Chunk {
34    pub blocks: SmallVec<[Block; 1]>,
35}
36
37impl Default for Chunk {
38    fn default() -> Self {
39        Chunk {
40            blocks: SmallVec::new(),
41        }
42    }
43}
44
45impl Chunk {
46    pub fn is_empty(&self) -> bool {
47        self.blocks.is_empty()
48    }
49
50    pub fn len(&self) -> usize {
51        self.blocks.len()
52    }
53
54    pub fn explicit_silence() -> Self {
55        let mut block = Block::default();
56        block.explicit_silence();
57        let blocks = smallvec![block];
58        Self { blocks }
59    }
60}
61
62/// We render audio in blocks of size FRAMES_PER_BLOCK
63///
64/// A single block may contain multiple channels
65#[derive(Clone, Serialize, Deserialize, Debug)]
66pub struct Block {
67    /// The number of channels in this block
68    channels: u8,
69    /// This is an optimization which means that the buffer is representing multiple channels with the
70    /// same content at once. Happens when audio is upmixed or when a source like
71    /// an oscillator node has multiple channel outputs
72    repeat: bool,
73    /// If this vector is empty, it is a shorthand for "silence"
74    /// It is possible to obtain an explicitly silent buffer via .explicit_silence()
75    ///
76    /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true,
77    /// in which case it will be of length FRAMES_PER_BLOCK
78    buffer: Vec<f32>,
79}
80
81impl Default for Block {
82    fn default() -> Self {
83        Block {
84            channels: 1,
85            repeat: false,
86            buffer: Vec::new(),
87        }
88    }
89}
90
91impl Block {
92    /// Empty block with no channels, for pushing
93    /// new channels to.
94    ///
95    /// Must be used with push_chan
96    pub fn empty() -> Self {
97        Block {
98            channels: 0,
99            ..Default::default()
100        }
101    }
102
103    pub fn for_channels_explicit(channels: u8) -> Self {
104        Block {
105            channels,
106            repeat: false,
107            buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize],
108        }
109    }
110
111    /// This provides the entire buffer as a mutable slice of u8
112    pub fn as_mut_byte_slice(&mut self) -> &mut [u8] {
113        self.data_mut().as_mut_byte_slice()
114    }
115
116    pub fn for_vec(buffer: Vec<f32>) -> Self {
117        assert!(buffer.len() % FRAMES_PER_BLOCK_USIZE == 0);
118        Block {
119            channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8,
120            repeat: false,
121            buffer,
122        }
123    }
124
125    /// Zero-gain sum with another buffer
126    ///
127    /// Used after mixing multiple inputs to a single port
128    pub fn sum(mut self, mut other: Self) -> Self {
129        if self.is_silence() {
130            other
131        } else if other.is_silence() {
132            self
133        } else {
134            debug_assert_eq!(self.channels, other.channels);
135            if self.repeat ^ other.repeat {
136                self.explicit_repeat();
137                other.explicit_repeat();
138            }
139            debug_assert_eq!(self.buffer.len(), other.buffer.len());
140            for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) {
141                *a += b
142            }
143            self
144        }
145    }
146
147    /// If this is in "silence" mode without a buffer, allocate a silent buffer
148    pub fn explicit_silence(&mut self) {
149        if self.buffer.is_empty() {
150            self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.);
151            self.repeat = true;
152        }
153    }
154
155    /// This provides the entire buffer as a mutable slice of f32
156    pub fn data_mut(&mut self) -> &mut [f32] {
157        self.explicit_silence();
158        &mut self.buffer
159    }
160
161    pub fn explicit_repeat(&mut self) {
162        if self.repeat {
163            debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE);
164            if self.channels > 1 {
165                let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize);
166                for _ in 0..self.channels {
167                    new.extend(&self.buffer)
168                }
169
170                self.buffer = new;
171            }
172            self.repeat = false;
173        } else if self.is_silence() {
174            self.buffer
175                .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.);
176        }
177    }
178
179    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
180        self.explicit_repeat();
181        let start = chan as usize * FRAMES_PER_BLOCK_USIZE;
182        &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE]
183    }
184
185    #[inline]
186    pub fn data_chan(&self, chan: u8) -> &[f32] {
187        debug_assert!(
188            !self.is_silence(),
189            "data_chan doesn't work with silent buffers"
190        );
191        let offset = if self.repeat {
192            0
193        } else {
194            chan as usize * FRAMES_PER_BLOCK_USIZE
195        };
196        &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE]
197    }
198
199    pub fn take(&mut self) -> Block {
200        let new = Block {
201            channels: self.channels,
202            ..Default::default()
203        };
204        mem::replace(self, new)
205    }
206
207    pub fn chan_count(&self) -> u8 {
208        self.channels
209    }
210
211    pub fn iter(&mut self) -> FrameIterator<'_> {
212        FrameIterator::new(self)
213    }
214
215    pub fn is_silence(&self) -> bool {
216        self.buffer.is_empty()
217    }
218
219    pub fn is_repeat(&self) -> bool {
220        self.repeat
221    }
222
223    pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 {
224        if self.is_silence() {
225            0.
226        } else {
227            self.data_chan(chan)[frame]
228        }
229    }
230
231    pub fn push_chan(&mut self, data: &[f32]) {
232        assert!(!self.repeat);
233        assert!(!self.is_silence() || self.channels == 0);
234        assert!(data.len() == FRAMES_PER_BLOCK_USIZE);
235        self.buffer.extend(data);
236        self.channels += 1;
237    }
238
239    /// upmix/downmix the channels if necessary
240    ///
241    /// Currently only supports upmixing from 1
242    pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) {
243        // If we're not changing the number of channels, we
244        // don't actually need to mix
245        if self.channels == channels {
246            return;
247        }
248
249        // Silent buffers stay silent
250        if self.is_silence() {
251            self.channels = channels;
252            return;
253        }
254
255        if interpretation == ChannelInterpretation::Discrete {
256            // discrete downmixes by truncation, upmixes by adding
257            // silent channels
258
259            // If we're discrete, have a repeat, and are downmixing,
260            // just truncate by changing the channel value
261            if self.repeat && self.channels > channels {
262                self.channels = channels;
263            } else {
264                // otherwise resize the buffer, silent-filling when necessary
265                self.resize_silence(channels);
266            }
267        } else {
268            // For speakers, we have to do special things based on the
269            // interpretation of the channels for each kind of speakers
270
271            // The layout of each speaker kind is:
272            //
273            // - Mono: [The mono channel]
274            // - Stereo: [L, R]
275            // - Quad: [L, R, SL, SR]
276            // - 5.1: [L, R, C, LFE, SL, SR]
277
278            match (self.channels, channels) {
279                // Upmixing
280                // https://webaudio.github.io/web-audio-api/#UpMix-sub
281
282                // mono
283                (1, 2) => {
284                    // output.{L, R} = input
285                    self.repeat(2);
286                },
287                (1, 4) => {
288                    // output.{L, R} = input
289                    self.repeat(2);
290                    // output.{SL, SR} = 0
291                    self.resize_silence(4);
292                },
293                (1, 6) => {
294                    let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE);
295                    // output.{L, R} = 0
296                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
297                    // output.C = input
298                    v.extend(&self.buffer);
299                    self.buffer = v;
300                    // output.{LFE, SL, SR} = 0
301                    self.resize_silence(6);
302                },
303
304                // stereo
305                (2, 4) | (2, 6) => {
306                    // output.{L, R} = input.{L, R}
307                    // (5.1) output.{C, LFE} = 0
308                    // output.{SL, SR} = 0
309                    self.resize_silence(channels);
310                },
311
312                // quad
313                (4, 6) => {
314                    // we can avoid this and instead calculate offsets
315                    // based off whether or not this is `repeat`, but
316                    // a `repeat` quad block should be rare
317                    self.explicit_repeat();
318
319                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
320                    // output.{L, R} = input.{L, R}
321                    v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]);
322                    // output.{C, LFE} = 0
323                    v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.);
324                    // output.{SL, R} = input.{SL, SR}
325                    v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]);
326                    self.buffer = v;
327                    self.channels = channels;
328                },
329
330                // Downmixing
331                // https://webaudio.github.io/web-audio-api/#down-mix
332
333                // mono
334                (2, 1) => {
335                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
336                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
337                        // output = 0.5 * (input.L + input.R);
338                        let o =
339                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1));
340                        v.push(o);
341                    }
342                    self.buffer = v;
343                    self.channels = 1;
344                    self.repeat = false;
345                },
346                (4, 1) => {
347                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
348                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
349                        // output = 0.5 * (input.L + input.R + input.SL + input.SR);
350                        let o = 0.25 *
351                            (self.data_chan_frame(frame, 0) +
352                                self.data_chan_frame(frame, 1) +
353                                self.data_chan_frame(frame, 2) +
354                                self.data_chan_frame(frame, 3));
355                        v.push(o);
356                    }
357                    self.buffer = v;
358                    self.channels = 1;
359                    self.repeat = false;
360                },
361                (6, 1) => {
362                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
363                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
364                        // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR)
365                        let o =
366                            // sqrt(0.5) * (input.L + input.R)
367                            SQRT_2 * (self.data_chan_frame(frame, 0) +
368                                      self.data_chan_frame(frame, 1)) +
369                            // input.C
370                            self.data_chan_frame(frame, 2) +
371                            // (ignore LFE)
372                            // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE]
373                            // 0.5 * (input.SL + input.SR)
374                            0.5 * (self.data_chan_frame(frame, 4) +
375                                   self.data_chan_frame(frame, 5));
376                        v.push(o);
377                    }
378                    self.buffer = v;
379                    self.channels = 1;
380                    self.repeat = false;
381                },
382
383                // stereo
384                (4, 2) => {
385                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
386                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
387                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
388                        // output.L = 0.5 * (input.L + input.SL)
389                        v[frame] =
390                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2));
391                        // output.R = 0.5 * (input.R + input.SR)
392                        v[frame + FRAMES_PER_BLOCK_USIZE] =
393                            0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3));
394                    }
395                    self.buffer = v;
396                    self.channels = 2;
397                    self.repeat = false;
398                },
399                (6, 2) => {
400                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
401                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
402                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
403                        // output.L = L + sqrt(0.5) * (input.C + input.SL)
404                        v[frame] = self.data_chan_frame(frame, 0) +
405                            SQRT_2 *
406                                (self.data_chan_frame(frame, 2) +
407                                    self.data_chan_frame(frame, 4));
408                        // output.R = R + sqrt(0.5) * (input.C + input.SR)
409                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
410                            SQRT_2 *
411                                (self.data_chan_frame(frame, 2) +
412                                    self.data_chan_frame(frame, 5));
413                    }
414                    self.buffer = v;
415                    self.channels = 2;
416                    self.repeat = false;
417                },
418
419                // quad
420                (6, 4) => {
421                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
422                    v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.);
423                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
424                        // output.L = L + sqrt(0.5) * input.C
425                        v[frame] = self.data_chan_frame(frame, 0) +
426                            SQRT_2 * self.data_chan_frame(frame, 2);
427                        // output.R = R + sqrt(0.5) * input.C
428                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) +
429                            SQRT_2 * self.data_chan_frame(frame, 2);
430                        // output.SL = input.SL
431                        v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4);
432                        // output.SR = input.SR
433                        v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5);
434                    }
435                    self.buffer = v;
436                    self.channels = 4;
437                    self.repeat = false;
438                },
439
440                // If it's not a known kind of speaker configuration, treat as
441                // discrete
442                _ => {
443                    self.mix(channels, ChannelInterpretation::Discrete);
444                },
445            }
446            debug_assert!(self.channels == channels);
447        }
448    }
449
450    /// Resize to add or remove channels, fill extra channels with silence
451    pub fn resize_silence(&mut self, channels: u8) {
452        self.explicit_repeat();
453        self.buffer
454            .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.);
455        self.channels = channels;
456    }
457
458    /// Take a single-channel block and repeat the
459    /// channel
460    pub fn repeat(&mut self, channels: u8) {
461        debug_assert!(self.channels == 1);
462        self.channels = channels;
463        if !self.is_silence() {
464            self.repeat = true;
465        }
466    }
467
468    pub fn interleave(&mut self) -> Vec<f32> {
469        self.explicit_repeat();
470        let mut vec = Vec::with_capacity(self.buffer.len());
471        // FIXME this isn't too efficient
472        vec.resize(self.buffer.len(), 0.);
473        for frame in 0..FRAMES_PER_BLOCK_USIZE {
474            let channels = self.channels as usize;
475            for chan in 0..channels {
476                vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame]
477            }
478        }
479        vec
480    }
481
482    pub fn is_empty(&self) -> bool {
483        self.buffer.is_empty()
484    }
485
486    /// Get the position, forward, and up vectors for a given
487    /// AudioListener-produced block
488    pub fn listener_data(&self, frame: Tick) -> (Vector3D<f32>, Vector3D<f32>, Vector3D<f32>) {
489        let frame = frame.0 as usize;
490        (
491            Vector3D::new(
492                self.data_chan_frame(frame, 0),
493                self.data_chan_frame(frame, 1),
494                self.data_chan_frame(frame, 2),
495            ),
496            Vector3D::new(
497                self.data_chan_frame(frame, 3),
498                self.data_chan_frame(frame, 4),
499                self.data_chan_frame(frame, 5),
500            ),
501            Vector3D::new(
502                self.data_chan_frame(frame, 6),
503                self.data_chan_frame(frame, 7),
504                self.data_chan_frame(frame, 8),
505            ),
506        )
507    }
508}
509
510/// An iterator over frames in a block
511pub struct FrameIterator<'a> {
512    frame: Tick,
513    block: &'a mut Block,
514}
515
516impl<'a> FrameIterator<'a> {
517    #[inline]
518    pub fn new(block: &'a mut Block) -> Self {
519        FrameIterator {
520            frame: Tick(0),
521            block,
522        }
523    }
524
525    /// Advance the iterator
526    ///
527    /// We can't implement Iterator since it doesn't support
528    /// streaming iterators, but we can call `while let Some(frame) = iter.next()`
529    /// here
530    #[inline]
531    pub fn next<'b>(&'b mut self) -> Option<FrameRef<'b>> {
532        let curr = self.frame;
533        if curr < FRAMES_PER_BLOCK {
534            self.frame.advance();
535            Some(FrameRef {
536                frame: curr,
537                block: self.block,
538            })
539        } else {
540            None
541        }
542    }
543}
544
545/// A reference to a frame
546pub struct FrameRef<'a> {
547    frame: Tick,
548    block: &'a mut Block,
549}
550
551impl<'a> FrameRef<'a> {
552    #[inline]
553    pub fn tick(&self) -> Tick {
554        self.frame
555    }
556
557    /// Given a block and a function `f`, mutate the frame through all channels with `f`
558    ///
559    /// Use this when you plan to do the same operation for each channel.
560    /// (Helpers for the other cases will eventually exist)
561    ///
562    /// Block must not be silence
563    ///
564    /// The second parameter to f is the channel number, 0 in case of a repeat()
565    #[inline]
566    pub fn mutate_with<F>(&mut self, mut f: F)
567    where
568        F: FnMut(&mut f32, u8),
569    {
570        debug_assert!(
571            !self.block.is_silence(),
572            "mutate_frame_with should not be called with a silenced block, \
573             call .explicit_silence() if you wish to use this"
574        );
575        if self.block.repeat {
576            f(&mut self.block.buffer[self.frame.0 as usize], 0)
577        } else {
578            for chan in 0..self.block.channels {
579                f(
580                    &mut self.block.buffer
581                        [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
582                    chan,
583                )
584            }
585        }
586    }
587}
588
589// operator impls
590
591impl<T: PortKind> IndexMut<PortIndex<T>> for Chunk {
592    fn index_mut(&mut self, i: PortIndex<T>) -> &mut Block {
593        if let PortIndex::Port(i) = i {
594            &mut self.blocks[i as usize]
595        } else {
596            panic!("attempted to index chunk with param")
597        }
598    }
599}
600
601impl<T: PortKind> Index<PortIndex<T>> for Chunk {
602    type Output = Block;
603    fn index(&self, i: PortIndex<T>) -> &Block {
604        if let PortIndex::Port(i) = i {
605            &self.blocks[i as usize]
606        } else {
607            panic!("attempted to index chunk with param")
608        }
609    }
610}
611
612impl Add<Tick> for Tick {
613    type Output = Tick;
614    fn add(self, other: Tick) -> Self {
615        self + other.0
616    }
617}
618
619impl AddAssign for Tick {
620    fn add_assign(&mut self, other: Tick) {
621        *self = *self + other
622    }
623}
624
625impl Sub<Tick> for Tick {
626    type Output = Tick;
627    fn sub(self, other: Tick) -> Self {
628        self - other.0
629    }
630}
631
632impl Add<u64> for Tick {
633    type Output = Tick;
634    fn add(self, other: u64) -> Self {
635        Tick(self.0 + other)
636    }
637}
638
639impl Sub<u64> for Tick {
640    type Output = Tick;
641    fn sub(self, other: u64) -> Self {
642        Tick(self.0 - other)
643    }
644}
645
646impl Div<f64> for Tick {
647    type Output = f64;
648    fn div(self, other: f64) -> f64 {
649        self.0 as f64 / other
650    }
651}
652
653impl Tick {
654    pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK;
655    const EPSILON: f64 = 1e-7;
656    pub fn from_time(time: f64, rate: f32) -> Tick {
657        Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64)
658    }
659
660    pub fn advance(&mut self) {
661        self.0 += 1;
662    }
663}
servo_media_audio/block.rs

servo_media_audio/
block.rs
