servo_media_audio/
block.rs

1use crate::graph::{PortIndex, PortKind};
2use crate::node::ChannelInterpretation;
3use byte_slice_cast::*;
4use euclid::default::Vector3D;
5use smallvec::{SmallVec, smallvec};
6use std::f32::consts::SQRT_2;
7use std::mem;
8use std::ops::*;
9
10// defined by spec
11// https://webaudio.github.io/web-audio-api/#render-quantum
12pub const FRAMES_PER_BLOCK: Tick = Tick(128);
13pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize;
14
15/// A tick, i.e. the time taken for a single frame
16#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
17pub struct Tick(pub u64);
18
19/// A collection of blocks received as input by a node
20/// or outputted by a node.
21///
22/// This will usually be a single block.
23///
24/// Some nodes have multiple inputs or outputs, which is
25/// where this becomes useful. Source nodes have an input
26/// of an empty chunk.
27pub struct Chunk {
28    pub blocks: SmallVec<[Block; 1]>,
29}
30
31impl Default for Chunk {
32    fn default() -> Self {
33        Chunk {
34            blocks: SmallVec::new(),
35        }
36    }
37}
38
39impl Chunk {
40    pub fn len(&self) -> usize {
41        self.blocks.len()
42    }
43
44    pub fn explicit_silence() -> Self {
45        let mut block = Block::default();
46        block.explicit_silence();
47        let blocks = smallvec![block];
48        Self { blocks }
49    }
50}
51
52/// We render audio in blocks of size FRAMES_PER_BLOCK
53///
54/// A single block may contain multiple channels
55#[derive(Clone, Serialize, Deserialize, Debug)]
56pub struct Block {
57    /// The number of channels in this block
58    channels: u8,
59    /// This is an optimization which means that the buffer is representing multiple channels with the
60    /// same content at once. Happens when audio is upmixed or when a source like
61    /// an oscillator node has multiple channel outputs
62    repeat: bool,
63    /// If this vector is empty, it is a shorthand for "silence"
64    /// It is possible to obtain an explicitly silent buffer via .explicit_silence()
65    ///
66    /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true,
67    /// in which case it will be of length FRAMES_PER_BLOCK
68    buffer: Vec<f32>,
69}
70
71impl Default for Block {
72    fn default() -> Self {
73        Block {
74            channels: 1,
75            repeat: false,
76            buffer: Vec::new(),
77        }
78    }
79}
80
81impl Block {
82    /// Empty block with no channels, for pushing
83    /// new channels to.
84    ///
85    /// Must be used with push_chan
86    pub fn empty() -> Self {
87        Block {
88            channels: 0,
89            ..Default::default()
90        }
91    }
92
93    pub fn for_channels_explicit(channels: u8) -> Self {
94        Block {
95            channels,
96            repeat: false,
97            buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize],
98        }
99    }
100
101    /// This provides the entire buffer as a mutable slice of u8
102    pub fn as_mut_byte_slice(&mut self) -> &mut [u8] {
103        self.data_mut().as_mut_byte_slice()
104    }
105
106    pub fn for_vec(buffer: Vec<f32>) -> Self {
107        assert!(buffer.len() % FRAMES_PER_BLOCK_USIZE == 0);
108        Block {
109            channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8,
110            repeat: false,
111            buffer,
112        }
113    }
114
115    /// Zero-gain sum with another buffer
116    ///
117    /// Used after mixing multiple inputs to a single port
118    pub fn sum(mut self, mut other: Self) -> Self {
119        if self.is_silence() {
120            other
121        } else if other.is_silence() {
122            self
123        } else {
124            debug_assert_eq!(self.channels, other.channels);
125            if self.repeat ^ other.repeat {
126                self.explicit_repeat();
127                other.explicit_repeat();
128            }
129            debug_assert_eq!(self.buffer.len(), other.buffer.len());
130            for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) {
131                *a += b
132            }
133            self
134        }
135    }
136
137    /// If this is in "silence" mode without a buffer, allocate a silent buffer
138    pub fn explicit_silence(&mut self) {
139        if self.buffer.is_empty() {
140            self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.);
141            self.repeat = true;
142        }
143    }
144
145    /// This provides the entire buffer as a mutable slice of f32
146    pub fn data_mut(&mut self) -> &mut [f32] {
147        self.explicit_silence();
148        &mut self.buffer
149    }
150
151    pub fn explicit_repeat(&mut self) {
152        if self.repeat {
153            debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE);
154            if self.channels > 1 {
155                let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize);
156                for _ in 0..self.channels {
157                    new.extend(&self.buffer)
158                }
159
160                self.buffer = new;
161            }
162            self.repeat = false;
163        } else if self.is_silence() {
164            self.buffer
165                .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.);
166        }
167    }
168
169    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
170        self.explicit_repeat();
171        let start = chan as usize * FRAMES_PER_BLOCK_USIZE;
172        &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE]
173    }
174
175    #[inline]
176    pub fn data_chan(&self, chan: u8) -> &[f32] {
177        debug_assert!(
178            !self.is_silence(),
179            "data_chan doesn't work with silent buffers"
180        );
181        let offset = if self.repeat {
182            0
183        } else {
184            chan as usize * FRAMES_PER_BLOCK_USIZE
185        };
186        &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE]
187    }
188
189    pub fn take(&mut self) -> Block {
190        let mut new = Block::default();
191        new.channels = self.channels;
192        mem::replace(self, new)
193    }
194
195    pub fn chan_count(&self) -> u8 {
196        self.channels
197    }
198
199    pub fn iter(&mut self) -> FrameIterator<'_> {
200        FrameIterator::new(self)
201    }
202
203    pub fn is_silence(&self) -> bool {
204        self.buffer.is_empty()
205    }
206
207    pub fn is_repeat(&self) -> bool {
208        self.repeat
209    }
210
211    pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 {
212        if self.is_silence() {
213            0.
214        } else {
215            self.data_chan(chan)[frame]
216        }
217    }
218
219    pub fn push_chan(&mut self, data: &[f32]) {
220        assert!(!self.repeat);
221        assert!(!self.is_silence() || self.channels == 0);
222        assert!(data.len() == FRAMES_PER_BLOCK_USIZE);
223        self.buffer.extend(data);
224        self.channels += 1;
225    }
226
227    /// upmix/downmix the channels if necessary
228    ///
229    /// Currently only supports upmixing from 1
230    pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) {
231        // If we're not changing the number of channels, we
232        // don't actually need to mix
233        if self.channels == channels {
234            return;
235        }
236
237        // Silent buffers stay silent
238        if self.is_silence() {
239            self.channels = channels;
240            return;
241        }
242
243        if interpretation == ChannelInterpretation::Discrete {
244            // discrete downmixes by truncation, upmixes by adding
245            // silent channels
246
247            // If we're discrete, have a repeat, and are downmixing,
248            // just truncate by changing the channel value
249            if self.repeat && self.channels > channels {
250                self.channels = channels;
251            } else {
252                // otherwise resize the buffer, silent-filling when necessary
253                self.resize_silence(channels);
254            }
255        } else {
256            // For speakers, we have to do special things based on the
257            // interpretation of the channels for each kind of speakers
258
259            // The layout of each speaker kind is:
260            //
261            // - Mono: [The mono channel]
262            // - Stereo: [L, R]
263            // - Quad: [L, R, SL, SR]
264            // - 5.1: [L, R, C, LFE, SL, SR]
265
266            match (self.channels, channels) {
267                // Upmixing
268                // https://webaudio.github.io/web-audio-api/#UpMix-sub
269
270                // mono
271                (1, 2) => {
272                    // output.{L, R} = input
273                    self.repeat(2);
274                },
275                (1, 4) => {
276                    // output.{L, R} = input
277                    self.repeat(2);
278                    // output.{SL, SR} = 0
279                    self.resize_silence(4);
280                },
281                (1, 6) => {
282                    let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE);
283                    // output.{L, R} = 0
284                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
285                    // output.C = input
286                    v.extend(&self.buffer);
287                    self.buffer = v;
288                    // output.{LFE, SL, SR} = 0
289                    self.resize_silence(6);
290                },
291
292                // stereo
293                (2, 4) | (2, 6) => {
294                    // output.{L, R} = input.{L, R}
295                    // (5.1) output.{C, LFE} = 0
296                    // output.{SL, SR} = 0
297                    self.resize_silence(channels);
298                },
299
300                // quad
301                (4, 6) => {
302                    // we can avoid this and instead calculate offsets
303                    // based off whether or not this is `repeat`, but
304                    // a `repeat` quad block should be rare
305                    self.explicit_repeat();
306
307                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
308                    // output.{L, R} = input.{L, R}
309                    v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]);
310                    // output.{C, LFE} = 0
311                    v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.);
312                    // output.{SL, R} = input.{SL, SR}
313                    v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]);
314                    self.buffer = v;
315                    self.channels = channels;
316                },
317
318                // Downmixing
319                // https://webaudio.github.io/web-audio-api/#down-mix
320
321                // mono
322                (2, 1) => {
323                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
324                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
325                        // output = 0.5 * (input.L + input.R);
326                        let o =
327                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1));
328                        v.push(o);
329                    }
330                    self.buffer = v;
331                    self.channels = 1;
332                    self.repeat = false;
333                },
334                (4, 1) => {
335                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
336                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
337                        // output = 0.5 * (input.L + input.R + input.SL + input.SR);
338                        let o = 0.25
339                            * (self.data_chan_frame(frame, 0)
340                                + self.data_chan_frame(frame, 1)
341                                + self.data_chan_frame(frame, 2)
342                                + self.data_chan_frame(frame, 3));
343                        v.push(o);
344                    }
345                    self.buffer = v;
346                    self.channels = 1;
347                    self.repeat = false;
348                },
349                (6, 1) => {
350                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
351                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
352                        // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR)
353                        let o =
354                            // sqrt(0.5) * (input.L + input.R)
355                            SQRT_2 * (self.data_chan_frame(frame, 0) +
356                                      self.data_chan_frame(frame, 1)) +
357                            // input.C
358                            self.data_chan_frame(frame, 2) +
359                            // (ignore LFE)
360                            // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE]
361                            // 0.5 * (input.SL + input.SR)
362                            0.5 * (self.data_chan_frame(frame, 4) +
363                                   self.data_chan_frame(frame, 5));
364                        v.push(o);
365                    }
366                    self.buffer = v;
367                    self.channels = 1;
368                    self.repeat = false;
369                },
370
371                // stereo
372                (4, 2) => {
373                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
374                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
375                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
376                        // output.L = 0.5 * (input.L + input.SL)
377                        v[frame] =
378                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2));
379                        // output.R = 0.5 * (input.R + input.SR)
380                        v[frame + FRAMES_PER_BLOCK_USIZE] =
381                            0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3));
382                    }
383                    self.buffer = v;
384                    self.channels = 2;
385                    self.repeat = false;
386                },
387                (6, 2) => {
388                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
389                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
390                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
391                        // output.L = L + sqrt(0.5) * (input.C + input.SL)
392                        v[frame] = self.data_chan_frame(frame, 0)
393                            + SQRT_2
394                                * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 4));
395                        // output.R = R + sqrt(0.5) * (input.C + input.SR)
396                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1)
397                            + SQRT_2
398                                * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 5));
399                    }
400                    self.buffer = v;
401                    self.channels = 2;
402                    self.repeat = false;
403                },
404
405                // quad
406                (6, 4) => {
407                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
408                    v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.);
409                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
410                        // output.L = L + sqrt(0.5) * input.C
411                        v[frame] = self.data_chan_frame(frame, 0)
412                            + SQRT_2 * self.data_chan_frame(frame, 2);
413                        // output.R = R + sqrt(0.5) * input.C
414                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1)
415                            + SQRT_2 * self.data_chan_frame(frame, 2);
416                        // output.SL = input.SL
417                        v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4);
418                        // output.SR = input.SR
419                        v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5);
420                    }
421                    self.buffer = v;
422                    self.channels = 4;
423                    self.repeat = false;
424                },
425
426                // If it's not a known kind of speaker configuration, treat as
427                // discrete
428                _ => {
429                    self.mix(channels, ChannelInterpretation::Discrete);
430                },
431            }
432            debug_assert!(self.channels == channels);
433        }
434    }
435
436    /// Resize to add or remove channels, fill extra channels with silence
437    pub fn resize_silence(&mut self, channels: u8) {
438        self.explicit_repeat();
439        self.buffer
440            .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.);
441        self.channels = channels;
442    }
443
444    /// Take a single-channel block and repeat the
445    /// channel
446    pub fn repeat(&mut self, channels: u8) {
447        debug_assert!(self.channels == 1);
448        self.channels = channels;
449        if !self.is_silence() {
450            self.repeat = true;
451        }
452    }
453
454    pub fn interleave(&mut self) -> Vec<f32> {
455        self.explicit_repeat();
456        let mut vec = Vec::with_capacity(self.buffer.len());
457        // FIXME this isn't too efficient
458        vec.resize(self.buffer.len(), 0.);
459        for frame in 0..FRAMES_PER_BLOCK_USIZE {
460            let channels = self.channels as usize;
461            for chan in 0..channels {
462                vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame]
463            }
464        }
465        vec
466    }
467
468    pub fn is_empty(&self) -> bool {
469        self.buffer.is_empty()
470    }
471
472    /// Get the position, forward, and up vectors for a given
473    /// AudioListener-produced block
474    pub fn listener_data(&self, frame: Tick) -> (Vector3D<f32>, Vector3D<f32>, Vector3D<f32>) {
475        let frame = frame.0 as usize;
476        (
477            Vector3D::new(
478                self.data_chan_frame(frame, 0),
479                self.data_chan_frame(frame, 1),
480                self.data_chan_frame(frame, 2),
481            ),
482            Vector3D::new(
483                self.data_chan_frame(frame, 3),
484                self.data_chan_frame(frame, 4),
485                self.data_chan_frame(frame, 5),
486            ),
487            Vector3D::new(
488                self.data_chan_frame(frame, 6),
489                self.data_chan_frame(frame, 7),
490                self.data_chan_frame(frame, 8),
491            ),
492        )
493    }
494}
495
496/// An iterator over frames in a block
497pub struct FrameIterator<'a> {
498    frame: Tick,
499    block: &'a mut Block,
500}
501
502impl<'a> FrameIterator<'a> {
503    #[inline]
504    pub fn new(block: &'a mut Block) -> Self {
505        FrameIterator {
506            frame: Tick(0),
507            block,
508        }
509    }
510
511    /// Advance the iterator
512    ///
513    /// We can't implement Iterator since it doesn't support
514    /// streaming iterators, but we can call `while let Some(frame) = iter.next()`
515    /// here
516    #[inline]
517    pub fn next<'b>(&'b mut self) -> Option<FrameRef<'b>> {
518        let curr = self.frame;
519        if curr < FRAMES_PER_BLOCK {
520            self.frame.advance();
521            Some(FrameRef {
522                frame: curr,
523                block: &mut self.block,
524            })
525        } else {
526            None
527        }
528    }
529}
530
531/// A reference to a frame
532pub struct FrameRef<'a> {
533    frame: Tick,
534    block: &'a mut Block,
535}
536
537impl<'a> FrameRef<'a> {
538    #[inline]
539    pub fn tick(&self) -> Tick {
540        self.frame
541    }
542
543    /// Given a block and a function `f`, mutate the frame through all channels with `f`
544    ///
545    /// Use this when you plan to do the same operation for each channel.
546    /// (Helpers for the other cases will eventually exist)
547    ///
548    /// Block must not be silence
549    ///
550    /// The second parameter to f is the channel number, 0 in case of a repeat()
551    #[inline]
552    pub fn mutate_with<F>(&mut self, mut f: F)
553    where
554        F: FnMut(&mut f32, u8),
555    {
556        debug_assert!(
557            !self.block.is_silence(),
558            "mutate_frame_with should not be called with a silenced block, \
559             call .explicit_silence() if you wish to use this"
560        );
561        if self.block.repeat {
562            f(&mut self.block.buffer[self.frame.0 as usize], 0)
563        } else {
564            for chan in 0..self.block.channels {
565                f(
566                    &mut self.block.buffer
567                        [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
568                    chan,
569                )
570            }
571        }
572    }
573}
574
575// operator impls
576
577impl<T: PortKind> IndexMut<PortIndex<T>> for Chunk {
578    fn index_mut(&mut self, i: PortIndex<T>) -> &mut Block {
579        if let PortIndex::Port(i) = i {
580            &mut self.blocks[i as usize]
581        } else {
582            panic!("attempted to index chunk with param")
583        }
584    }
585}
586
587impl<T: PortKind> Index<PortIndex<T>> for Chunk {
588    type Output = Block;
589    fn index(&self, i: PortIndex<T>) -> &Block {
590        if let PortIndex::Port(i) = i {
591            &self.blocks[i as usize]
592        } else {
593            panic!("attempted to index chunk with param")
594        }
595    }
596}
597
598impl Add<Tick> for Tick {
599    type Output = Tick;
600    fn add(self, other: Tick) -> Self {
601        self + other.0
602    }
603}
604
605impl AddAssign for Tick {
606    fn add_assign(&mut self, other: Tick) {
607        *self = *self + other
608    }
609}
610
611impl Sub<Tick> for Tick {
612    type Output = Tick;
613    fn sub(self, other: Tick) -> Self {
614        self - other.0
615    }
616}
617
618impl Add<u64> for Tick {
619    type Output = Tick;
620    fn add(self, other: u64) -> Self {
621        Tick(self.0 + other)
622    }
623}
624
625impl Sub<u64> for Tick {
626    type Output = Tick;
627    fn sub(self, other: u64) -> Self {
628        Tick(self.0 - other)
629    }
630}
631
632impl Div<f64> for Tick {
633    type Output = f64;
634    fn div(self, other: f64) -> f64 {
635        self.0 as f64 / other
636    }
637}
638
639impl Tick {
640    pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK;
641    const EPSILON: f64 = 1e-7;
642    pub fn from_time(time: f64, rate: f32) -> Tick {
643        Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64)
644    }
645
646    pub fn advance(&mut self) {
647        self.0 += 1;
648    }
649}
servo_media_audio/block.rs

servo_media_audio/
block.rs
