servo_media_audio/
block.rs

1use byte_slice_cast::*;
2use euclid::default::Vector3D;
3use crate::graph::{PortIndex, PortKind};
4use crate::node::ChannelInterpretation;
5use smallvec::SmallVec;
6use std::f32::consts::SQRT_2;
7use std::mem;
8use std::ops::*;
9
10// defined by spec
11// https://webaudio.github.io/web-audio-api/#render-quantum
12pub const FRAMES_PER_BLOCK: Tick = Tick(128);
13pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize;
14
15/// A tick, i.e. the time taken for a single frame
16#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
17pub struct Tick(pub u64);
18
19/// A collection of blocks received as input by a node
20/// or outputted by a node.
21///
22/// This will usually be a single block.
23///
24/// Some nodes have multiple inputs or outputs, which is
25/// where this becomes useful. Source nodes have an input
26/// of an empty chunk.
27pub struct Chunk {
28    pub blocks: SmallVec<[Block; 1]>,
29}
30
31impl Default for Chunk {
32    fn default() -> Self {
33        Chunk {
34            blocks: SmallVec::new(),
35        }
36    }
37}
38
39impl Chunk {
40    pub fn len(&self) -> usize {
41        self.blocks.len()
42    }
43
44    pub fn explicit_silence() -> Chunk {
45        let blocks: SmallVec<[Block; 1]> = SmallVec::new();
46        Chunk {
47            blocks: blocks
48                .as_slice()
49                .iter()
50                .map(|_| {
51                    let mut block = Block::default();
52                    block.explicit_silence();
53                    block
54                })
55                .collect(),
56        }
57    }
58}
59
60/// We render audio in blocks of size FRAMES_PER_BLOCK
61///
62/// A single block may contain multiple channels
63#[derive(Clone, Serialize, Deserialize, Debug)]
64pub struct Block {
65    /// The number of channels in this block
66    channels: u8,
67    /// This is an optimization which means that the buffer is representing multiple channels with the
68    /// same content at once. Happens when audio is upmixed or when a source like
69    /// an oscillator node has multiple channel outputs
70    repeat: bool,
71    /// If this vector is empty, it is a shorthand for "silence"
72    /// It is possible to obtain an explicitly silent buffer via .explicit_silence()
73    ///
74    /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true,
75    /// in which case it will be of length FRAMES_PER_BLOCK
76    buffer: Vec<f32>,
77}
78
79impl Default for Block {
80    fn default() -> Self {
81        Block {
82            channels: 1,
83            repeat: false,
84            buffer: Vec::new(),
85        }
86    }
87}
88
89impl Block {
90    /// Empty block with no channels, for pushing
91    /// new channels to.
92    ///
93    /// Must be used with push_chan
94    pub fn empty() -> Self {
95        Block {
96            channels: 0,
97            ..Default::default()
98        }
99    }
100
101    pub fn for_channels_explicit(channels: u8) -> Self {
102        Block {
103            channels,
104            repeat: false,
105            buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize],
106        }
107    }
108
109    /// This provides the entire buffer as a mutable slice of u8
110    pub fn as_mut_byte_slice(&mut self) -> &mut [u8] {
111        self.data_mut().as_mut_byte_slice().expect("casting failed")
112    }
113
114    pub fn for_vec(buffer: Vec<f32>) -> Self {
115        assert!(buffer.len() % FRAMES_PER_BLOCK_USIZE == 0);
116        Block {
117            channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8,
118            repeat: false,
119            buffer,
120        }
121    }
122
123    /// Zero-gain sum with another buffer
124    ///
125    /// Used after mixing multiple inputs to a single port
126    pub fn sum(mut self, mut other: Self) -> Self {
127        if self.is_silence() {
128            other
129        } else if other.is_silence() {
130            self
131        } else {
132            debug_assert_eq!(self.channels, other.channels);
133            if self.repeat ^ other.repeat {
134                self.explicit_repeat();
135                other.explicit_repeat();
136            }
137            debug_assert_eq!(self.buffer.len(), other.buffer.len());
138            for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) {
139                *a += b
140            }
141            self
142        }
143    }
144
145    /// If this is in "silence" mode without a buffer, allocate a silent buffer
146    pub fn explicit_silence(&mut self) {
147        if self.buffer.is_empty() {
148            self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.);
149            self.repeat = true;
150        }
151    }
152
153    /// This provides the entire buffer as a mutable slice of f32
154    pub fn data_mut(&mut self) -> &mut [f32] {
155        self.explicit_silence();
156        &mut self.buffer
157    }
158
159    pub fn explicit_repeat(&mut self) {
160        if self.repeat {
161            debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE);
162            if self.channels > 1 {
163                let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize);
164                for _ in 0..self.channels {
165                    new.extend(&self.buffer)
166                }
167
168                self.buffer = new;
169            }
170            self.repeat = false;
171        } else if self.is_silence() {
172            self.buffer
173                .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.);
174        }
175    }
176
177    pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] {
178        self.explicit_repeat();
179        let start = chan as usize * FRAMES_PER_BLOCK_USIZE;
180        &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE]
181    }
182
183    #[inline]
184    pub fn data_chan(&self, chan: u8) -> &[f32] {
185        debug_assert!(
186            !self.is_silence(),
187            "data_chan doesn't work with silent buffers"
188        );
189        let offset = if self.repeat {
190            0
191        } else {
192            chan as usize * FRAMES_PER_BLOCK_USIZE
193        };
194        &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE]
195    }
196
197    pub fn take(&mut self) -> Block {
198        let mut new = Block::default();
199        new.channels = self.channels;
200        mem::replace(self, new)
201    }
202
203    pub fn chan_count(&self) -> u8 {
204        self.channels
205    }
206
207    pub fn iter(&mut self) -> FrameIterator {
208        FrameIterator::new(self)
209    }
210
211    pub fn is_silence(&self) -> bool {
212        self.buffer.is_empty()
213    }
214
215    pub fn is_repeat(&self) -> bool {
216        self.repeat
217    }
218
219    pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 {
220        if self.is_silence() {
221            0.
222        } else {
223            self.data_chan(chan)[frame]
224        }
225    }
226
227    pub fn push_chan(&mut self, data: &[f32]) {
228        assert!(!self.repeat);
229        assert!(!self.is_silence() || self.channels == 0);
230        assert!(data.len() == FRAMES_PER_BLOCK_USIZE);
231        self.buffer.extend(data);
232        self.channels += 1;
233    }
234
235    /// upmix/downmix the channels if necessary
236    ///
237    /// Currently only supports upmixing from 1
238    pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) {
239        // If we're not changing the number of channels, we
240        // don't actually need to mix
241        if self.channels == channels {
242            return;
243        }
244
245        // Silent buffers stay silent
246        if self.is_silence() {
247            self.channels = channels;
248            return;
249        }
250
251        if interpretation == ChannelInterpretation::Discrete {
252            // discrete downmixes by truncation, upmixes by adding
253            // silent channels
254
255            // If we're discrete, have a repeat, and are downmixing,
256            // just truncate by changing the channel value
257            if self.repeat && self.channels > channels {
258                self.channels = channels;
259            } else {
260                // otherwise resize the buffer, silent-filling when necessary
261                self.resize_silence(channels);
262            }
263        } else {
264            // For speakers, we have to do special things based on the
265            // interpretation of the channels for each kind of speakers
266
267            // The layout of each speaker kind is:
268            //
269            // - Mono: [The mono channel]
270            // - Stereo: [L, R]
271            // - Quad: [L, R, SL, SR]
272            // - 5.1: [L, R, C, LFE, SL, SR]
273
274            match (self.channels, channels) {
275                // Upmixing
276                // https://webaudio.github.io/web-audio-api/#UpMix-sub
277
278                // mono
279                (1, 2) => {
280                    // output.{L, R} = input
281                    self.repeat(2);
282                }
283                (1, 4) => {
284                    // output.{L, R} = input
285                    self.repeat(2);
286                    // output.{SL, SR} = 0
287                    self.resize_silence(4);
288                }
289                (1, 6) => {
290                    let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE);
291                    // output.{L, R} = 0
292                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
293                    // output.C = input
294                    v.extend(&self.buffer);
295                    self.buffer = v;
296                    // output.{LFE, SL, SR} = 0
297                    self.resize_silence(6);
298                }
299
300                // stereo
301                (2, 4) | (2, 6) => {
302                    // output.{L, R} = input.{L, R}
303                    // (5.1) output.{C, LFE} = 0
304                    // output.{SL, SR} = 0
305                    self.resize_silence(channels);
306                }
307
308                // quad
309                (4, 6) => {
310                    // we can avoid this and instead calculate offsets
311                    // based off whether or not this is `repeat`, but
312                    // a `repeat` quad block should be rare
313                    self.explicit_repeat();
314
315                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
316                    // output.{L, R} = input.{L, R}
317                    v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]);
318                    // output.{C, LFE} = 0
319                    v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.);
320                    // output.{SL, R} = input.{SL, SR}
321                    v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]);
322                    self.buffer = v;
323                    self.channels = channels;
324                }
325
326                // Downmixing
327                // https://webaudio.github.io/web-audio-api/#down-mix
328
329                // mono
330                (2, 1) => {
331                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
332                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
333                        // output = 0.5 * (input.L + input.R);
334                        let o =
335                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1));
336                        v.push(o);
337                    }
338                    self.buffer = v;
339                    self.channels = 1;
340                    self.repeat = false;
341                }
342                (4, 1) => {
343                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
344                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
345                        // output = 0.5 * (input.L + input.R + input.SL + input.SR);
346                        let o = 0.25
347                            * (self.data_chan_frame(frame, 0)
348                                + self.data_chan_frame(frame, 1)
349                                + self.data_chan_frame(frame, 2)
350                                + self.data_chan_frame(frame, 3));
351                        v.push(o);
352                    }
353                    self.buffer = v;
354                    self.channels = 1;
355                    self.repeat = false;
356                }
357                (6, 1) => {
358                    let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE);
359                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
360                        // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR)
361                        let o =
362                            // sqrt(0.5) * (input.L + input.R)
363                            SQRT_2 * (self.data_chan_frame(frame, 0) +
364                                      self.data_chan_frame(frame, 1)) +
365                            // input.C
366                            self.data_chan_frame(frame, 2) +
367                            // (ignore LFE)
368                            // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE]
369                            // 0.5 * (input.SL + input.SR)
370                            0.5 * (self.data_chan_frame(frame, 4) +
371                                   self.data_chan_frame(frame, 5));
372                        v.push(o);
373                    }
374                    self.buffer = v;
375                    self.channels = 1;
376                    self.repeat = false;
377                }
378
379                // stereo
380                (4, 2) => {
381                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
382                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
383                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
384                        // output.L = 0.5 * (input.L + input.SL)
385                        v[frame] =
386                            0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2));
387                        // output.R = 0.5 * (input.R + input.SR)
388                        v[frame + FRAMES_PER_BLOCK_USIZE] =
389                            0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3));
390                    }
391                    self.buffer = v;
392                    self.channels = 2;
393                    self.repeat = false;
394                }
395                (6, 2) => {
396                    let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE);
397                    v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.);
398                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
399                        // output.L = L + sqrt(0.5) * (input.C + input.SL)
400                        v[frame] = self.data_chan_frame(frame, 0)
401                            + SQRT_2
402                                * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 4));
403                        // output.R = R + sqrt(0.5) * (input.C + input.SR)
404                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1)
405                            + SQRT_2
406                                * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 5));
407                    }
408                    self.buffer = v;
409                    self.channels = 2;
410                    self.repeat = false;
411                }
412
413                // quad
414                (6, 4) => {
415                    let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE);
416                    v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.);
417                    for frame in 0..FRAMES_PER_BLOCK_USIZE {
418                        // output.L = L + sqrt(0.5) * input.C
419                        v[frame] = self.data_chan_frame(frame, 0)
420                            + SQRT_2 * self.data_chan_frame(frame, 2);
421                        // output.R = R + sqrt(0.5) * input.C
422                        v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1)
423                            + SQRT_2 * self.data_chan_frame(frame, 2);
424                        // output.SL = input.SL
425                        v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4);
426                        // output.SR = input.SR
427                        v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5);
428                    }
429                    self.buffer = v;
430                    self.channels = 4;
431                    self.repeat = false;
432                }
433
434                // If it's not a known kind of speaker configuration, treat as
435                // discrete
436                _ => {
437                    self.mix(channels, ChannelInterpretation::Discrete);
438                }
439            }
440            debug_assert!(self.channels == channels);
441        }
442    }
443
444    /// Resize to add or remove channels, fill extra channels with silence
445    pub fn resize_silence(&mut self, channels: u8) {
446        self.explicit_repeat();
447        self.buffer
448            .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.);
449        self.channels = channels;
450    }
451
452    /// Take a single-channel block and repeat the
453    /// channel
454    pub fn repeat(&mut self, channels: u8) {
455        debug_assert!(self.channels == 1);
456        self.channels = channels;
457        if !self.is_silence() {
458            self.repeat = true;
459        }
460    }
461
462    pub fn interleave(&mut self) -> Vec<f32> {
463        self.explicit_repeat();
464        let mut vec = Vec::with_capacity(self.buffer.len());
465        // FIXME this isn't too efficient
466        vec.resize(self.buffer.len(), 0.);
467        for frame in 0..FRAMES_PER_BLOCK_USIZE {
468            let channels = self.channels as usize;
469            for chan in 0..channels {
470                vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame]
471            }
472        }
473        vec
474    }
475
476    pub fn is_empty(&self) -> bool {
477        self.buffer.is_empty()
478    }
479
480    /// Get the position, forward, and up vectors for a given
481    /// AudioListener-produced block
482    pub fn listener_data(&self, frame: Tick) -> (Vector3D<f32>, Vector3D<f32>, Vector3D<f32>) {
483        let frame = frame.0 as usize;
484        (
485            Vector3D::new(
486                self.data_chan_frame(frame, 0),
487                self.data_chan_frame(frame, 1),
488                self.data_chan_frame(frame, 2),
489            ),
490            Vector3D::new(
491                self.data_chan_frame(frame, 3),
492                self.data_chan_frame(frame, 4),
493                self.data_chan_frame(frame, 5),
494            ),
495            Vector3D::new(
496                self.data_chan_frame(frame, 6),
497                self.data_chan_frame(frame, 7),
498                self.data_chan_frame(frame, 8),
499            ),
500        )
501    }
502}
503
504/// An iterator over frames in a block
505pub struct FrameIterator<'a> {
506    frame: Tick,
507    block: &'a mut Block,
508}
509
510impl<'a> FrameIterator<'a> {
511    #[inline]
512    pub fn new(block: &'a mut Block) -> Self {
513        FrameIterator {
514            frame: Tick(0),
515            block,
516        }
517    }
518
519    /// Advance the iterator
520    ///
521    /// We can't implement Iterator since it doesn't support
522    /// streaming iterators, but we can call `while let Some(frame) = iter.next()`
523    /// here
524    #[inline]
525    pub fn next<'b>(&'b mut self) -> Option<FrameRef<'b>> {
526        let curr = self.frame;
527        if curr < FRAMES_PER_BLOCK {
528            self.frame.advance();
529            Some(FrameRef {
530                frame: curr,
531                block: &mut self.block,
532            })
533        } else {
534            None
535        }
536    }
537}
538
539/// A reference to a frame
540pub struct FrameRef<'a> {
541    frame: Tick,
542    block: &'a mut Block,
543}
544
545impl<'a> FrameRef<'a> {
546    #[inline]
547    pub fn tick(&self) -> Tick {
548        self.frame
549    }
550
551    /// Given a block and a function `f`, mutate the frame through all channels with `f`
552    ///
553    /// Use this when you plan to do the same operation for each channel.
554    /// (Helpers for the other cases will eventually exist)
555    ///
556    /// Block must not be silence
557    ///
558    /// The second parameter to f is the channel number, 0 in case of a repeat()
559    #[inline]
560    pub fn mutate_with<F>(&mut self, mut f: F)
561    where
562        F: FnMut(&mut f32, u8),
563    {
564        debug_assert!(
565            !self.block.is_silence(),
566            "mutate_frame_with should not be called with a silenced block, \
567             call .explicit_silence() if you wish to use this"
568        );
569        if self.block.repeat {
570            f(&mut self.block.buffer[self.frame.0 as usize], 0)
571        } else {
572            for chan in 0..self.block.channels {
573                f(
574                    &mut self.block.buffer
575                        [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize],
576                    chan,
577                )
578            }
579        }
580    }
581}
582
583// operator impls
584
585impl<T: PortKind> IndexMut<PortIndex<T>> for Chunk {
586    fn index_mut(&mut self, i: PortIndex<T>) -> &mut Block {
587        if let PortIndex::Port(i) = i {
588            &mut self.blocks[i as usize]
589        } else {
590            panic!("attempted to index chunk with param")
591        }
592    }
593}
594
595impl<T: PortKind> Index<PortIndex<T>> for Chunk {
596    type Output = Block;
597    fn index(&self, i: PortIndex<T>) -> &Block {
598        if let PortIndex::Port(i) = i {
599            &self.blocks[i as usize]
600        } else {
601            panic!("attempted to index chunk with param")
602        }
603    }
604}
605
606impl Add<Tick> for Tick {
607    type Output = Tick;
608    fn add(self, other: Tick) -> Self {
609        self + other.0
610    }
611}
612
613impl AddAssign for Tick {
614    fn add_assign(&mut self, other: Tick) {
615        *self = *self + other
616    }
617}
618
619impl Sub<Tick> for Tick {
620    type Output = Tick;
621    fn sub(self, other: Tick) -> Self {
622        self - other.0
623    }
624}
625
626impl Add<u64> for Tick {
627    type Output = Tick;
628    fn add(self, other: u64) -> Self {
629        Tick(self.0 + other)
630    }
631}
632
633impl Sub<u64> for Tick {
634    type Output = Tick;
635    fn sub(self, other: u64) -> Self {
636        Tick(self.0 - other)
637    }
638}
639
640impl Div<f64> for Tick {
641    type Output = f64;
642    fn div(self, other: f64) -> f64 {
643        self.0 as f64 / other
644    }
645}
646
647impl Tick {
648    pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK;
649    const EPSILON: f64 = 1e-7;
650    pub fn from_time(time: f64, rate: f32) -> Tick {
651        Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64)
652    }
653
654    pub fn advance(&mut self) {
655        self.0 += 1;
656    }
657}
servo_media_audio/block.rs

servo_media_audio/
block.rs
