servo_media_audio/
analyser_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cmp;
6use std::f32::consts::PI;
7
8use malloc_size_of_derive::MallocSizeOf;
9
10use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE};
11use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo, ChannelInterpretation};
12
13#[derive(AudioNodeCommon)]
14pub(crate) struct AnalyserNode {
15    channel_info: ChannelInfo,
16    callback: Box<dyn FnMut(Block) + Send>,
17}
18
19impl AnalyserNode {
20    pub fn new(callback: Box<dyn FnMut(Block) + Send>, channel_info: ChannelInfo) -> Self {
21        Self {
22            callback,
23            channel_info,
24        }
25    }
26}
27
28impl AudioNodeEngine for AnalyserNode {
29    fn node_type(&self) -> AudioNodeType {
30        AudioNodeType::AnalyserNode
31    }
32
33    fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk {
34        debug_assert!(inputs.len() == 1);
35
36        let mut push = inputs.blocks[0].clone();
37        push.mix(1, ChannelInterpretation::Speakers);
38
39        (self.callback)(push);
40
41        // analyser node doesn't modify the inputs
42        inputs
43    }
44}
45
46/// From <https://webaudio.github.io/web-audio-api/#dom-analysernode-fftsize>
47pub const MAX_FFT_SIZE: usize = 32768;
48pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE;
49
50/// The actual analysis is done on the DOM side. We provide
51/// the actual base functionality in this struct, so the DOM
52/// just has to do basic shimming
53#[derive(MallocSizeOf)]
54pub struct AnalysisEngine {
55    /// The number of past sample-frames to consider in the FFT
56    fft_size: usize,
57    smoothing_constant: f64,
58    min_decibels: f64,
59    max_decibels: f64,
60    /// This is a ring buffer containing the last MAX_FFT_SIZE
61    /// sample-frames
62    #[ignore_malloc_size_of = "Do not know how"]
63    data: Box<[f32; MAX_FFT_SIZE]>,
64    /// The index of the current block
65    current_block: usize,
66    /// Have we computed the FFT already?
67    fft_computed: bool,
68    /// Cached blackman window data
69    blackman_windows: Vec<f32>,
70    /// The smoothed FFT data (in frequency domain)
71    smoothed_fft_data: Vec<f32>,
72    /// The computed FFT data, in decibels
73    computed_fft_data: Vec<f32>,
74    /// The windowed time domain data
75    /// Used during FFT computation
76    windowed: Vec<f32>,
77}
78
79impl AnalysisEngine {
80    pub fn new(
81        fft_size: usize,
82        smoothing_constant: f64,
83        min_decibels: f64,
84        max_decibels: f64,
85    ) -> Self {
86        debug_assert!((32..=32768).contains(&fft_size));
87        // must be a power of two
88        debug_assert!(fft_size & (fft_size - 1) == 0);
89        debug_assert!((0. ..=1.).contains(&smoothing_constant));
90        debug_assert!(max_decibels > min_decibels);
91        Self {
92            fft_size,
93            smoothing_constant,
94            min_decibels,
95            max_decibels,
96            data: Box::new([0.; MAX_FFT_SIZE]),
97            current_block: MAX_BLOCK_COUNT - 1,
98            fft_computed: false,
99            blackman_windows: Vec::with_capacity(fft_size),
100            computed_fft_data: Vec::with_capacity(fft_size / 2),
101            smoothed_fft_data: Vec::with_capacity(fft_size / 2),
102            windowed: Vec::with_capacity(fft_size),
103        }
104    }
105
106    pub fn set_fft_size(&mut self, fft_size: usize) {
107        debug_assert!((32..=32768).contains(&fft_size));
108        // must be a power of two
109        debug_assert!(fft_size & (fft_size - 1) == 0);
110        self.fft_size = fft_size;
111        self.fft_computed = false;
112    }
113
114    pub fn get_fft_size(&self) -> usize {
115        self.fft_size
116    }
117
118    pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) {
119        debug_assert!((0. ..=1.).contains(&smoothing_constant));
120        self.smoothing_constant = smoothing_constant;
121        self.fft_computed = false;
122    }
123
124    pub fn get_smoothing_constant(&self) -> f64 {
125        self.smoothing_constant
126    }
127
128    pub fn set_min_decibels(&mut self, min_decibels: f64) {
129        debug_assert!(min_decibels < self.max_decibels);
130        self.min_decibels = min_decibels;
131    }
132
133    pub fn get_min_decibels(&self) -> f64 {
134        self.min_decibels
135    }
136
137    pub fn set_max_decibels(&mut self, max_decibels: f64) {
138        debug_assert!(self.min_decibels < max_decibels);
139        self.max_decibels = max_decibels;
140    }
141
142    pub fn get_max_decibels(&self) -> f64 {
143        self.max_decibels
144    }
145
146    fn advance(&mut self) {
147        self.current_block += 1;
148        if self.current_block >= MAX_BLOCK_COUNT {
149            self.current_block = 0;
150        }
151    }
152
153    /// Get the data of the current block
154    fn curent_block_mut(&mut self) -> &mut [f32] {
155        let index = FRAMES_PER_BLOCK_USIZE * self.current_block;
156        &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)]
157    }
158
159    /// Given an index from 0 to fft_size, convert it into an index into
160    /// the backing array
161    fn convert_index(&self, index: usize) -> usize {
162        let offset = self.fft_size - index;
163        let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1;
164        if offset > last_element {
165            MAX_FFT_SIZE - offset + last_element
166        } else {
167            last_element - offset
168        }
169    }
170
171    /// Given an index into the backing array, increment it
172    fn advance_index(&self, index: &mut usize) {
173        *index += 1;
174        if *index >= MAX_FFT_SIZE {
175            *index = 0;
176        }
177    }
178
179    pub fn push(&mut self, mut block: Block) {
180        debug_assert!(block.chan_count() == 1);
181        self.advance();
182        if !block.is_silence() {
183            self.curent_block_mut().copy_from_slice(block.data_mut());
184        }
185        self.fft_computed = false;
186    }
187
188    /// <https://webaudio.github.io/web-audio-api/#blackman-window>
189    fn compute_blackman_windows(&mut self) {
190        if self.blackman_windows.len() == self.fft_size {
191            return;
192        }
193        const ALPHA: f32 = 0.16;
194        const ALPHA_0: f32 = (1. - ALPHA) / 2.;
195        const ALPHA_1: f32 = 1. / 2.;
196        const ALPHA_2: f32 = ALPHA / 2.;
197        self.blackman_windows.resize(self.fft_size, 0.);
198        let coeff = PI * 2. / self.fft_size as f32;
199        for n in 0..self.fft_size {
200            self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() +
201                ALPHA_2 * (2. * coeff * n as f32).cos();
202        }
203    }
204
205    fn apply_blackman_window(&mut self) {
206        self.compute_blackman_windows();
207        self.windowed.resize(self.fft_size, 0.);
208
209        let mut data_idx = self.convert_index(0);
210        for n in 0..self.fft_size {
211            self.windowed[n] = self.blackman_windows[n] * self.data[data_idx];
212            self.advance_index(&mut data_idx);
213        }
214    }
215
216    fn compute_fft(&mut self) {
217        if self.fft_computed {
218            return;
219        }
220        self.fft_computed = true;
221        self.apply_blackman_window();
222        self.computed_fft_data.resize(self.fft_size / 2, 0.);
223        self.smoothed_fft_data.resize(self.fft_size / 2, 0.);
224
225        for k in 0..(self.fft_size / 2) {
226            let mut sum_real = 0.;
227            let mut sum_imaginary = 0.;
228            let factor = -2. * PI * k as f32 / self.fft_size as f32;
229            for n in 0..(self.fft_size) {
230                sum_real += self.windowed[n] * (factor * n as f32).cos();
231                sum_imaginary += self.windowed[n] * (factor * n as f32).sin();
232            }
233            let sum_real = sum_real / self.fft_size as f32;
234            let sum_imaginary = sum_imaginary / self.fft_size as f32;
235            let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt();
236            self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 +
237                (1. - self.smoothing_constant) * magnitude as f64)
238                as f32;
239            self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.);
240        }
241    }
242
243    pub fn fill_time_domain_data(&self, dest: &mut [f32]) {
244        let mut data_idx = self.convert_index(0);
245        let end = cmp::min(self.fft_size, dest.len());
246        for entry in &mut dest[0..end] {
247            *entry = self.data[data_idx];
248            self.advance_index(&mut data_idx);
249        }
250    }
251
252    pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) {
253        let mut data_idx = self.convert_index(0);
254        let end = cmp::min(self.fft_size, dest.len());
255        for entry in &mut dest[0..end] {
256            let result = 128. * (1. + self.data[data_idx]);
257            *entry = clamp_255(result);
258            self.advance_index(&mut data_idx)
259        }
260    }
261
262    pub fn fill_frequency_data(&mut self, dest: &mut [f32]) {
263        self.compute_fft();
264        let len = cmp::min(dest.len(), self.computed_fft_data.len());
265        dest[0..len].copy_from_slice(&self.computed_fft_data[0..len]);
266    }
267
268    pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) {
269        self.compute_fft();
270        let len = cmp::min(dest.len(), self.computed_fft_data.len());
271        let ratio = 255. / (self.max_decibels - self.min_decibels);
272        for (index, freq) in dest[0..len].iter_mut().enumerate() {
273            let result = ratio * (self.computed_fft_data[index] as f64 - self.min_decibels);
274            *freq = clamp_255(result as f32);
275        }
276    }
277}
278
279fn clamp_255(val: f32) -> u8 {
280    if val > 255. {
281        255
282    } else if val < 0. {
283        0
284    } else {
285        val as u8
286    }
287}
servo_media_audio/analyser_node.rs

servo_media_audio/
analyser_node.rs
