servo_media_audio/
analyser_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cmp;
6use std::f32::consts::PI;
7use std::sync::{Arc, OnceLock};
8
9use malloc_size_of_derive::MallocSizeOf;
10use servo_base::generic_channel::GenericCallback;
11
12use crate::audio_node::{
13    AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo, ChannelInterpretation,
14};
15use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE};
16
17#[derive(AudioNodeCommon)]
18pub(crate) struct AnalyserNode {
19    channel_info: ChannelInfo,
20    callback: Arc<OnceLock<GenericCallback<Block>>>,
21}
22
23impl AnalyserNode {
24    pub fn new(callback: Arc<OnceLock<GenericCallback<Block>>>, channel_info: ChannelInfo) -> Self {
25        Self {
26            callback,
27            channel_info,
28        }
29    }
30}
31
32impl AudioNodeEngine for AnalyserNode {
33    fn node_type(&self) -> AudioNodeType {
34        AudioNodeType::AnalyserNode
35    }
36
37    fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk {
38        debug_assert!(inputs.len() == 1);
39
40        let mut push = inputs.blocks[0].clone();
41        push.mix(1, ChannelInterpretation::Speakers);
42
43        if let Some(callback) = self.callback.get() {
44            let _ = callback.send(push);
45        }
46
47        // analyser node doesn't modify the inputs
48        inputs
49    }
50}
51
52/// From <https://webaudio.github.io/web-audio-api/#dom-analysernode-fftsize>
53pub const MAX_FFT_SIZE: usize = 32768;
54pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE;
55
56/// The actual analysis is done on the DOM side. We provide
57/// the actual base functionality in this struct, so the DOM
58/// just has to do basic shimming
59#[derive(MallocSizeOf)]
60pub struct AnalysisEngine {
61    /// The number of past sample-frames to consider in the FFT
62    fft_size: usize,
63    smoothing_constant: f64,
64    min_decibels: f64,
65    max_decibels: f64,
66    /// This is a ring buffer containing the last MAX_FFT_SIZE
67    /// sample-frames
68    #[ignore_malloc_size_of = "Do not know how"]
69    data: Box<[f32; MAX_FFT_SIZE]>,
70    /// The index of the current block
71    current_block: usize,
72    /// Have we computed the FFT already?
73    fft_computed: bool,
74    /// Cached blackman window data
75    blackman_windows: Vec<f32>,
76    /// The smoothed FFT data (in frequency domain)
77    smoothed_fft_data: Vec<f32>,
78    /// The computed FFT data, in decibels
79    computed_fft_data: Vec<f32>,
80    /// The windowed time domain data
81    /// Used during FFT computation
82    windowed: Vec<f32>,
83}
84
85impl AnalysisEngine {
86    pub fn new(
87        fft_size: usize,
88        smoothing_constant: f64,
89        min_decibels: f64,
90        max_decibels: f64,
91    ) -> Self {
92        debug_assert!((32..=32768).contains(&fft_size));
93        // must be a power of two
94        debug_assert!(fft_size & (fft_size - 1) == 0);
95        debug_assert!((0. ..=1.).contains(&smoothing_constant));
96        debug_assert!(max_decibels > min_decibels);
97        Self {
98            fft_size,
99            smoothing_constant,
100            min_decibels,
101            max_decibels,
102            data: Box::new([0.; MAX_FFT_SIZE]),
103            current_block: MAX_BLOCK_COUNT - 1,
104            fft_computed: false,
105            blackman_windows: Vec::with_capacity(fft_size),
106            computed_fft_data: Vec::with_capacity(fft_size / 2),
107            smoothed_fft_data: Vec::with_capacity(fft_size / 2),
108            windowed: Vec::with_capacity(fft_size),
109        }
110    }
111
112    pub fn set_fft_size(&mut self, fft_size: usize) {
113        debug_assert!((32..=32768).contains(&fft_size));
114        // must be a power of two
115        debug_assert!(fft_size & (fft_size - 1) == 0);
116        self.fft_size = fft_size;
117        self.fft_computed = false;
118    }
119
120    pub fn get_fft_size(&self) -> usize {
121        self.fft_size
122    }
123
124    pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) {
125        debug_assert!((0. ..=1.).contains(&smoothing_constant));
126        self.smoothing_constant = smoothing_constant;
127        self.fft_computed = false;
128    }
129
130    pub fn get_smoothing_constant(&self) -> f64 {
131        self.smoothing_constant
132    }
133
134    pub fn set_min_decibels(&mut self, min_decibels: f64) {
135        debug_assert!(min_decibels < self.max_decibels);
136        self.min_decibels = min_decibels;
137    }
138
139    pub fn get_min_decibels(&self) -> f64 {
140        self.min_decibels
141    }
142
143    pub fn set_max_decibels(&mut self, max_decibels: f64) {
144        debug_assert!(self.min_decibels < max_decibels);
145        self.max_decibels = max_decibels;
146    }
147
148    pub fn get_max_decibels(&self) -> f64 {
149        self.max_decibels
150    }
151
152    fn advance(&mut self) {
153        self.current_block += 1;
154        if self.current_block >= MAX_BLOCK_COUNT {
155            self.current_block = 0;
156        }
157    }
158
159    /// Get the data of the current block
160    fn curent_block_mut(&mut self) -> &mut [f32] {
161        let index = FRAMES_PER_BLOCK_USIZE * self.current_block;
162        &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)]
163    }
164
165    /// Given an index from 0 to fft_size, convert it into an index into
166    /// the backing array
167    fn convert_index(&self, index: usize) -> usize {
168        let offset = self.fft_size - index;
169        let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1;
170        if offset > last_element {
171            MAX_FFT_SIZE - offset + last_element
172        } else {
173            last_element - offset
174        }
175    }
176
177    /// Given an index into the backing array, increment it
178    fn advance_index(&self, index: &mut usize) {
179        *index += 1;
180        if *index >= MAX_FFT_SIZE {
181            *index = 0;
182        }
183    }
184
185    pub fn push(&mut self, mut block: Block) {
186        debug_assert!(block.chan_count() == 1);
187        self.advance();
188        if !block.is_silence() {
189            self.curent_block_mut().copy_from_slice(block.data_mut());
190        }
191        self.fft_computed = false;
192    }
193
194    /// <https://webaudio.github.io/web-audio-api/#blackman-window>
195    fn compute_blackman_windows(&mut self) {
196        if self.blackman_windows.len() == self.fft_size {
197            return;
198        }
199        const ALPHA: f32 = 0.16;
200        const ALPHA_0: f32 = (1. - ALPHA) / 2.;
201        const ALPHA_1: f32 = 1. / 2.;
202        const ALPHA_2: f32 = ALPHA / 2.;
203        self.blackman_windows.resize(self.fft_size, 0.);
204        let coeff = PI * 2. / self.fft_size as f32;
205        for n in 0..self.fft_size {
206            self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() +
207                ALPHA_2 * (2. * coeff * n as f32).cos();
208        }
209    }
210
211    fn apply_blackman_window(&mut self) {
212        self.compute_blackman_windows();
213        self.windowed.resize(self.fft_size, 0.);
214
215        let mut data_idx = self.convert_index(0);
216        for n in 0..self.fft_size {
217            self.windowed[n] = self.blackman_windows[n] * self.data[data_idx];
218            self.advance_index(&mut data_idx);
219        }
220    }
221
222    fn compute_fft(&mut self) {
223        if self.fft_computed {
224            return;
225        }
226        self.fft_computed = true;
227        self.apply_blackman_window();
228        self.computed_fft_data.resize(self.fft_size / 2, 0.);
229        self.smoothed_fft_data.resize(self.fft_size / 2, 0.);
230
231        for k in 0..(self.fft_size / 2) {
232            let mut sum_real = 0.;
233            let mut sum_imaginary = 0.;
234            let factor = -2. * PI * k as f32 / self.fft_size as f32;
235            for n in 0..(self.fft_size) {
236                sum_real += self.windowed[n] * (factor * n as f32).cos();
237                sum_imaginary += self.windowed[n] * (factor * n as f32).sin();
238            }
239            let sum_real = sum_real / self.fft_size as f32;
240            let sum_imaginary = sum_imaginary / self.fft_size as f32;
241            let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt();
242            self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 +
243                (1. - self.smoothing_constant) * magnitude as f64)
244                as f32;
245            self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.);
246        }
247    }
248
249    pub fn fill_time_domain_data(&self, dest: &mut [f32]) {
250        let mut data_idx = self.convert_index(0);
251        let end = cmp::min(self.fft_size, dest.len());
252        for entry in &mut dest[0..end] {
253            *entry = self.data[data_idx];
254            self.advance_index(&mut data_idx);
255        }
256    }
257
258    pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) {
259        let mut data_idx = self.convert_index(0);
260        let end = cmp::min(self.fft_size, dest.len());
261        for entry in &mut dest[0..end] {
262            let result = 128. * (1. + self.data[data_idx]);
263            *entry = clamp_255(result);
264            self.advance_index(&mut data_idx)
265        }
266    }
267
268    pub fn fill_frequency_data(&mut self, dest: &mut [f32]) {
269        self.compute_fft();
270        let len = cmp::min(dest.len(), self.computed_fft_data.len());
271        dest[0..len].copy_from_slice(&self.computed_fft_data[0..len]);
272    }
273
274    pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) {
275        self.compute_fft();
276        let len = cmp::min(dest.len(), self.computed_fft_data.len());
277        let ratio = 255. / (self.max_decibels - self.min_decibels);
278        for (index, freq) in dest[0..len].iter_mut().enumerate() {
279            let result = ratio * (self.computed_fft_data[index] as f64 - self.min_decibels);
280            *freq = clamp_255(result as f32);
281        }
282    }
283}
284
285fn clamp_255(val: f32) -> u8 {
286    if val > 255. {
287        255
288    } else if val < 0. {
289        0
290    } else {
291        val as u8
292    }
293}
servo_media_audio/analyser_node.rs

servo_media_audio/
analyser_node.rs
