servo_media_audio/
analyser_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cmp;
6use std::f32::consts::PI;
7use std::sync::{Arc, OnceLock};
8
9use malloc_size_of_derive::MallocSizeOf;
10use servo_base::generic_channel::GenericCallback;
11
12use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE};
13use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo, ChannelInterpretation};
14
15#[derive(AudioNodeCommon)]
16pub(crate) struct AnalyserNode {
17    channel_info: ChannelInfo,
18    callback: Arc<OnceLock<GenericCallback<Block>>>,
19}
20
21impl AnalyserNode {
22    pub fn new(callback: Arc<OnceLock<GenericCallback<Block>>>, channel_info: ChannelInfo) -> Self {
23        Self {
24            callback,
25            channel_info,
26        }
27    }
28}
29
30impl AudioNodeEngine for AnalyserNode {
31    fn node_type(&self) -> AudioNodeType {
32        AudioNodeType::AnalyserNode
33    }
34
35    fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk {
36        debug_assert!(inputs.len() == 1);
37
38        let mut push = inputs.blocks[0].clone();
39        push.mix(1, ChannelInterpretation::Speakers);
40
41        if let Some(callback) = self.callback.get() {
42            let _ = callback.send(push);
43        }
44
45        // analyser node doesn't modify the inputs
46        inputs
47    }
48}
49
50/// From <https://webaudio.github.io/web-audio-api/#dom-analysernode-fftsize>
51pub const MAX_FFT_SIZE: usize = 32768;
52pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE;
53
54/// The actual analysis is done on the DOM side. We provide
55/// the actual base functionality in this struct, so the DOM
56/// just has to do basic shimming
57#[derive(MallocSizeOf)]
58pub struct AnalysisEngine {
59    /// The number of past sample-frames to consider in the FFT
60    fft_size: usize,
61    smoothing_constant: f64,
62    min_decibels: f64,
63    max_decibels: f64,
64    /// This is a ring buffer containing the last MAX_FFT_SIZE
65    /// sample-frames
66    #[ignore_malloc_size_of = "Do not know how"]
67    data: Box<[f32; MAX_FFT_SIZE]>,
68    /// The index of the current block
69    current_block: usize,
70    /// Have we computed the FFT already?
71    fft_computed: bool,
72    /// Cached blackman window data
73    blackman_windows: Vec<f32>,
74    /// The smoothed FFT data (in frequency domain)
75    smoothed_fft_data: Vec<f32>,
76    /// The computed FFT data, in decibels
77    computed_fft_data: Vec<f32>,
78    /// The windowed time domain data
79    /// Used during FFT computation
80    windowed: Vec<f32>,
81}
82
83impl AnalysisEngine {
84    pub fn new(
85        fft_size: usize,
86        smoothing_constant: f64,
87        min_decibels: f64,
88        max_decibels: f64,
89    ) -> Self {
90        debug_assert!((32..=32768).contains(&fft_size));
91        // must be a power of two
92        debug_assert!(fft_size & (fft_size - 1) == 0);
93        debug_assert!((0. ..=1.).contains(&smoothing_constant));
94        debug_assert!(max_decibels > min_decibels);
95        Self {
96            fft_size,
97            smoothing_constant,
98            min_decibels,
99            max_decibels,
100            data: Box::new([0.; MAX_FFT_SIZE]),
101            current_block: MAX_BLOCK_COUNT - 1,
102            fft_computed: false,
103            blackman_windows: Vec::with_capacity(fft_size),
104            computed_fft_data: Vec::with_capacity(fft_size / 2),
105            smoothed_fft_data: Vec::with_capacity(fft_size / 2),
106            windowed: Vec::with_capacity(fft_size),
107        }
108    }
109
110    pub fn set_fft_size(&mut self, fft_size: usize) {
111        debug_assert!((32..=32768).contains(&fft_size));
112        // must be a power of two
113        debug_assert!(fft_size & (fft_size - 1) == 0);
114        self.fft_size = fft_size;
115        self.fft_computed = false;
116    }
117
118    pub fn get_fft_size(&self) -> usize {
119        self.fft_size
120    }
121
122    pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) {
123        debug_assert!((0. ..=1.).contains(&smoothing_constant));
124        self.smoothing_constant = smoothing_constant;
125        self.fft_computed = false;
126    }
127
128    pub fn get_smoothing_constant(&self) -> f64 {
129        self.smoothing_constant
130    }
131
132    pub fn set_min_decibels(&mut self, min_decibels: f64) {
133        debug_assert!(min_decibels < self.max_decibels);
134        self.min_decibels = min_decibels;
135    }
136
137    pub fn get_min_decibels(&self) -> f64 {
138        self.min_decibels
139    }
140
141    pub fn set_max_decibels(&mut self, max_decibels: f64) {
142        debug_assert!(self.min_decibels < max_decibels);
143        self.max_decibels = max_decibels;
144    }
145
146    pub fn get_max_decibels(&self) -> f64 {
147        self.max_decibels
148    }
149
150    fn advance(&mut self) {
151        self.current_block += 1;
152        if self.current_block >= MAX_BLOCK_COUNT {
153            self.current_block = 0;
154        }
155    }
156
157    /// Get the data of the current block
158    fn curent_block_mut(&mut self) -> &mut [f32] {
159        let index = FRAMES_PER_BLOCK_USIZE * self.current_block;
160        &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)]
161    }
162
163    /// Given an index from 0 to fft_size, convert it into an index into
164    /// the backing array
165    fn convert_index(&self, index: usize) -> usize {
166        let offset = self.fft_size - index;
167        let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1;
168        if offset > last_element {
169            MAX_FFT_SIZE - offset + last_element
170        } else {
171            last_element - offset
172        }
173    }
174
175    /// Given an index into the backing array, increment it
176    fn advance_index(&self, index: &mut usize) {
177        *index += 1;
178        if *index >= MAX_FFT_SIZE {
179            *index = 0;
180        }
181    }
182
183    pub fn push(&mut self, mut block: Block) {
184        debug_assert!(block.chan_count() == 1);
185        self.advance();
186        if !block.is_silence() {
187            self.curent_block_mut().copy_from_slice(block.data_mut());
188        }
189        self.fft_computed = false;
190    }
191
192    /// <https://webaudio.github.io/web-audio-api/#blackman-window>
193    fn compute_blackman_windows(&mut self) {
194        if self.blackman_windows.len() == self.fft_size {
195            return;
196        }
197        const ALPHA: f32 = 0.16;
198        const ALPHA_0: f32 = (1. - ALPHA) / 2.;
199        const ALPHA_1: f32 = 1. / 2.;
200        const ALPHA_2: f32 = ALPHA / 2.;
201        self.blackman_windows.resize(self.fft_size, 0.);
202        let coeff = PI * 2. / self.fft_size as f32;
203        for n in 0..self.fft_size {
204            self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() +
205                ALPHA_2 * (2. * coeff * n as f32).cos();
206        }
207    }
208
209    fn apply_blackman_window(&mut self) {
210        self.compute_blackman_windows();
211        self.windowed.resize(self.fft_size, 0.);
212
213        let mut data_idx = self.convert_index(0);
214        for n in 0..self.fft_size {
215            self.windowed[n] = self.blackman_windows[n] * self.data[data_idx];
216            self.advance_index(&mut data_idx);
217        }
218    }
219
220    fn compute_fft(&mut self) {
221        if self.fft_computed {
222            return;
223        }
224        self.fft_computed = true;
225        self.apply_blackman_window();
226        self.computed_fft_data.resize(self.fft_size / 2, 0.);
227        self.smoothed_fft_data.resize(self.fft_size / 2, 0.);
228
229        for k in 0..(self.fft_size / 2) {
230            let mut sum_real = 0.;
231            let mut sum_imaginary = 0.;
232            let factor = -2. * PI * k as f32 / self.fft_size as f32;
233            for n in 0..(self.fft_size) {
234                sum_real += self.windowed[n] * (factor * n as f32).cos();
235                sum_imaginary += self.windowed[n] * (factor * n as f32).sin();
236            }
237            let sum_real = sum_real / self.fft_size as f32;
238            let sum_imaginary = sum_imaginary / self.fft_size as f32;
239            let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt();
240            self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 +
241                (1. - self.smoothing_constant) * magnitude as f64)
242                as f32;
243            self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.);
244        }
245    }
246
247    pub fn fill_time_domain_data(&self, dest: &mut [f32]) {
248        let mut data_idx = self.convert_index(0);
249        let end = cmp::min(self.fft_size, dest.len());
250        for entry in &mut dest[0..end] {
251            *entry = self.data[data_idx];
252            self.advance_index(&mut data_idx);
253        }
254    }
255
256    pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) {
257        let mut data_idx = self.convert_index(0);
258        let end = cmp::min(self.fft_size, dest.len());
259        for entry in &mut dest[0..end] {
260            let result = 128. * (1. + self.data[data_idx]);
261            *entry = clamp_255(result);
262            self.advance_index(&mut data_idx)
263        }
264    }
265
266    pub fn fill_frequency_data(&mut self, dest: &mut [f32]) {
267        self.compute_fft();
268        let len = cmp::min(dest.len(), self.computed_fft_data.len());
269        dest[0..len].copy_from_slice(&self.computed_fft_data[0..len]);
270    }
271
272    pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) {
273        self.compute_fft();
274        let len = cmp::min(dest.len(), self.computed_fft_data.len());
275        let ratio = 255. / (self.max_decibels - self.min_decibels);
276        for (index, freq) in dest[0..len].iter_mut().enumerate() {
277            let result = ratio * (self.computed_fft_data[index] as f64 - self.min_decibels);
278            *freq = clamp_255(result as f32);
279        }
280    }
281}
282
283fn clamp_255(val: f32) -> u8 {
284    if val > 255. {
285        255
286    } else if val < 0. {
287        0
288    } else {
289        val as u8
290    }
291}
servo_media_audio/analyser_node.rs

servo_media_audio/
analyser_node.rs
