servo_media_audio/
analyser_node.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cmp;
6use std::f32::consts::PI;
7
8use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE};
9use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo, ChannelInterpretation};
10
11#[derive(AudioNodeCommon)]
12pub(crate) struct AnalyserNode {
13    channel_info: ChannelInfo,
14    callback: Box<dyn FnMut(Block) + Send>,
15}
16
17impl AnalyserNode {
18    pub fn new(callback: Box<dyn FnMut(Block) + Send>, channel_info: ChannelInfo) -> Self {
19        Self {
20            callback,
21            channel_info,
22        }
23    }
24}
25
26impl AudioNodeEngine for AnalyserNode {
27    fn node_type(&self) -> AudioNodeType {
28        AudioNodeType::AnalyserNode
29    }
30
31    fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk {
32        debug_assert!(inputs.len() == 1);
33
34        let mut push = inputs.blocks[0].clone();
35        push.mix(1, ChannelInterpretation::Speakers);
36
37        (self.callback)(push);
38
39        // analyser node doesn't modify the inputs
40        inputs
41    }
42}
43
44/// From <https://webaudio.github.io/web-audio-api/#dom-analysernode-fftsize>
45pub const MAX_FFT_SIZE: usize = 32768;
46pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE;
47
48/// The actual analysis is done on the DOM side. We provide
49/// the actual base functionality in this struct, so the DOM
50/// just has to do basic shimming
51pub struct AnalysisEngine {
52    /// The number of past sample-frames to consider in the FFT
53    fft_size: usize,
54    smoothing_constant: f64,
55    min_decibels: f64,
56    max_decibels: f64,
57    /// This is a ring buffer containing the last MAX_FFT_SIZE
58    /// sample-frames
59    data: Box<[f32; MAX_FFT_SIZE]>,
60    /// The index of the current block
61    current_block: usize,
62    /// Have we computed the FFT already?
63    fft_computed: bool,
64    /// Cached blackman window data
65    blackman_windows: Vec<f32>,
66    /// The smoothed FFT data (in frequency domain)
67    smoothed_fft_data: Vec<f32>,
68    /// The computed FFT data, in decibels
69    computed_fft_data: Vec<f32>,
70    /// The windowed time domain data
71    /// Used during FFT computation
72    windowed: Vec<f32>,
73}
74
75impl AnalysisEngine {
76    pub fn new(
77        fft_size: usize,
78        smoothing_constant: f64,
79        min_decibels: f64,
80        max_decibels: f64,
81    ) -> Self {
82        debug_assert!((32..=32768).contains(&fft_size));
83        // must be a power of two
84        debug_assert!(fft_size & (fft_size - 1) == 0);
85        debug_assert!((0. ..=1.).contains(&smoothing_constant));
86        debug_assert!(max_decibels > min_decibels);
87        Self {
88            fft_size,
89            smoothing_constant,
90            min_decibels,
91            max_decibels,
92            data: Box::new([0.; MAX_FFT_SIZE]),
93            current_block: MAX_BLOCK_COUNT - 1,
94            fft_computed: false,
95            blackman_windows: Vec::with_capacity(fft_size),
96            computed_fft_data: Vec::with_capacity(fft_size / 2),
97            smoothed_fft_data: Vec::with_capacity(fft_size / 2),
98            windowed: Vec::with_capacity(fft_size),
99        }
100    }
101
102    pub fn set_fft_size(&mut self, fft_size: usize) {
103        debug_assert!((32..=32768).contains(&fft_size));
104        // must be a power of two
105        debug_assert!(fft_size & (fft_size - 1) == 0);
106        self.fft_size = fft_size;
107        self.fft_computed = false;
108    }
109
110    pub fn get_fft_size(&self) -> usize {
111        self.fft_size
112    }
113
114    pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) {
115        debug_assert!((0. ..=1.).contains(&smoothing_constant));
116        self.smoothing_constant = smoothing_constant;
117        self.fft_computed = false;
118    }
119
120    pub fn get_smoothing_constant(&self) -> f64 {
121        self.smoothing_constant
122    }
123
124    pub fn set_min_decibels(&mut self, min_decibels: f64) {
125        debug_assert!(min_decibels < self.max_decibels);
126        self.min_decibels = min_decibels;
127    }
128
129    pub fn get_min_decibels(&self) -> f64 {
130        self.min_decibels
131    }
132
133    pub fn set_max_decibels(&mut self, max_decibels: f64) {
134        debug_assert!(self.min_decibels < max_decibels);
135        self.max_decibels = max_decibels;
136    }
137
138    pub fn get_max_decibels(&self) -> f64 {
139        self.max_decibels
140    }
141
142    fn advance(&mut self) {
143        self.current_block += 1;
144        if self.current_block >= MAX_BLOCK_COUNT {
145            self.current_block = 0;
146        }
147    }
148
149    /// Get the data of the current block
150    fn curent_block_mut(&mut self) -> &mut [f32] {
151        let index = FRAMES_PER_BLOCK_USIZE * self.current_block;
152        &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)]
153    }
154
155    /// Given an index from 0 to fft_size, convert it into an index into
156    /// the backing array
157    fn convert_index(&self, index: usize) -> usize {
158        let offset = self.fft_size - index;
159        let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1;
160        if offset > last_element {
161            MAX_FFT_SIZE - offset + last_element
162        } else {
163            last_element - offset
164        }
165    }
166
167    /// Given an index into the backing array, increment it
168    fn advance_index(&self, index: &mut usize) {
169        *index += 1;
170        if *index >= MAX_FFT_SIZE {
171            *index = 0;
172        }
173    }
174
175    pub fn push(&mut self, mut block: Block) {
176        debug_assert!(block.chan_count() == 1);
177        self.advance();
178        if !block.is_silence() {
179            self.curent_block_mut().copy_from_slice(block.data_mut());
180        }
181        self.fft_computed = false;
182    }
183
184    /// <https://webaudio.github.io/web-audio-api/#blackman-window>
185    fn compute_blackman_windows(&mut self) {
186        if self.blackman_windows.len() == self.fft_size {
187            return;
188        }
189        const ALPHA: f32 = 0.16;
190        const ALPHA_0: f32 = (1. - ALPHA) / 2.;
191        const ALPHA_1: f32 = 1. / 2.;
192        const ALPHA_2: f32 = ALPHA / 2.;
193        self.blackman_windows.resize(self.fft_size, 0.);
194        let coeff = PI * 2. / self.fft_size as f32;
195        for n in 0..self.fft_size {
196            self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() +
197                ALPHA_2 * (2. * coeff * n as f32).cos();
198        }
199    }
200
201    fn apply_blackman_window(&mut self) {
202        self.compute_blackman_windows();
203        self.windowed.resize(self.fft_size, 0.);
204
205        let mut data_idx = self.convert_index(0);
206        for n in 0..self.fft_size {
207            self.windowed[n] = self.blackman_windows[n] * self.data[data_idx];
208            self.advance_index(&mut data_idx);
209        }
210    }
211
212    fn compute_fft(&mut self) {
213        if self.fft_computed {
214            return;
215        }
216        self.fft_computed = true;
217        self.apply_blackman_window();
218        self.computed_fft_data.resize(self.fft_size / 2, 0.);
219        self.smoothed_fft_data.resize(self.fft_size / 2, 0.);
220
221        for k in 0..(self.fft_size / 2) {
222            let mut sum_real = 0.;
223            let mut sum_imaginary = 0.;
224            let factor = -2. * PI * k as f32 / self.fft_size as f32;
225            for n in 0..(self.fft_size) {
226                sum_real += self.windowed[n] * (factor * n as f32).cos();
227                sum_imaginary += self.windowed[n] * (factor * n as f32).sin();
228            }
229            let sum_real = sum_real / self.fft_size as f32;
230            let sum_imaginary = sum_imaginary / self.fft_size as f32;
231            let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt();
232            self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 +
233                (1. - self.smoothing_constant) * magnitude as f64)
234                as f32;
235            self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.);
236        }
237    }
238
239    pub fn fill_time_domain_data(&self, dest: &mut [f32]) {
240        let mut data_idx = self.convert_index(0);
241        let end = cmp::min(self.fft_size, dest.len());
242        for entry in &mut dest[0..end] {
243            *entry = self.data[data_idx];
244            self.advance_index(&mut data_idx);
245        }
246    }
247
248    pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) {
249        let mut data_idx = self.convert_index(0);
250        let end = cmp::min(self.fft_size, dest.len());
251        for entry in &mut dest[0..end] {
252            let result = 128. * (1. + self.data[data_idx]);
253            *entry = clamp_255(result);
254            self.advance_index(&mut data_idx)
255        }
256    }
257
258    pub fn fill_frequency_data(&mut self, dest: &mut [f32]) {
259        self.compute_fft();
260        let len = cmp::min(dest.len(), self.computed_fft_data.len());
261        dest[0..len].copy_from_slice(&self.computed_fft_data[0..len]);
262    }
263
264    pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) {
265        self.compute_fft();
266        let len = cmp::min(dest.len(), self.computed_fft_data.len());
267        let ratio = 255. / (self.max_decibels - self.min_decibels);
268        for (index, freq) in dest[0..len].iter_mut().enumerate() {
269            let result = ratio * (self.computed_fft_data[index] as f64 - self.min_decibels);
270            *freq = clamp_255(result as f32);
271        }
272    }
273}
274
275fn clamp_255(val: f32) -> u8 {
276    if val > 255. {
277        255
278    } else if val < 0. {
279        0
280    } else {
281        val as u8
282    }
283}
servo_media_audio/analyser_node.rs

servo_media_audio/
analyser_node.rs
