1use std::cell::Cell;
6use std::collections::hash_map::Entry;
7use std::collections::{HashMap, VecDeque};
8use std::rc::Rc;
9use std::sync::{Arc, Mutex};
10
11use dom_struct::dom_struct;
12use js::context::JSContext;
13use js::realm::CurrentRealm;
14use js::rust::CustomAutoRooterGuard;
15use js::typedarray::ArrayBuffer;
16use script_bindings::cell::DomRefCell;
17use script_bindings::cformat;
18use servo_base::id::PipelineId;
19use servo_media::audio::context::{
20 AudioContext, AudioContextOptions, OfflineAudioContextOptions, ProcessingState,
21 RealTimeAudioContextOptions,
22};
23use servo_media::audio::decoder::AudioDecoderCallbacksBuilder;
24use servo_media::audio::graph::NodeId;
25use servo_media::{ClientContextId, ServoMedia};
26use uuid::Uuid;
27
28use crate::conversions::Convert;
29use crate::dom::audio::analysernode::AnalyserNode;
30use crate::dom::audio::audiobuffer::AudioBuffer;
31use crate::dom::audio::audiobuffersourcenode::AudioBufferSourceNode;
32use crate::dom::audio::audiodestinationnode::AudioDestinationNode;
33use crate::dom::audio::audiolistener::AudioListener;
34use crate::dom::audio::audionode::MAX_CHANNEL_COUNT;
35use crate::dom::audio::biquadfilternode::BiquadFilterNode;
36use crate::dom::audio::channelmergernode::ChannelMergerNode;
37use crate::dom::audio::channelsplitternode::ChannelSplitterNode;
38use crate::dom::audio::constantsourcenode::ConstantSourceNode;
39use crate::dom::audio::gainnode::GainNode;
40use crate::dom::audio::iirfilternode::IIRFilterNode;
41use crate::dom::audio::oscillatornode::OscillatorNode;
42use crate::dom::audio::pannernode::PannerNode;
43use crate::dom::audio::stereopannernode::StereoPannerNode;
44use crate::dom::bindings::callback::ExceptionHandling;
45use crate::dom::bindings::codegen::Bindings::AnalyserNodeBinding::AnalyserOptions;
46use crate::dom::bindings::codegen::Bindings::AudioBufferSourceNodeBinding::AudioBufferSourceOptions;
47use crate::dom::bindings::codegen::Bindings::AudioNodeBinding::{
48 AudioNodeOptions, ChannelCountMode, ChannelInterpretation,
49};
50use crate::dom::bindings::codegen::Bindings::BaseAudioContextBinding::{
51 AudioContextState, BaseAudioContextMethods, DecodeErrorCallback, DecodeSuccessCallback,
52};
53use crate::dom::bindings::codegen::Bindings::BiquadFilterNodeBinding::BiquadFilterOptions;
54use crate::dom::bindings::codegen::Bindings::ChannelMergerNodeBinding::ChannelMergerOptions;
55use crate::dom::bindings::codegen::Bindings::ChannelSplitterNodeBinding::ChannelSplitterOptions;
56use crate::dom::bindings::codegen::Bindings::ConstantSourceNodeBinding::ConstantSourceOptions;
57use crate::dom::bindings::codegen::Bindings::GainNodeBinding::GainOptions;
58use crate::dom::bindings::codegen::Bindings::IIRFilterNodeBinding::IIRFilterOptions;
59use crate::dom::bindings::codegen::Bindings::OscillatorNodeBinding::OscillatorOptions;
60use crate::dom::bindings::codegen::Bindings::PannerNodeBinding::PannerOptions;
61use crate::dom::bindings::codegen::Bindings::StereoPannerNodeBinding::StereoPannerOptions;
62use crate::dom::bindings::error::{Error, ErrorResult, Fallible};
63use crate::dom::bindings::inheritance::Castable;
64use crate::dom::bindings::num::Finite;
65use crate::dom::bindings::refcounted::Trusted;
66use crate::dom::bindings::reflector::DomGlobal;
67use crate::dom::bindings::root::{DomRoot, MutNullableDom};
68use crate::dom::domexception::{DOMErrorName, DOMException};
69use crate::dom::eventtarget::EventTarget;
70use crate::dom::promise::Promise;
71
72pub(crate) enum BaseAudioContextOptions {
73 AudioContext(RealTimeAudioContextOptions),
74 OfflineAudioContext(OfflineAudioContextOptions),
75}
76
77#[derive(JSTraceable, MallocSizeOf)]
78struct DecodeResolver {
79 #[conditional_malloc_size_of]
80 pub(crate) promise: Rc<Promise>,
81 #[conditional_malloc_size_of]
82 pub(crate) success_callback: Option<Rc<DecodeSuccessCallback>>,
83 #[conditional_malloc_size_of]
84 pub(crate) error_callback: Option<Rc<DecodeErrorCallback>>,
85}
86
87type BoxedSliceOfPromises = Box<[Rc<Promise>]>;
88
89#[dom_struct]
90pub(crate) struct BaseAudioContext {
91 eventtarget: EventTarget,
92 #[ignore_malloc_size_of = "servo_media"]
93 #[no_trace]
94 audio_context_impl: Arc<Mutex<AudioContext>>,
95 destination: MutNullableDom<AudioDestinationNode>,
97 listener: MutNullableDom<AudioListener>,
98 #[conditional_malloc_size_of]
100 in_flight_resume_promises_queue: DomRefCell<VecDeque<(BoxedSliceOfPromises, ErrorResult)>>,
101 #[conditional_malloc_size_of]
103 pending_resume_promises: DomRefCell<Vec<Rc<Promise>>>,
104 decode_resolvers: DomRefCell<HashMap<String, DecodeResolver>>,
105 sample_rate: f32,
107 state: Cell<AudioContextState>,
113 channel_count: u32,
114}
115
116impl BaseAudioContext {
117 pub(crate) fn new_inherited(
118 options: BaseAudioContextOptions,
119 pipeline_id: PipelineId,
120 ) -> Fallible<BaseAudioContext> {
121 let (sample_rate, channel_count) = match options {
122 BaseAudioContextOptions::AudioContext(ref opt) => (opt.sample_rate, 2),
123 BaseAudioContextOptions::OfflineAudioContext(ref opt) => {
124 (opt.sample_rate, opt.channels)
125 },
126 };
127
128 let client_context_id =
129 ClientContextId::build(pipeline_id.namespace_id.0, pipeline_id.index.0.get());
130 let audio_context_impl = ServoMedia::get()
131 .create_audio_context(&client_context_id, options.convert())
132 .map_err(|_| Error::NotSupported(None))?;
133
134 Ok(BaseAudioContext {
135 eventtarget: EventTarget::new_inherited(),
136 audio_context_impl,
137 destination: Default::default(),
138 listener: Default::default(),
139 in_flight_resume_promises_queue: Default::default(),
140 pending_resume_promises: Default::default(),
141 decode_resolvers: Default::default(),
142 sample_rate,
143 state: Cell::new(AudioContextState::Suspended),
144 channel_count: channel_count.into(),
145 })
146 }
147
148 pub(crate) fn is_offline(&self) -> bool {
150 false
151 }
152
153 pub(crate) fn audio_context_impl(&self) -> Arc<Mutex<AudioContext>> {
154 self.audio_context_impl.clone()
155 }
156
157 pub(crate) fn destination_node(&self) -> NodeId {
158 self.audio_context_impl.lock().unwrap().dest_node()
159 }
160
161 pub(crate) fn listener(&self) -> NodeId {
162 self.audio_context_impl.lock().unwrap().listener()
163 }
164
165 pub(crate) fn is_allowed_to_start(&self) -> bool {
167 self.state.get() == AudioContextState::Suspended
168 }
169
170 fn push_pending_resume_promise(&self, promise: &Rc<Promise>) {
171 self.pending_resume_promises
172 .borrow_mut()
173 .push(promise.clone());
174 }
175
176 fn take_pending_resume_promises(&self, result: ErrorResult) {
187 let pending_resume_promises =
188 std::mem::take(&mut *self.pending_resume_promises.borrow_mut());
189 self.in_flight_resume_promises_queue
190 .borrow_mut()
191 .push_back((pending_resume_promises.into(), result));
192 }
193
194 fn fulfill_in_flight_resume_promises<F>(&self, cx: &mut JSContext, f: F)
203 where
204 F: FnOnce(),
205 {
206 let (promises, result) = self
207 .in_flight_resume_promises_queue
208 .borrow_mut()
209 .pop_front()
210 .expect("there should be at least one list of in flight resume promises");
211 f();
212 for promise in &*promises {
213 match result {
214 Ok(ref value) => promise.resolve_native(cx, value),
215 Err(ref error) => promise.reject_error(cx, error.clone()),
216 }
217 }
218 }
219
220 pub(crate) fn control_thread_state(&self) -> ProcessingState {
222 self.audio_context_impl.lock().unwrap().state()
223 }
224
225 pub(crate) fn set_state_attribute(&self, state: AudioContextState) {
227 self.state.set(state);
228 }
229
230 pub(crate) fn resume(&self) {
231 let this = Trusted::new(self);
232 match self.audio_context_impl.lock().unwrap().resume() {
235 Some(()) => {
236 self.take_pending_resume_promises(Ok(()));
237 self.global().task_manager().dom_manipulation_task_source().queue(
238 task!(resume_success: move |cx| {
239 let this = this.root();
240 this.fulfill_in_flight_resume_promises(cx, || {
241 if this.state.get() != AudioContextState::Running {
242 this.state.set(AudioContextState::Running);
243 this.global().task_manager().dom_manipulation_task_source().queue_simple_event(
244 this.upcast(),
245 atom!("statechange"),
246 );
247 }
248 });
249 })
250 );
251 },
252 None => {
253 self.take_pending_resume_promises(Err(Error::Type(
254 c"Something went wrong".to_owned(),
255 )));
256 self.global()
257 .task_manager()
258 .dom_manipulation_task_source()
259 .queue(task!(resume_error: move |cx| {
260 this.root().fulfill_in_flight_resume_promises(cx, || {})
261 }));
262 },
263 }
264 }
265
266 pub(crate) fn channel_count(&self) -> u32 {
267 self.channel_count
268 }
269}
270
271impl BaseAudioContextMethods<crate::DomTypeHolder> for BaseAudioContext {
272 fn SampleRate(&self) -> Finite<f32> {
274 Finite::wrap(self.sample_rate)
275 }
276
277 fn CurrentTime(&self) -> Finite<f64> {
279 let current_time = self.audio_context_impl.lock().unwrap().current_time();
280 Finite::wrap(current_time)
281 }
282
283 fn State(&self) -> AudioContextState {
285 self.state.get()
286 }
287
288 fn Resume(&self, cx: &mut CurrentRealm) -> Rc<Promise> {
290 let promise = Promise::new_in_realm(cx);
292
293 if self.audio_context_impl.lock().unwrap().state() == ProcessingState::Closed {
295 promise.reject_error(cx, Error::InvalidState(None));
296 return promise;
297 }
298
299 if self.state.get() == AudioContextState::Running {
301 promise.resolve_native(cx, &());
302 return promise;
303 }
304
305 self.push_pending_resume_promise(&promise);
306
307 if !self.is_allowed_to_start() {
309 return promise;
310 }
311
312 self.resume();
314
315 promise
317 }
318
319 fn Destination(&self, cx: &mut JSContext) -> DomRoot<AudioDestinationNode> {
321 let global = self.global();
322 self.destination.or_init(|| {
323 let mut options = AudioNodeOptions::empty();
324 options.channelCount = Some(self.channel_count);
325 options.channelCountMode = Some(ChannelCountMode::Explicit);
326 options.channelInterpretation = Some(ChannelInterpretation::Speakers);
327 AudioDestinationNode::new(cx, &global, self, &options)
328 })
329 }
330
331 fn Listener(&self, cx: &mut JSContext) -> DomRoot<AudioListener> {
333 let global = self.global();
334 let window = global.as_window();
335 self.listener
336 .or_init(|| AudioListener::new(cx, window, self))
337 }
338
339 event_handler!(statechange, GetOnstatechange, SetOnstatechange);
341
342 fn CreateOscillator(&self, cx: &mut JSContext) -> Fallible<DomRoot<OscillatorNode>> {
344 OscillatorNode::new(
345 cx,
346 self.global().as_window(),
347 self,
348 &OscillatorOptions::empty(),
349 )
350 }
351
352 fn CreateGain(&self, cx: &mut JSContext) -> Fallible<DomRoot<GainNode>> {
354 GainNode::new(cx, self.global().as_window(), self, &GainOptions::empty())
355 }
356
357 fn CreatePanner(&self, cx: &mut JSContext) -> Fallible<DomRoot<PannerNode>> {
359 PannerNode::new(cx, self.global().as_window(), self, &PannerOptions::empty())
360 }
361
362 fn CreateAnalyser(&self, cx: &mut JSContext) -> Fallible<DomRoot<AnalyserNode>> {
364 AnalyserNode::new(
365 cx,
366 self.global().as_window(),
367 self,
368 &AnalyserOptions::empty(),
369 )
370 }
371
372 fn CreateBiquadFilter(&self, cx: &mut JSContext) -> Fallible<DomRoot<BiquadFilterNode>> {
374 BiquadFilterNode::new(
375 cx,
376 self.global().as_window(),
377 self,
378 &BiquadFilterOptions::empty(),
379 )
380 }
381
382 fn CreateStereoPanner(&self, cx: &mut JSContext) -> Fallible<DomRoot<StereoPannerNode>> {
384 StereoPannerNode::new(
385 cx,
386 self.global().as_window(),
387 self,
388 &StereoPannerOptions::empty(),
389 )
390 }
391
392 fn CreateConstantSource(&self, cx: &mut JSContext) -> Fallible<DomRoot<ConstantSourceNode>> {
394 ConstantSourceNode::new(
395 cx,
396 self.global().as_window(),
397 self,
398 &ConstantSourceOptions::empty(),
399 )
400 }
401
402 fn CreateChannelMerger(
404 &self,
405 cx: &mut JSContext,
406 count: u32,
407 ) -> Fallible<DomRoot<ChannelMergerNode>> {
408 let mut opts = ChannelMergerOptions::empty();
409 opts.numberOfInputs = count;
410 ChannelMergerNode::new(cx, self.global().as_window(), self, &opts)
411 }
412
413 fn CreateChannelSplitter(
415 &self,
416 cx: &mut JSContext,
417 count: u32,
418 ) -> Fallible<DomRoot<ChannelSplitterNode>> {
419 let mut opts = ChannelSplitterOptions::empty();
420 opts.numberOfOutputs = count;
421 ChannelSplitterNode::new(cx, self.global().as_window(), self, &opts)
422 }
423
424 fn CreateBuffer(
426 &self,
427 cx: &mut JSContext,
428 number_of_channels: u32,
429 length: u32,
430 sample_rate: Finite<f32>,
431 ) -> Fallible<DomRoot<AudioBuffer>> {
432 if number_of_channels == 0 ||
433 number_of_channels > MAX_CHANNEL_COUNT ||
434 length == 0 ||
435 *sample_rate <= 0.
436 {
437 return Err(Error::NotSupported(None));
438 }
439 Ok(AudioBuffer::new(
440 cx,
441 self.global().as_window(),
442 number_of_channels,
443 length,
444 *sample_rate,
445 None,
446 ))
447 }
448
449 fn CreateBufferSource(&self, cx: &mut JSContext) -> Fallible<DomRoot<AudioBufferSourceNode>> {
451 AudioBufferSourceNode::new(
452 cx,
453 self.global().as_window(),
454 self,
455 &AudioBufferSourceOptions::empty(),
456 )
457 }
458
459 fn DecodeAudioData(
461 &self,
462 cx: &mut CurrentRealm,
463 audio_data: CustomAutoRooterGuard<ArrayBuffer>,
464 decode_success_callback: Option<Rc<DecodeSuccessCallback>>,
465 decode_error_callback: Option<Rc<DecodeErrorCallback>>,
466 ) -> Rc<Promise> {
467 let promise = Promise::new_in_realm(cx);
469
470 if audio_data.len() > 0 {
471 let uuid = Uuid::new_v4().simple().to_string();
474 let uuid_ = uuid.clone();
475 self.decode_resolvers.borrow_mut().insert(
476 uuid.clone(),
477 DecodeResolver {
478 promise: promise.clone(),
479 success_callback: decode_success_callback,
480 error_callback: decode_error_callback,
481 },
482 );
483 let audio_data = audio_data.to_vec();
484 let decoded_audio = Arc::new(Mutex::new(Vec::new()));
485 let decoded_audio_ = decoded_audio.clone();
486 let decoded_audio__ = decoded_audio.clone();
487 let channels = Arc::new(Mutex::new(HashMap::new()));
492 let this = Trusted::new(self);
493 let this_ = this.clone();
494 let task_source = self
495 .global()
496 .task_manager()
497 .dom_manipulation_task_source()
498 .to_sendable();
499 let task_source_clone = task_source.clone();
500 let callbacks = AudioDecoderCallbacksBuilder::default()
501 .ready(move |channel_count| {
502 decoded_audio
503 .lock()
504 .unwrap()
505 .resize(channel_count as usize, Vec::new());
506 })
507 .progress(move |buffer, channel_pos_mask| {
508 let mut decoded_audio = decoded_audio_.lock().unwrap();
509 let mut channels = channels.lock().unwrap();
510 let channel = match channels.entry(channel_pos_mask) {
511 Entry::Occupied(entry) => *entry.get(),
512 Entry::Vacant(entry) => {
513 let x = (channel_pos_mask as f32).log2() as usize;
514 *entry.insert(x)
515 },
516 };
517 decoded_audio[channel].extend_from_slice((*buffer).as_ref());
518 })
519 .eos(move || {
520 task_source.queue(task!(audio_decode_eos: move |cx| {
521 let this = this.root();
522 let decoded_audio = decoded_audio__.lock().unwrap();
523 let length = if !decoded_audio.is_empty() {
524 decoded_audio[0].len()
525 } else {
526 0
527 };
528 let buffer = AudioBuffer::new(
529 cx,
530 this.global().as_window(),
531 decoded_audio.len() as u32 ,
532 length as u32,
533 this.sample_rate,
534 Some(decoded_audio.as_slice()),
535 );
536 let mut resolvers = this.decode_resolvers.borrow_mut();
537 assert!(resolvers.contains_key(&uuid_));
538 let resolver = resolvers.remove(&uuid_).unwrap();
539 if let Some(callback) = resolver.success_callback {
540 let _ = callback.Call__(cx, &buffer, ExceptionHandling::Report);
541 }
542 resolver.promise.resolve_native(cx, &buffer);
543 }));
544 })
545 .error(move |error| {
546 task_source_clone.queue(task!(audio_decode_eos: move |cx| {
547 let this = this_.root();
548 let mut resolvers = this.decode_resolvers.borrow_mut();
549 assert!(resolvers.contains_key(&uuid));
550 let resolver = resolvers.remove(&uuid).unwrap();
551 if let Some(callback) = resolver.error_callback {
552 let exception = DOMException::new(cx,
553 &this.global(),
554 DOMErrorName::DataCloneError,
555 );
556 let _ = callback.Call__(cx, &exception, ExceptionHandling::Report);
557 }
558 let error = cformat!("Audio decode error {:?}", error);
559 resolver.promise.reject_error(cx, Error::Type(error));
560 }));
561 })
562 .build();
563 self.audio_context_impl
564 .lock()
565 .unwrap()
566 .decode_audio_data(audio_data, callbacks);
567 } else {
568 promise.reject_error(cx, Error::DataClone(None));
570 return promise;
571 }
572
573 promise
575 }
576
577 fn CreateIIRFilter(
579 &self,
580 cx: &mut JSContext,
581 feedforward: Vec<Finite<f64>>,
582 feedback: Vec<Finite<f64>>,
583 ) -> Fallible<DomRoot<IIRFilterNode>> {
584 let opts = IIRFilterOptions {
585 parent: AudioNodeOptions::empty(),
586 feedback,
587 feedforward,
588 };
589 IIRFilterNode::new(cx, self.global().as_window(), self, &opts)
590 }
591}
592
593impl Convert<AudioContextOptions> for BaseAudioContextOptions {
594 fn convert(self) -> AudioContextOptions {
595 match self {
596 BaseAudioContextOptions::AudioContext(options) => {
597 AudioContextOptions::RealTimeAudioContext(options)
598 },
599 BaseAudioContextOptions::OfflineAudioContext(options) => {
600 AudioContextOptions::OfflineAudioContext(options)
601 },
602 }
603 }
604}
605
606impl Convert<AudioContextState> for ProcessingState {
607 fn convert(self) -> AudioContextState {
608 match self {
609 ProcessingState::Suspended => AudioContextState::Suspended,
610 ProcessingState::Running => AudioContextState::Running,
611 ProcessingState::Closed => AudioContextState::Closed,
612 }
613 }
614}