1use std::cell::Cell;
6use std::collections::hash_map::Entry;
7use std::collections::{HashMap, VecDeque};
8use std::rc::Rc;
9use std::sync::{Arc, Mutex};
10
11use dom_struct::dom_struct;
12use js::context::JSContext;
13use js::jsapi::IsDetachedArrayBufferObject;
14use js::realm::CurrentRealm;
15use js::rust::CustomAutoRooterGuard;
16use js::typedarray::ArrayBuffer;
17use script_bindings::cell::DomRefCell;
18use script_bindings::cformat;
19use script_bindings::codegen::GenericBindings::PeriodicWaveBinding::PeriodicWaveMethods;
20use servo_base::id::PipelineId;
21use servo_media::audio::context::{
22 AudioContext, AudioContextOptions, OfflineAudioContextOptions, ProcessingState,
23 RealTimeAudioContextOptions,
24};
25use servo_media::audio::decoder::AudioDecoderCallbacksBuilder;
26use servo_media::audio::graph::NodeId;
27use servo_media::{ClientContextId, ServoMedia};
28use uuid::Uuid;
29
30use crate::conversions::Convert;
31use crate::dom::audio::analysernode::AnalyserNode;
32use crate::dom::audio::audiobuffer::AudioBuffer;
33use crate::dom::audio::audiobuffersourcenode::AudioBufferSourceNode;
34use crate::dom::audio::audiodestinationnode::AudioDestinationNode;
35use crate::dom::audio::audiolistener::AudioListener;
36use crate::dom::audio::audionode::MAX_CHANNEL_COUNT;
37use crate::dom::audio::biquadfilternode::BiquadFilterNode;
38use crate::dom::audio::channelmergernode::ChannelMergerNode;
39use crate::dom::audio::channelsplitternode::ChannelSplitterNode;
40use crate::dom::audio::constantsourcenode::ConstantSourceNode;
41use crate::dom::audio::gainnode::GainNode;
42use crate::dom::audio::iirfilternode::IIRFilterNode;
43use crate::dom::audio::oscillatornode::OscillatorNode;
44use crate::dom::audio::pannernode::PannerNode;
45use crate::dom::audio::stereopannernode::StereoPannerNode;
46use crate::dom::bindings::callback::ExceptionHandling;
47use crate::dom::bindings::codegen::Bindings::AnalyserNodeBinding::AnalyserOptions;
48use crate::dom::bindings::codegen::Bindings::AudioBufferSourceNodeBinding::AudioBufferSourceOptions;
49use crate::dom::bindings::codegen::Bindings::AudioNodeBinding::{
50 AudioNodeOptions, ChannelCountMode, ChannelInterpretation,
51};
52use crate::dom::bindings::codegen::Bindings::BaseAudioContextBinding::{
53 AudioContextState, BaseAudioContextMethods, DecodeErrorCallback, DecodeSuccessCallback,
54};
55use crate::dom::bindings::codegen::Bindings::BiquadFilterNodeBinding::BiquadFilterOptions;
56use crate::dom::bindings::codegen::Bindings::ChannelMergerNodeBinding::ChannelMergerOptions;
57use crate::dom::bindings::codegen::Bindings::ChannelSplitterNodeBinding::ChannelSplitterOptions;
58use crate::dom::bindings::codegen::Bindings::ConstantSourceNodeBinding::ConstantSourceOptions;
59use crate::dom::bindings::codegen::Bindings::GainNodeBinding::GainOptions;
60use crate::dom::bindings::codegen::Bindings::IIRFilterNodeBinding::IIRFilterOptions;
61use crate::dom::bindings::codegen::Bindings::OscillatorNodeBinding::OscillatorOptions;
62use crate::dom::bindings::codegen::Bindings::PannerNodeBinding::PannerOptions;
63use crate::dom::bindings::codegen::Bindings::PeriodicWaveBinding::{
64 PeriodicWaveConstraints, PeriodicWaveOptions,
65};
66use crate::dom::bindings::codegen::Bindings::StereoPannerNodeBinding::StereoPannerOptions;
67use crate::dom::bindings::codegen::Bindings::WindowBinding::WindowMethods;
68use crate::dom::bindings::error::{Error, ErrorResult, Fallible};
69use crate::dom::bindings::inheritance::Castable;
70use crate::dom::bindings::num::Finite;
71use crate::dom::bindings::refcounted::Trusted;
72use crate::dom::bindings::reflector::DomGlobal;
73use crate::dom::bindings::root::{DomRoot, MutNullableDom};
74use crate::dom::domexception::{DOMErrorName, DOMException};
75use crate::dom::eventtarget::EventTarget;
76use crate::dom::promise::Promise;
77use crate::dom::types::PeriodicWave;
78
79pub(crate) enum BaseAudioContextOptions {
80 AudioContext(RealTimeAudioContextOptions),
81 OfflineAudioContext(OfflineAudioContextOptions),
82}
83
84#[derive(JSTraceable, MallocSizeOf)]
85struct DecodeResolver {
86 #[conditional_malloc_size_of]
87 pub(crate) promise: Rc<Promise>,
88 #[conditional_malloc_size_of]
89 pub(crate) success_callback: Option<Rc<DecodeSuccessCallback>>,
90 #[conditional_malloc_size_of]
91 pub(crate) error_callback: Option<Rc<DecodeErrorCallback>>,
92}
93
94type BoxedSliceOfPromises = Box<[Rc<Promise>]>;
95
96#[dom_struct]
97pub(crate) struct BaseAudioContext {
98 eventtarget: EventTarget,
99 #[ignore_malloc_size_of = "servo_media"]
100 #[no_trace]
101 audio_context_impl: Arc<Mutex<AudioContext>>,
102 destination: MutNullableDom<AudioDestinationNode>,
104 listener: MutNullableDom<AudioListener>,
105 #[conditional_malloc_size_of]
107 in_flight_resume_promises_queue: DomRefCell<VecDeque<(BoxedSliceOfPromises, ErrorResult)>>,
108 #[conditional_malloc_size_of]
110 pending_resume_promises: DomRefCell<Vec<Rc<Promise>>>,
111 decode_resolvers: DomRefCell<HashMap<String, DecodeResolver>>,
112 sample_rate: f32,
114 state: Cell<AudioContextState>,
120 channel_count: u32,
121}
122
123impl BaseAudioContext {
124 pub(crate) fn new_inherited(
125 options: BaseAudioContextOptions,
126 pipeline_id: PipelineId,
127 ) -> Fallible<BaseAudioContext> {
128 let (sample_rate, channel_count) = match options {
129 BaseAudioContextOptions::AudioContext(ref opt) => (opt.sample_rate, 2),
130 BaseAudioContextOptions::OfflineAudioContext(ref opt) => {
131 (opt.sample_rate, opt.channels)
132 },
133 };
134
135 let client_context_id =
136 ClientContextId::build(pipeline_id.namespace_id.0, pipeline_id.index.0.get());
137 let audio_context_impl = ServoMedia::get()
138 .create_audio_context(&client_context_id, options.convert())
139 .map_err(|_| Error::NotSupported(None))?;
140
141 Ok(BaseAudioContext {
142 eventtarget: EventTarget::new_inherited(),
143 audio_context_impl,
144 destination: Default::default(),
145 listener: Default::default(),
146 in_flight_resume_promises_queue: Default::default(),
147 pending_resume_promises: Default::default(),
148 decode_resolvers: Default::default(),
149 sample_rate,
150 state: Cell::new(AudioContextState::Suspended),
151 channel_count: channel_count.into(),
152 })
153 }
154
155 pub(crate) fn is_offline(&self) -> bool {
157 false
158 }
159
160 pub(crate) fn audio_context_impl(&self) -> Arc<Mutex<AudioContext>> {
161 self.audio_context_impl.clone()
162 }
163
164 pub(crate) fn destination_node(&self) -> NodeId {
165 self.audio_context_impl.lock().unwrap().dest_node()
166 }
167
168 pub(crate) fn listener(&self) -> NodeId {
169 self.audio_context_impl.lock().unwrap().listener()
170 }
171
172 pub(crate) fn is_allowed_to_start(&self) -> bool {
174 self.state.get() == AudioContextState::Suspended
175 }
176
177 #[expect(unsafe_code)]
179 fn is_detached_array_buffer(array_buffer: &ArrayBuffer) -> bool {
180 unsafe { IsDetachedArrayBufferObject(*array_buffer.underlying_object()) }
181 }
182
183 fn push_pending_resume_promise(&self, promise: &Rc<Promise>) {
184 self.pending_resume_promises
185 .borrow_mut()
186 .push(promise.clone());
187 }
188
189 fn take_pending_resume_promises(&self, result: ErrorResult) {
200 let pending_resume_promises =
201 std::mem::take(&mut *self.pending_resume_promises.borrow_mut());
202 self.in_flight_resume_promises_queue
203 .borrow_mut()
204 .push_back((pending_resume_promises.into(), result));
205 }
206
207 fn fulfill_in_flight_resume_promises<F>(&self, cx: &mut JSContext, f: F)
216 where
217 F: FnOnce(),
218 {
219 let (promises, result) = self
220 .in_flight_resume_promises_queue
221 .borrow_mut()
222 .pop_front()
223 .expect("there should be at least one list of in flight resume promises");
224 f();
225 for promise in &*promises {
226 match result {
227 Ok(ref value) => promise.resolve_native(cx, value),
228 Err(ref error) => promise.reject_error(cx, error.clone()),
229 }
230 }
231 }
232
233 pub(crate) fn control_thread_state(&self) -> ProcessingState {
235 self.audio_context_impl.lock().unwrap().state()
236 }
237
238 pub(crate) fn set_state_attribute(&self, state: AudioContextState) {
240 self.state.set(state);
241 }
242
243 pub(crate) fn resume(&self) {
244 let this = Trusted::new(self);
245 match self.audio_context_impl.lock().unwrap().resume() {
248 Some(()) => {
249 self.take_pending_resume_promises(Ok(()));
250 self.global().task_manager().dom_manipulation_task_source().queue(
251 task!(resume_success: move |cx| {
252 let this = this.root();
253 this.fulfill_in_flight_resume_promises(cx, || {
254 if this.state.get() != AudioContextState::Running {
255 this.state.set(AudioContextState::Running);
256 this.global().task_manager().dom_manipulation_task_source().queue_simple_event(
257 this.upcast(),
258 atom!("statechange"),
259 );
260 }
261 });
262 })
263 );
264 },
265 None => {
266 self.take_pending_resume_promises(Err(Error::Type(
267 c"Something went wrong".to_owned(),
268 )));
269 self.global()
270 .task_manager()
271 .dom_manipulation_task_source()
272 .queue(task!(resume_error: move |cx| {
273 this.root().fulfill_in_flight_resume_promises(cx, || {})
274 }));
275 },
276 }
277 }
278
279 pub(crate) fn channel_count(&self) -> u32 {
280 self.channel_count
281 }
282}
283
284impl BaseAudioContextMethods<crate::DomTypeHolder> for BaseAudioContext {
285 fn SampleRate(&self) -> Finite<f32> {
287 Finite::wrap(self.sample_rate)
288 }
289
290 fn CurrentTime(&self) -> Finite<f64> {
292 let current_time = self.audio_context_impl.lock().unwrap().current_time();
293 Finite::wrap(current_time)
294 }
295
296 fn State(&self) -> AudioContextState {
298 self.state.get()
299 }
300
301 fn Resume(&self, cx: &mut CurrentRealm) -> Rc<Promise> {
303 let promise = Promise::new_in_realm(cx);
305
306 if self.audio_context_impl.lock().unwrap().state() == ProcessingState::Closed {
308 promise.reject_error(cx, Error::InvalidState(None));
309 return promise;
310 }
311
312 if self.state.get() == AudioContextState::Running {
314 promise.resolve_native(cx, &());
315 return promise;
316 }
317
318 self.push_pending_resume_promise(&promise);
319
320 if !self.is_allowed_to_start() {
322 return promise;
323 }
324
325 self.resume();
327
328 promise
330 }
331
332 fn Destination(&self, cx: &mut JSContext) -> DomRoot<AudioDestinationNode> {
334 let global = self.global();
335 self.destination.or_init(|| {
336 let mut options = AudioNodeOptions::empty();
337 options.channelCount = Some(self.channel_count);
338 options.channelCountMode = Some(ChannelCountMode::Explicit);
339 options.channelInterpretation = Some(ChannelInterpretation::Speakers);
340 AudioDestinationNode::new(cx, &global, self, &options)
341 })
342 }
343
344 fn Listener(&self, cx: &mut JSContext) -> DomRoot<AudioListener> {
346 let global = self.global();
347 let window = global.as_window();
348 self.listener
349 .or_init(|| AudioListener::new(cx, window, self))
350 }
351
352 event_handler!(statechange, GetOnstatechange, SetOnstatechange);
354
355 fn CreateOscillator(&self, cx: &mut JSContext) -> Fallible<DomRoot<OscillatorNode>> {
357 OscillatorNode::new(
358 cx,
359 self.global().as_window(),
360 self,
361 &OscillatorOptions::empty(),
362 )
363 }
364
365 fn CreateGain(&self, cx: &mut JSContext) -> Fallible<DomRoot<GainNode>> {
367 GainNode::new(cx, self.global().as_window(), self, &GainOptions::empty())
368 }
369
370 fn CreatePanner(&self, cx: &mut JSContext) -> Fallible<DomRoot<PannerNode>> {
372 PannerNode::new(cx, self.global().as_window(), self, &PannerOptions::empty())
373 }
374
375 fn CreatePeriodicWave(
377 &self,
378 cx: &mut JSContext,
379 real: Vec<Finite<f32>>,
380 imag: Vec<Finite<f32>>,
381 constraints: &PeriodicWaveConstraints,
382 ) -> Fallible<DomRoot<PeriodicWave>> {
383 let mut options = PeriodicWaveOptions::empty();
385 let mut constraints_copy = PeriodicWaveConstraints::empty();
386 constraints_copy.disableNormalization = constraints.disableNormalization;
389 options.real = Some(real);
392 options.imag = Some(imag);
393 options.parent = constraints_copy;
394 PeriodicWave::Constructor(cx, self.global().as_window(), None, self, &options)
395 }
396
397 fn CreateAnalyser(&self, cx: &mut JSContext) -> Fallible<DomRoot<AnalyserNode>> {
399 AnalyserNode::new(
400 cx,
401 self.global().as_window(),
402 self,
403 &AnalyserOptions::empty(),
404 )
405 }
406
407 fn CreateBiquadFilter(&self, cx: &mut JSContext) -> Fallible<DomRoot<BiquadFilterNode>> {
409 BiquadFilterNode::new(
410 cx,
411 self.global().as_window(),
412 self,
413 &BiquadFilterOptions::empty(),
414 )
415 }
416
417 fn CreateStereoPanner(&self, cx: &mut JSContext) -> Fallible<DomRoot<StereoPannerNode>> {
419 StereoPannerNode::new(
420 cx,
421 self.global().as_window(),
422 self,
423 &StereoPannerOptions::empty(),
424 )
425 }
426
427 fn CreateConstantSource(&self, cx: &mut JSContext) -> Fallible<DomRoot<ConstantSourceNode>> {
429 ConstantSourceNode::new(
430 cx,
431 self.global().as_window(),
432 self,
433 &ConstantSourceOptions::empty(),
434 )
435 }
436
437 fn CreateChannelMerger(
439 &self,
440 cx: &mut JSContext,
441 count: u32,
442 ) -> Fallible<DomRoot<ChannelMergerNode>> {
443 let mut opts = ChannelMergerOptions::empty();
444 opts.numberOfInputs = count;
445 ChannelMergerNode::new(cx, self.global().as_window(), self, &opts)
446 }
447
448 fn CreateChannelSplitter(
450 &self,
451 cx: &mut JSContext,
452 count: u32,
453 ) -> Fallible<DomRoot<ChannelSplitterNode>> {
454 let mut opts = ChannelSplitterOptions::empty();
455 opts.numberOfOutputs = count;
456 ChannelSplitterNode::new(cx, self.global().as_window(), self, &opts)
457 }
458
459 fn CreateBuffer(
461 &self,
462 cx: &mut JSContext,
463 number_of_channels: u32,
464 length: u32,
465 sample_rate: Finite<f32>,
466 ) -> Fallible<DomRoot<AudioBuffer>> {
467 if number_of_channels == 0 ||
468 number_of_channels > MAX_CHANNEL_COUNT ||
469 length == 0 ||
470 *sample_rate <= 0.
471 {
472 return Err(Error::NotSupported(None));
473 }
474 Ok(AudioBuffer::new(
475 cx,
476 self.global().as_window(),
477 number_of_channels,
478 length,
479 *sample_rate,
480 None,
481 ))
482 }
483
484 fn CreateBufferSource(&self, cx: &mut JSContext) -> Fallible<DomRoot<AudioBufferSourceNode>> {
486 AudioBufferSourceNode::new(
487 cx,
488 self.global().as_window(),
489 self,
490 &AudioBufferSourceOptions::empty(),
491 )
492 }
493
494 fn DecodeAudioData(
496 &self,
497 cx: &mut CurrentRealm,
498 audio_data: CustomAutoRooterGuard<ArrayBuffer>,
499 decode_success_callback: Option<Option<Rc<DecodeSuccessCallback>>>,
500 decode_error_callback: Option<Option<Rc<DecodeErrorCallback>>>,
501 ) -> Rc<Promise> {
502 if !self.global().as_window().Document().is_fully_active() {
505 let promise = Promise::new_in_realm(cx);
506 promise.reject_error(
507 cx,
508 Error::InvalidState(Some("Audio context's document is not fully active.".into())),
509 );
510 return promise;
511 }
512
513 let promise = Promise::new_in_realm(cx);
515
516 let decode_success_callback = decode_success_callback.flatten();
518 let decode_error_callback = decode_error_callback.flatten();
519
520 if !Self::is_detached_array_buffer(&audio_data) {
525 if let Some(audio_data) = audio_data.to_vec() {
526 let uuid = Uuid::new_v4().simple().to_string();
528 let uuid_ = uuid.clone();
529 self.decode_resolvers.safe_borrow_mut(cx.no_gc()).insert(
530 uuid.clone(),
531 DecodeResolver {
532 promise: promise.clone(),
533 success_callback: decode_success_callback,
534 error_callback: decode_error_callback,
535 },
536 );
537 let decoded_audio = Arc::new(Mutex::new(Vec::new()));
538 let decoded_audio_ = decoded_audio.clone();
539 let decoded_audio__ = decoded_audio.clone();
540 let channels = Arc::new(Mutex::new(HashMap::new()));
545 let this = Trusted::new(self);
546 let this_ = this.clone();
547 let task_source = self
548 .global()
549 .task_manager()
550 .dom_manipulation_task_source()
551 .to_sendable();
552 let task_source_clone = task_source.clone();
553 let callbacks = AudioDecoderCallbacksBuilder::default()
554 .ready(move |channel_count| {
555 decoded_audio
556 .lock()
557 .unwrap()
558 .resize(channel_count as usize, Vec::new());
559 })
560 .progress(move |buffer, channel_pos_mask| {
561 let mut decoded_audio = decoded_audio_.lock().unwrap();
562 let mut channels = channels.lock().unwrap();
563 let channel = match channels.entry(channel_pos_mask) {
564 Entry::Occupied(entry) => *entry.get(),
565 Entry::Vacant(entry) => {
566 let x = (channel_pos_mask as f32).log2() as usize;
567 *entry.insert(x)
568 },
569 };
570 decoded_audio[channel].extend_from_slice((*buffer).as_ref());
571 })
572 .eos(move || {
573 task_source.queue(task!(audio_decode_eos: move |cx| {
574 let this = this.root();
575 let decoded_audio = decoded_audio__.lock().unwrap();
576 let length = if !decoded_audio.is_empty() {
577 decoded_audio[0].len()
578 } else {
579 0
580 };
581 let buffer = AudioBuffer::new(
582 cx,
583 this.global().as_window(),
584 decoded_audio.len() as u32 ,
585 length as u32,
586 this.sample_rate,
587 Some(decoded_audio.as_slice()),
588 );
589 let resolver = {
591 let mut resolvers =
592 this.decode_resolvers.safe_borrow_mut(cx.no_gc());
593 assert!(resolvers.contains_key(&uuid_));
594 resolvers.remove(&uuid_).unwrap()
595 };
596 if let Some(callback) = resolver.success_callback {
597 let _ = callback.Call__(cx, &buffer, ExceptionHandling::Report);
598 }
599 resolver.promise.resolve_native(cx, &buffer);
600 }));
601 })
602 .error(move |error| {
603 task_source_clone.queue(task!(audio_decode_eos: move |cx| {
604 let this = this_.root();
605 let resolver = {
607 let mut resolvers =
608 this.decode_resolvers.safe_borrow_mut(cx.no_gc());
609 assert!(resolvers.contains_key(&uuid));
610 resolvers.remove(&uuid).unwrap()
611 };
612 if let Some(callback) = resolver.error_callback {
613 let exception = DOMException::new(
614 cx,
615 &this.global(),
616 DOMErrorName::DataCloneError,
617 );
618 let _ =
619 callback.Call__(cx, &exception, ExceptionHandling::Report);
620 }
621 let error = cformat!("Audio decode error {:?}", error);
622 resolver.promise.reject_error(cx, Error::Type(error));
623 }));
624 })
625 .build();
626 self.audio_context_impl
627 .lock()
628 .unwrap()
629 .decode_audio_data(audio_data, callbacks);
630 } else {
631 debug_assert!(false, "ArrayBuffer::to_vec failed on non-detached buffer");
632 promise.reject_error(
633 cx,
634 Error::DataClone(Some(
635 "Failed to detach the ArrayBuffer while decoding audio data.".into(),
636 )),
637 );
638 return promise;
639 }
640 } else {
641 let exception = DOMException::new(cx, &self.global(), DOMErrorName::DataCloneError);
647 promise.reject_native(cx, &exception);
648
649 if let Some(callback) = decode_error_callback {
650 let uuid = Uuid::new_v4().simple().to_string();
654 let uuid_ = uuid.clone();
655 let this = Trusted::new(self);
656 let exception = Trusted::new(&*exception);
657 let task = task!(decode_audio_data_detached_buffer: move |cx| {
658 let this = this.root();
659 let exception = exception.root();
660 let resolver = {
661 let mut resolvers = this.decode_resolvers.safe_borrow_mut(cx.no_gc());
662 resolvers.remove(&uuid).unwrap()
663 };
664 if let Some(callback) = resolver.error_callback {
665 let _ = callback.Call__(
666 cx,
667 &exception,
668 ExceptionHandling::Report
669 );
670 }
671 });
672 self.decode_resolvers.safe_borrow_mut(cx.no_gc()).insert(
673 uuid_,
674 DecodeResolver {
675 promise: promise.clone(),
676 success_callback: None,
677 error_callback: Some(callback),
678 },
679 );
680 self.global()
681 .task_manager()
682 .media_element_task_source()
683 .queue(task);
684 }
685 }
686
687 promise
689 }
690
691 fn CreateIIRFilter(
693 &self,
694 cx: &mut JSContext,
695 feedforward: Vec<Finite<f64>>,
696 feedback: Vec<Finite<f64>>,
697 ) -> Fallible<DomRoot<IIRFilterNode>> {
698 let opts = IIRFilterOptions {
699 parent: AudioNodeOptions::empty(),
700 feedback,
701 feedforward,
702 };
703 IIRFilterNode::new(cx, self.global().as_window(), self, &opts)
704 }
705}
706
707impl Convert<AudioContextOptions> for BaseAudioContextOptions {
708 fn convert(self) -> AudioContextOptions {
709 match self {
710 BaseAudioContextOptions::AudioContext(options) => {
711 AudioContextOptions::RealTimeAudioContext(options)
712 },
713 BaseAudioContextOptions::OfflineAudioContext(options) => {
714 AudioContextOptions::OfflineAudioContext(options)
715 },
716 }
717 }
718}
719
720impl Convert<AudioContextState> for ProcessingState {
721 fn convert(self) -> AudioContextState {
722 match self {
723 ProcessingState::Suspended => AudioContextState::Suspended,
724 ProcessingState::Running => AudioContextState::Running,
725 ProcessingState::Closed => AudioContextState::Closed,
726 }
727 }
728}