servo_webvtt/lib.rs
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::marker::PhantomData;
6use std::mem;
7
8use html5ever::buffer_queue::{BufferQueue, SetResult};
9use html5ever::tendril::StrTendril;
10use markup5ever::small_char_set;
11
12mod collectors;
13pub mod cue;
14
15use collectors::collect_webvtt_cue_timings_and_settings;
16
17use crate::cue::settings::WebVttCue;
18
19#[derive(Debug, PartialEq)]
20pub enum WebVttParserError {
21 InvalidHeader,
22}
23
24impl std::fmt::Display for WebVttParserError {
25 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
26 match self {
27 WebVttParserError::InvalidHeader => write!(formatter, "Invalid WebVTT header in file"),
28 }
29 }
30}
31
32#[derive(Clone, Copy, Default, PartialEq)]
33enum ParserState {
34 #[default]
35 FileTag,
36 WhitespaceAfterFileTag,
37 BeforeNewlineAfterFileTag,
38 BeforeHeader,
39 InBlockLoop,
40 AfterBlockLoop,
41 Region,
42 Finished,
43}
44
45pub trait WebVttParserSink<Context> {
46 fn consume_cue(&self, cx: &mut Context, cue: WebVttCue);
47}
48
49#[derive(Default)]
50pub struct IncrementalWebVTTParser<Context, Sink: WebVttParserSink<Context>> {
51 phantom: PhantomData<Context>,
52 pub sink: Sink,
53 buffer: BufferQueue,
54
55 // Checkpoint values
56 seen_cue: bool,
57 seen_eof: bool,
58 seen_arrow: bool,
59
60 // State values
61 in_header: bool,
62 line_count: u32,
63 state: ParserState,
64
65 // Storage values
66 current_line_in_block: StrTendril,
67 current_buffer_in_block: String,
68
69 current_cue_in_block: Option<WebVttCue>,
70}
71
72pub type ParserUpdate = Result<(), WebVttParserError>;
73
74impl<Context, Sink> IncrementalWebVTTParser<Context, Sink>
75where
76 Sink: WebVttParserSink<Context>,
77{
78 /// <https://w3c.github.io/webvtt/#webvtt-parser-algorithm>
79 pub fn new(sink: Sink) -> Self {
80 Self {
81 sink,
82 phantom: Default::default(),
83 buffer: Default::default(),
84 seen_cue: Default::default(),
85 seen_eof: Default::default(),
86 seen_arrow: Default::default(),
87 in_header: Default::default(),
88 line_count: Default::default(),
89 state: Default::default(),
90 current_line_in_block: Default::default(),
91 current_buffer_in_block: Default::default(),
92 current_cue_in_block: Default::default(),
93 }
94 }
95
96 pub fn end(&mut self, cx: &mut Context) -> ParserUpdate {
97 self.seen_eof = true;
98 self.step(cx)
99 }
100
101 pub fn parse_sync(&mut self, cx: &mut Context, input: &str) -> ParserUpdate {
102 self.seen_eof = true;
103 self.parse(cx, input)
104 }
105
106 /// <https://w3c.github.io/webvtt/#webvtt-parser-algorithm>
107 pub fn parse(&mut self, cx: &mut Context, input: &str) -> ParserUpdate {
108 // Step 1. Let input be the string being parsed, after conversion to Unicode,
109 // and with the following transformations applied:
110 // > Replace all U+0000 NULL characters by U+FFFD REPLACEMENT CHARACTERs.
111 // > Replace each U+000D CARRIAGE RETURN U+000A LINE FEED (CRLF) character pair
112 // > by a single U+000A LINE FEED (LF) character.
113 // > Replace all remaining U+000D CARRIAGE RETURN characters by U+000A LINE FEED (LF) characters.
114 // TODO
115 // Step 2. Let position be a pointer into input, initially pointing at the start of the string.
116 // In an incremental WebVTT parser, when this algorithm (or further algorithms that it uses)
117 // moves the position pointer, the user agent must wait until appropriate further characters
118 // from the byte stream have been added to input before moving the pointer,
119 // so that the algorithm never reads past the end of the input string.
120 // Once the byte stream has ended, and all characters have been added to input,
121 // then the position pointer may, when so instructed by the algorithms,
122 // be moved past the end of input.
123 self.buffer.push_back(StrTendril::from(input));
124 self.step(cx)
125 }
126
127 fn step(&mut self, cx: &mut Context) -> ParserUpdate {
128 loop {
129 let current_state = self.state;
130 match current_state {
131 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
132 ParserState::FileTag => {
133 // https://w3c.github.io/webvtt/#webvtt-file-body
134 // > An optional U+FEFF BYTE ORDER MARK (BOM) character.
135 if self.buffer.peek().is_some_and(|c| c == '\u{FEFF}') {
136 let _ = self.buffer.next();
137 }
138 let Some(input) = self.buffer.eat("WEBVTT", u8::eq) else {
139 // Step 4. If input is less than six characters long, then abort these steps.
140 // The file does not start with the correct WebVTT file signature
141 // and was therefore not successfully processed.
142 if self.seen_eof {
143 return Err(WebVttParserError::InvalidHeader);
144 }
145 return Ok(());
146 };
147 // Step 5. If input is exactly six characters long but does not exactly equal "WEBVTT",
148 // then abort these steps. The file does not start with the correct WebVTT
149 // file signature and was therefore not successfully processed.
150 // Step 6. If input is more than six characters long but the first six characters
151 // do not exactly equal "WEBVTT", or the seventh character is not a U+0020 SPACE character,
152 // a U+0009 CHARACTER TABULATION (tab) character, or a U+000A LINE FEED (LF) character,
153 // then abort these steps. The file does not start with the correct WebVTT file signature
154 // and was therefore not successfully processed.
155 //
156 // We check the first part of this step here
157 if !input {
158 return Err(WebVttParserError::InvalidHeader);
159 }
160 self.state = ParserState::WhitespaceAfterFileTag;
161 continue;
162 },
163 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
164 ParserState::WhitespaceAfterFileTag => {
165 let Some(seventh) = self.buffer.peek() else {
166 // Input is exactly six characters and is a valid header
167 if self.seen_eof {
168 self.state = ParserState::Finished;
169 continue;
170 }
171 return Ok(());
172 };
173 // Step 6. If input is more than six characters long but the first six characters
174 // do not exactly equal "WEBVTT", or the seventh character is not a U+0020 SPACE character,
175 // a U+0009 CHARACTER TABULATION (tab) character, or a U+000A LINE FEED (LF) character,
176 // then abort these steps. The file does not start with the correct WebVTT file signature
177 // and was therefore not successfully processed.
178 //
179 // We check the second part of this step here
180 if !matches!(seventh, '\u{0020}' | '\u{0009}' | '\u{000A}') {
181 return Err(WebVttParserError::InvalidHeader);
182 }
183 self.state = ParserState::BeforeNewlineAfterFileTag;
184 continue;
185 },
186 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
187 ParserState::BeforeNewlineAfterFileTag => {
188 // Step 7. collect a sequence of code points that are not U+000A LINE FEED (LF) characters.
189 let Some(current_char) = self.buffer.next() else {
190 // Step 8. If position is past the end of input, then abort these steps.
191 // The file was successfully processed, but it contains no useful data and so
192 // no WebVTT cues were added to output.
193 if self.seen_eof {
194 self.state = ParserState::Finished;
195 continue;
196 }
197 return Ok(());
198 };
199 // Step 9. The character indicated by position is a U+000A LINE FEED (LF) character.
200 // Advance position to the next character in input.
201 if current_char == '\u{000A}' {
202 self.state = ParserState::BeforeHeader;
203 }
204 },
205 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
206 ParserState::BeforeHeader => {
207 let Some(current_char) = self.buffer.peek() else {
208 // Step 10. If position is past the end of input, then abort these steps.
209 // The file was successfully processed, but it contains no useful data and
210 // so no WebVTT cues were added to output.
211 if self.seen_eof {
212 self.state = ParserState::Finished;
213 continue;
214 }
215 return Ok(());
216 };
217 // Step 11. Header: If the character indicated by position is not
218 // a U+000A LINE FEED (LF) character,
219 // then collect a WebVTT block with the in header flag set.
220 // Otherwise, advance position to the next character in input.
221 if current_char != '\u{000A}' {
222 self.in_header = true;
223 self.start_collecting_webvtt_block();
224 } else {
225 self.buffer.next();
226 self.state = ParserState::Region;
227 }
228 },
229 // https://w3c.github.io/webvtt/#collect-a-webvtt-block
230 ParserState::InBlockLoop => {
231 let Some(current_char) =
232 self.buffer.pop_except_from(small_char_set!('\u{000A}'))
233 else {
234 // Step 11.3. If position is past the end of input, let seen EOF be true.
235 // Otherwise, the character indicated by position is a U+000A LINE FEED (LF) character;
236 // advance position to the next character in input.
237 //
238 // We check the first part of this step here
239 // Step 11.7. If seen EOF is true, break out of loop.
240 if self.seen_eof {
241 // It depends when we see the EOF whether there is still content in the buffer or not.
242 // In the case that the EOF is at the end of a line (e.g. no `\n` in between), we
243 // should copy the current line to the buffer. The buffer is then populated with the
244 // line as usual, so that the last line of a file can still be the cue text.
245 if !self.current_line_in_block.is_empty() {
246 if !self.current_buffer_in_block.is_empty() {
247 self.current_buffer_in_block.push('\n');
248 }
249 self.current_buffer_in_block
250 .push_str(&self.current_line_in_block);
251 }
252 self.state = ParserState::AfterBlockLoop;
253 continue;
254 }
255 return Ok(());
256 };
257 match current_char {
258 // Step 11.3. If position is past the end of input, let seen EOF be true.
259 // Otherwise, the character indicated by position is a U+000A LINE FEED (LF) character;
260 // advance position to the next character in input.
261 //
262 // We check the second part of this step here
263 SetResult::FromSet('\u{000A}') => {
264 // Step 11.2. Increment line count by 1.
265 self.line_count += 1;
266 if self.in_header {
267 self.state = ParserState::AfterBlockLoop;
268 } else {
269 // Step 11.4. If line contains the three-character substring "-->"
270 // (U+002D HYPHEN-MINUS, U+002D HYPHEN-MINUS, U+003E GREATER-THAN SIGN),
271 // then run these substeps:
272 if self.current_line_in_block.contains("-->") {
273 // Step 11.4.1. If in header is not set and at least
274 // one of the following conditions are true:
275 //
276 // We already checked for the header set after step 11.2.
277 if
278 // line count is 1
279 self.line_count == 1
280 // line count is 2 and seen arrow is false
281 || (self.line_count == 2 && !self.seen_arrow)
282 {
283 // Step 11.4.1.1. Let seen arrow be true.
284 self.seen_arrow = true;
285 // Step 11.4.1.2. Let previous position be position.
286 // TODO
287 // Step 11.4.1.3. Cue creation: Let cue be a new WebVTT cue and initialize it as follows:
288 // Step 11.4.1.3.1. Let cue’s text track cue identifier be buffer.
289 let identifier = self.current_buffer_in_block.clone();
290 // Step 11.4.1.4. Collect WebVTT cue timings and settings from line using regions for cue.
291 // If that fails, let cue be null.
292 // Otherwise, let buffer be the empty string and let seen cue be true.
293 let cue = collect_webvtt_cue_timings_and_settings(
294 identifier,
295 &self.current_line_in_block,
296 );
297 let has_cue = cue.is_some();
298 self.current_cue_in_block = cue;
299 if has_cue {
300 self.current_buffer_in_block.clear();
301 self.current_line_in_block.clear();
302 self.seen_cue = true;
303 }
304 } else {
305 // Otherwise, let position be previous position and break out of loop.
306 self.state = ParserState::AfterBlockLoop;
307 }
308 continue;
309 } else if self.current_line_in_block.is_empty() {
310 // Step 11.5. Otherwise, if line is the empty string, break out of loop.
311 self.state = ParserState::AfterBlockLoop;
312 } else {
313 // Step 11.6. Otherwise, run these substeps:
314 // Step 11.6.1. If in header is not set and line count is 2, run these substeps:
315 // TODO
316 // Step 11.6.2. If buffer is not the empty string,
317 // append a U+000A LINE FEED (LF) character to buffer.
318 if !self.current_buffer_in_block.is_empty() {
319 self.current_buffer_in_block.push('\u{000A}');
320 }
321 // Step 11.6.3. Append line to buffer.
322 self.current_buffer_in_block
323 .push_str(&self.current_line_in_block);
324 // Step 11.6.4. Let previous position be position.
325 self.current_line_in_block.clear();
326 }
327 }
328 continue;
329 },
330 // Step 11.1. collect a sequence of code points that are not U+000A LINE FEED (LF) characters.
331 // Let line be those characters, if any.
332 SetResult::NotFromSet(current_tendril) => {
333 if !self.in_header {
334 self.current_line_in_block.push_tendril(¤t_tendril);
335 }
336 },
337 _ => {
338 unreachable!();
339 },
340 }
341 },
342 ParserState::AfterBlockLoop => {
343 // https://w3c.github.io/webvtt/#collect-a-webvtt-block
344 // Step 12. If cue is not null, let the cue text of cue be buffer, and return cue.
345 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
346 // Step 14.2. If block is a WebVTT cue, add block to the text track list of cues output.
347 if let Some(mut cue) = self.current_cue_in_block.take() {
348 cue.text = self.current_buffer_in_block.clone();
349 self.sink.consume_cue(cx, cue);
350 }
351 // Step 14.3. Otherwise, if block is a CSS style sheet, add block to stylesheets.
352 // TODO
353 // Step 14.4. Otherwise, if block is a WebVTT region object, add block to regions.
354 // TODO
355 // Step 14.5. collect a sequence of code points that are U+000A LINE FEED (LF) characters.
356 let Some(current_char) = self.buffer.peek() else {
357 if self.seen_eof {
358 self.state = ParserState::Finished;
359 continue;
360 }
361 return Ok(());
362 };
363 if current_char == '\u{000A}' {
364 // Since we don't change the state here, it means that if the next character is
365 // also a newline, we re-enter this block and consume it again. Therefore, we
366 // only consume one-by-one.
367 let _ = self.buffer.next();
368 } else {
369 // If we were in the header block, then we should proceed with the next step
370 // which is collecting a region in step 12. Otherwise, we are in the general loop of
371 // step 14.
372
373 if mem::take(&mut self.in_header) {
374 self.state = ParserState::Region;
375 } else {
376 self.start_collecting_webvtt_block();
377 }
378 }
379 },
380 // https://w3c.github.io/webvtt/#webvtt-parser-algorithm
381 ParserState::Region => {
382 // Step 12. collect a sequence of code points that are U+000A LINE FEED (LF) characters.
383 // TODO
384 self.start_collecting_webvtt_block();
385 },
386 ParserState::Finished => {
387 // Step 15. End: The file has ended. Abort these steps. The WebVTT parser has finished.
388 // The file was successfully processed.
389 return Ok(());
390 },
391 }
392 }
393 }
394
395 /// <https://w3c.github.io/webvtt/#collect-a-webvtt-block>
396 fn start_collecting_webvtt_block(&mut self) {
397 // Step 2. Let line count be zero.
398 self.line_count = 0;
399 // Step 4. Let line be the empty string.
400 self.current_line_in_block.clear();
401 // Step 5. Let buffer be the empty string.
402 self.current_buffer_in_block.clear();
403 // Step 7. Let seen arrow be false.
404 self.seen_arrow = false;
405 // Step 8. Let cue be null.
406 self.current_cue_in_block = None;
407 self.state = ParserState::InBlockLoop;
408 }
409}
410
411#[cfg(any(test, feature = "test-util"))]
412pub mod shared_test_setup;
413
414#[cfg(test)]
415mod tests {
416 use crate::WebVttParserError;
417 use crate::shared_test_setup::{compute_result_in_seconds, parser_with_dummy_sink};
418
419 #[test]
420 fn test_header_in_two_chunks() {
421 let mut parser = parser_with_dummy_sink();
422 assert_eq!(parser.parse(&mut (), "WEB"), Ok(()));
423 assert_eq!(parser.parse(&mut (), "VTT"), Ok(()));
424 assert_eq!(parser.end(&mut ()), Ok(()));
425 }
426
427 #[test]
428 fn test_invalid_header_in_two_chunks() {
429 let mut parser = parser_with_dummy_sink();
430 assert_eq!(parser.parse(&mut (), "WEB"), Ok(()));
431 assert_eq!(
432 parser.parse(&mut (), "NOT"),
433 Err(WebVttParserError::InvalidHeader)
434 );
435 }
436
437 #[test]
438 fn test_valid_space_character_after_header() {
439 let mut parser = parser_with_dummy_sink();
440 assert_eq!(parser.parse_sync(&mut (), "WEBVTT "), Ok(()));
441 }
442
443 #[test]
444 fn test_no_space_character_after_header_multiple_chunks() {
445 let mut parser = parser_with_dummy_sink();
446 assert_eq!(parser.parse(&mut (), "WEB"), Ok(()));
447 assert_eq!(parser.parse(&mut (), "VTT"), Ok(()));
448 assert_eq!(
449 parser.parse(&mut (), "2"),
450 Err(WebVttParserError::InvalidHeader)
451 );
452 }
453
454 mod cue_settings {
455 use crate::tests::compute_result_in_seconds;
456 use crate::{WebVttCue, collect_webvtt_cue_timings_and_settings};
457
458 #[test]
459 fn test_parses_cue_correctly() {
460 assert_eq!(
461 collect_webvtt_cue_timings_and_settings(
462 Default::default(),
463 "01:10:03.000 --> 02:20:23.000"
464 ),
465 Some(WebVttCue {
466 start_time: compute_result_in_seconds(1., 10., 3., 0.),
467 end_time: compute_result_in_seconds(2., 20., 23., 0.),
468 ..Default::default()
469 })
470 );
471 }
472
473 #[test]
474 fn test_does_not_require_whitespace_around_arrow() {
475 assert_eq!(
476 collect_webvtt_cue_timings_and_settings(
477 Default::default(),
478 "01:10:03.000-->02:20:23.000"
479 ),
480 Some(WebVttCue {
481 start_time: compute_result_in_seconds(1., 10., 3., 0.),
482 end_time: compute_result_in_seconds(2., 20., 23., 0.),
483 ..Default::default()
484 })
485 );
486 }
487
488 #[test]
489 fn test_can_handle_tabs_around_arrow() {
490 assert_eq!(
491 collect_webvtt_cue_timings_and_settings(
492 Default::default(),
493 "01:10:03.000\t-->\t02:20:23.000"
494 ),
495 Some(WebVttCue {
496 start_time: compute_result_in_seconds(1., 10., 3., 0.),
497 end_time: compute_result_in_seconds(2., 20., 23., 0.),
498 ..Default::default()
499 })
500 );
501 }
502
503 #[test]
504 fn test_arrow_too_short_is_invalid() {
505 assert_eq!(
506 collect_webvtt_cue_timings_and_settings(
507 Default::default(),
508 "01:10:03.000 -> t02:20:23.000"
509 ),
510 None
511 );
512 }
513
514 #[test]
515 fn test_arrow_too_long_is_invalid() {
516 assert_eq!(
517 collect_webvtt_cue_timings_and_settings(
518 Default::default(),
519 "01:10:03.000 ---> t02:20:23.000"
520 ),
521 None
522 );
523 }
524
525 #[test]
526 fn test_skips_whitespace_at_start() {
527 assert_eq!(
528 collect_webvtt_cue_timings_and_settings(
529 Default::default(),
530 " 01:10:03.000 --> 02:20:23.000"
531 ),
532 Some(WebVttCue {
533 start_time: compute_result_in_seconds(1., 10., 3., 0.),
534 end_time: compute_result_in_seconds(2., 20., 23., 0.),
535 ..Default::default()
536 })
537 );
538 }
539 }
540}