toml_edit/parser/
document.rs

1use crate::Item;
2use crate::RawString;
3use crate::Value;
4use crate::key::Key;
5use crate::parser::key::on_key;
6use crate::parser::prelude::*;
7use crate::parser::value::value;
8use crate::repr::Decor;
9use crate::{ArrayOfTables, Document, Table};
10
11/// ```abnf
12/// ;; TOML
13///
14/// toml = expression *( newline expression )
15///
16/// expression = ( ( ws comment ) /
17///                ( ws keyval ws [ comment ] ) /
18///                ( ws table ws [ comment ] ) /
19///                  ws )
20/// ```
21pub(crate) fn document<'s>(
22    input: &mut Input<'_>,
23    source: toml_parser::Source<'s>,
24    errors: &mut dyn ErrorSink,
25) -> Document<&'s str> {
26    #[cfg(feature = "debug")]
27    let _scope = TraceScope::new("document::document");
28    let mut state = State::default();
29    while let Some(event) = input.next_token() {
30        match event.kind() {
31            EventKind::InlineTableOpen
32            | EventKind::InlineTableClose
33            | EventKind::ArrayOpen
34            | EventKind::ArrayClose
35            | EventKind::Scalar
36            | EventKind::ValueSep
37            | EventKind::Error
38            | EventKind::KeySep
39            | EventKind::KeyValSep
40            | EventKind::StdTableClose
41            | EventKind::ArrayTableClose => {
42                #[cfg(feature = "debug")]
43                trace(
44                    &format!("unexpected {event:?}"),
45                    anstyle::AnsiColor::Red.on_default(),
46                );
47                continue;
48            }
49            EventKind::StdTableOpen | EventKind::ArrayTableOpen => {
50                state.finish_table(errors);
51
52                let prefix = state.take_trailing();
53                let header = on_table(event, input, source, errors);
54                let suffix = ws_comment_newline(input)
55                    .map(|s| RawString::with_span(s.start()..s.end()))
56                    .unwrap_or_default();
57                let decor = Decor::new(prefix, suffix);
58
59                state.start_table(header, decor, errors);
60            }
61            EventKind::SimpleKey => {
62                let key_prefix = state.take_trailing();
63                let (path, key) = on_key(event, input, source, errors);
64                let Some(mut key) = key else {
65                    break;
66                };
67                let Some(next_event) = input.next_token() else {
68                    break;
69                };
70                let keyval_event;
71                let key_suffix;
72                if next_event.kind() == EventKind::Whitespace {
73                    key_suffix = Some(next_event);
74                    let Some(next_event) = input.next_token() else {
75                        break;
76                    };
77                    keyval_event = next_event;
78                } else {
79                    key_suffix = None;
80                    keyval_event = next_event;
81                }
82                if keyval_event.kind() != EventKind::KeyValSep {
83                    break;
84                }
85                let key_suffix = key_suffix
86                    .map(|e| RawString::with_span(e.span().start()..e.span().end()))
87                    .unwrap_or_default();
88                key.leaf_decor.set_prefix(key_prefix);
89                key.leaf_decor.set_suffix(key_suffix);
90
91                let value_prefix = if input
92                    .first()
93                    .map(|e| e.kind() == EventKind::Whitespace)
94                    .unwrap_or(false)
95                {
96                    input
97                        .next_token()
98                        .map(|e| RawString::with_span(e.span().start()..e.span().end()))
99                } else {
100                    None
101                }
102                .unwrap_or_default();
103                let mut value = value(input, source, errors);
104                let value_suffix = ws_comment_newline(input)
105                    .map(|s| RawString::with_span(s.start()..s.end()))
106                    .unwrap_or_default();
107                let decor = value.decor_mut();
108                decor.set_prefix(value_prefix);
109                decor.set_suffix(value_suffix);
110
111                state.capture_key_value(path, key, value, errors);
112            }
113            EventKind::Whitespace | EventKind::Comment | EventKind::Newline => {
114                state.capture_trailing(event);
115            }
116        }
117    }
118
119    state.finish_table(errors);
120
121    let trailing = state.take_trailing();
122    Document {
123        root: Item::Table(state.root),
124        trailing,
125        raw: source.input(),
126    }
127}
128
129/// ```abnf
130/// ;; Standard Table
131///
132/// std-table = std-table-open key *( table-key-sep key) std-table-close
133///
134/// ;; Array Table
135///
136/// array-table = array-table-open key *( table-key-sep key) array-table-close
137/// ```
138fn on_table(
139    open_event: &toml_parser::parser::Event,
140    input: &mut Input<'_>,
141    source: toml_parser::Source<'_>,
142    errors: &mut dyn ErrorSink,
143) -> TableHeader {
144    #[cfg(feature = "debug")]
145    let _scope = TraceScope::new("document::on_table");
146    let is_array = open_event.kind() == EventKind::ArrayTableOpen;
147    let mut current_path = None;
148    let mut current_key = None;
149    let mut current_span = open_event.span();
150    let mut current_prefix = None;
151    let mut current_suffix = None;
152
153    while let Some(event) = input.next_token() {
154        match event.kind() {
155            EventKind::InlineTableOpen
156            | EventKind::InlineTableClose
157            | EventKind::ArrayOpen
158            | EventKind::ArrayClose
159            | EventKind::Scalar
160            | EventKind::ValueSep
161            | EventKind::Error
162            | EventKind::KeySep
163            | EventKind::KeyValSep
164            | EventKind::StdTableOpen
165            | EventKind::ArrayTableOpen
166            | EventKind::Comment
167            | EventKind::Newline => {
168                #[cfg(feature = "debug")]
169                trace(
170                    &format!("unexpected {event:?}"),
171                    anstyle::AnsiColor::Red.on_default(),
172                );
173                continue;
174            }
175            EventKind::ArrayTableClose | EventKind::StdTableClose => {
176                current_span = current_span.append(event.span());
177                break;
178            }
179            EventKind::SimpleKey => {
180                current_prefix.get_or_insert_with(|| event.span().before());
181                let (path, key) = on_key(event, input, source, errors);
182                current_path = Some(path);
183                current_key = key;
184                current_suffix.get_or_insert_with(|| event.span().after());
185            }
186            EventKind::Whitespace => {
187                if current_key.is_some() {
188                    current_suffix = Some(event.span());
189                } else {
190                    current_prefix = Some(event.span());
191                }
192            }
193        }
194    }
195
196    let prefix = current_prefix
197        .take()
198        .expect("setting a key should set a prefix");
199    let suffix = current_suffix
200        .take()
201        .expect("setting a key should set a suffix");
202    if let Some(last_key) = current_key.as_mut() {
203        let prefix = RawString::with_span(prefix.start()..prefix.end());
204        let suffix = RawString::with_span(suffix.start()..suffix.end());
205        let leaf_decor = Decor::new(prefix, suffix);
206        *last_key.leaf_decor_mut() = leaf_decor;
207    }
208
209    TableHeader {
210        path: current_path.unwrap_or_default(),
211        key: current_key,
212        span: current_span,
213        is_array,
214    }
215}
216
217struct TableHeader {
218    path: Vec<Key>,
219    key: Option<Key>,
220    span: toml_parser::Span,
221    is_array: bool,
222}
223
224fn ws_comment_newline(input: &mut Input<'_>) -> Option<toml_parser::Span> {
225    let mut current_span = None;
226    while let Some(event) = input.next_token() {
227        match event.kind() {
228            EventKind::InlineTableOpen
229            | EventKind::InlineTableClose
230            | EventKind::ArrayOpen
231            | EventKind::ArrayClose
232            | EventKind::Scalar
233            | EventKind::ValueSep
234            | EventKind::Error
235            | EventKind::SimpleKey
236            | EventKind::KeySep
237            | EventKind::KeyValSep
238            | EventKind::StdTableOpen
239            | EventKind::ArrayTableOpen
240            | EventKind::StdTableClose
241            | EventKind::ArrayTableClose => {
242                #[cfg(feature = "debug")]
243                trace(
244                    &format!("unexpected {event:?}"),
245                    anstyle::AnsiColor::Red.on_default(),
246                );
247            }
248            EventKind::Whitespace | EventKind::Comment => {
249                let span = current_span.get_or_insert_with(|| event.span());
250                *span = span.append(event.span());
251            }
252            EventKind::Newline => {
253                break;
254            }
255        }
256    }
257
258    current_span
259}
260
261#[derive(Default)]
262struct State {
263    root: Table,
264    current_table: Table,
265    current_trailing: Option<toml_parser::Span>,
266    current_header: Option<TableHeader>,
267    current_position: isize,
268}
269
270impl State {
271    fn capture_trailing(&mut self, event: &toml_parser::parser::Event) {
272        let decor = self.current_trailing.get_or_insert(event.span());
273        *decor = decor.append(event.span());
274    }
275
276    fn capture_key_value(
277        &mut self,
278        path: Vec<Key>,
279        key: Key,
280        value: Value,
281        errors: &mut dyn ErrorSink,
282    ) {
283        #[cfg(feature = "debug")]
284        let _scope = TraceScope::new("document::capture_key_value");
285        #[cfg(feature = "debug")]
286        trace(
287            &format!(
288                "path={:?}",
289                path.iter().map(|k| k.get()).collect::<Vec<_>>()
290            ),
291            anstyle::AnsiColor::Blue.on_default(),
292        );
293        #[cfg(feature = "debug")]
294        trace(
295            &format!("key={key}",),
296            anstyle::AnsiColor::Blue.on_default(),
297        );
298        #[cfg(feature = "debug")]
299        trace(
300            &format!("value={value:?}",),
301            anstyle::AnsiColor::Blue.on_default(),
302        );
303
304        let dotted = !path.is_empty();
305        let Some(parent_table) = descend_path(&mut self.current_table, &path, dotted, errors)
306        else {
307            return;
308        };
309        // "Likewise, using dotted keys to redefine tables already defined in [table] form is not allowed"
310        let mixed_table_types = dotted && !parent_table.is_implicit();
311        if mixed_table_types {
312            #[cfg(feature = "debug")]
313            trace(
314                &format!("dotted={dotted}"),
315                anstyle::AnsiColor::Red.on_default(),
316            );
317            #[cfg(feature = "debug")]
318            trace(
319                &format!("parent_table.is_implicit={}", parent_table.is_implicit()),
320                anstyle::AnsiColor::Red.on_default(),
321            );
322            let key_span = get_key_span(&key).expect("all keys have spans");
323            errors.report_error(ParseError::new("duplicate key").with_unexpected(key_span));
324            return;
325        }
326        let key_span = get_key_span(&key).expect("all keys have spans");
327        match parent_table.items.entry(key) {
328            indexmap::map::Entry::Vacant(o) => {
329                o.insert(Item::Value(value));
330            }
331            indexmap::map::Entry::Occupied(existing) => {
332                // "Since tables cannot be defined more than once, redefining such tables using a [table] header is not allowed"
333                let old_span = existing.key().span().expect("all items have spans");
334                let old_span = toml_parser::Span::new_unchecked(old_span.start, old_span.end);
335                errors.report_error(
336                    ParseError::new("duplicate key")
337                        .with_unexpected(key_span)
338                        .with_context(old_span),
339                );
340            }
341        }
342    }
343
344    fn finish_table(&mut self, errors: &mut dyn ErrorSink) {
345        #[cfg(feature = "debug")]
346        let _scope = TraceScope::new("document::finish_table");
347        let mut prev_table = std::mem::take(&mut self.current_table);
348        if let Some(header) = self.current_header.take() {
349            let Some(key) = header.key else {
350                return;
351            };
352            prev_table.span = Some(header.span.start()..header.span.end());
353
354            let parent_key = &header.path;
355            let dotted = false;
356            let Some(parent_table) = descend_path(&mut self.root, parent_key, dotted, errors)
357            else {
358                return;
359            };
360            #[cfg(feature = "debug")]
361            trace(
362                &format!("key={key}",),
363                anstyle::AnsiColor::Blue.on_default(),
364            );
365            if header.is_array {
366                let key_span = get_key_span(&key).expect("all keys have spans");
367                let entry = parent_table
368                    .items
369                    .entry(key)
370                    .or_insert(Item::ArrayOfTables(ArrayOfTables::new()));
371                let Some(array) = entry.as_array_of_tables_mut() else {
372                    #[cfg(feature = "debug")]
373                    trace(
374                        "is_array_of_tables=false",
375                        anstyle::AnsiColor::Red.on_default(),
376                    );
377                    let old_span = entry.span().unwrap_or_default();
378                    let old_span = toml_parser::Span::new_unchecked(old_span.start, old_span.end);
379                    errors.report_error(
380                        ParseError::new("duplicate key")
381                            .with_unexpected(key_span)
382                            .with_context(old_span),
383                    );
384                    return;
385                };
386                array.push(prev_table);
387                let span = if let (Some(first), Some(last)) = (
388                    array.values.first().and_then(|t| t.span()),
389                    array.values.last().and_then(|t| t.span()),
390                ) {
391                    Some((first.start)..(last.end))
392                } else {
393                    None
394                };
395                array.span = span;
396            } else {
397                let existing = parent_table.items.insert(key, Item::Table(prev_table));
398                debug_assert!(existing.is_none());
399            }
400        } else {
401            prev_table.span = Some(Default::default());
402            self.root = prev_table;
403        }
404    }
405
406    fn start_table(&mut self, header: TableHeader, decor: Decor, errors: &mut dyn ErrorSink) {
407        if !header.is_array {
408            // 1. Look up the table on start to ensure the duplicate_key error points to the right line
409            // 2. Ensure any child tables from an implicit table are preserved
410            let root = &mut self.root;
411            if let (Some(parent_table), Some(key)) =
412                (descend_path(root, &header.path, false, errors), &header.key)
413            {
414                if let Some((old_key, old_value)) = parent_table.remove_entry(key.get()) {
415                    match old_value {
416                        Item::Table(t) if t.implicit && !t.is_dotted() => {
417                            self.current_table = t;
418                        }
419                        // Since tables cannot be defined more than once, redefining such tables using a [table] header is not allowed. Likewise, using dotted keys to redefine tables already defined in [table] form is not allowed.
420                        old_value => {
421                            #[cfg(feature = "debug")]
422                            if let Item::Table(t) = &old_value {
423                                trace(
424                                    &format!("t.dotted={}", t.is_dotted()),
425                                    anstyle::AnsiColor::Red.on_default(),
426                                );
427                                trace(
428                                    &format!("t.is_implicit={}", t.is_implicit()),
429                                    anstyle::AnsiColor::Red.on_default(),
430                                );
431                            } else {
432                                trace(
433                                    &format!("old_value.type_str={}", old_value.type_name()),
434                                    anstyle::AnsiColor::Red.on_default(),
435                                );
436                            }
437                            let old_span = old_key.span().expect("all items have spans");
438                            let old_span =
439                                toml_parser::Span::new_unchecked(old_span.start, old_span.end);
440                            let key_span = get_key_span(key).expect("all keys have spans");
441                            errors.report_error(
442                                ParseError::new("duplicate key")
443                                    .with_unexpected(key_span)
444                                    .with_context(old_span),
445                            );
446
447                            if let Item::Table(t) = old_value {
448                                self.current_table = t;
449                            }
450                        }
451                    }
452                }
453            }
454        }
455
456        self.current_position += 1;
457        self.current_table.decor = decor;
458        self.current_table.set_implicit(false);
459        self.current_table.set_dotted(false);
460        self.current_table.set_position(Some(self.current_position));
461        self.current_table.span = Some(header.span.start()..header.span.end());
462        self.current_header = Some(header);
463    }
464
465    fn take_trailing(&mut self) -> RawString {
466        self.current_trailing
467            .take()
468            .map(|s| RawString::with_span(s.start()..s.end()))
469            .unwrap_or_default()
470    }
471}
472
473fn descend_path<'t>(
474    mut table: &'t mut Table,
475    path: &[Key],
476    dotted: bool,
477    errors: &mut dyn ErrorSink,
478) -> Option<&'t mut Table> {
479    #[cfg(feature = "debug")]
480    let _scope = TraceScope::new("document::descend_path");
481    #[cfg(feature = "debug")]
482    trace(
483        &format!(
484            "path={:?}",
485            path.iter().map(|k| k.get()).collect::<Vec<_>>()
486        ),
487        anstyle::AnsiColor::Blue.on_default(),
488    );
489    for key in path.iter() {
490        #[cfg(feature = "debug")]
491        trace(
492            &format!("path[_]={key:?}"),
493            anstyle::AnsiColor::Blue.on_default(),
494        );
495        table = match table.entry_format(key) {
496            crate::Entry::Vacant(entry) => {
497                let mut new_table = Table::new();
498                new_table.span = key.span();
499                new_table.set_implicit(true);
500                new_table.set_dotted(dotted);
501
502                entry.insert(Item::Table(new_table)).as_table_mut().unwrap()
503            }
504            crate::Entry::Occupied(entry) => {
505                match entry.into_mut() {
506                    Item::ArrayOfTables(array) => {
507                        debug_assert!(!array.is_empty());
508
509                        let index = array.len() - 1;
510                        let last_child = array.get_mut(index).unwrap();
511
512                        last_child
513                    }
514                    Item::Table(sweet_child_of_mine) => {
515                        if dotted && sweet_child_of_mine.is_implicit() {
516                            // Since tables cannot be defined more than once, redefining such tables using a
517                            // [table] header is not allowed. Likewise, using dotted keys to redefine tables
518                            // already defined in [table] form is not allowed.
519                            sweet_child_of_mine.set_dotted(true);
520                        }
521                        // Since tables cannot be defined more than once, redefining such tables using a
522                        // [table] header is not allowed. Likewise, using dotted keys to redefine tables
523                        // already defined in [table] form is not allowed.
524                        let mixed_table_types = dotted && !sweet_child_of_mine.is_implicit();
525                        if mixed_table_types {
526                            #[cfg(feature = "debug")]
527                            trace(
528                                &format!("dotted={dotted}"),
529                                anstyle::AnsiColor::Red.on_default(),
530                            );
531                            #[cfg(feature = "debug")]
532                            trace(
533                                &format!(
534                                    "sweet_child_of_mine.is_implicit={}",
535                                    sweet_child_of_mine.is_implicit()
536                                ),
537                                anstyle::AnsiColor::Red.on_default(),
538                            );
539                            let key_span = get_key_span(key).expect("all keys have spans");
540                            errors.report_error(
541                                ParseError::new("duplicate key").with_unexpected(key_span),
542                            );
543                            return None;
544                        }
545                        sweet_child_of_mine
546                    }
547                    Item::Value(existing) => {
548                        let old_span = existing.span().expect("all items have spans");
549                        let old_span =
550                            toml_parser::Span::new_unchecked(old_span.start, old_span.end);
551                        let key_span = get_key_span(key).expect("all keys have spans");
552                        errors.report_error(
553                            ParseError::new(format!(
554                                "cannot extend value of type {} with a dotted key",
555                                existing.type_name()
556                            ))
557                            .with_unexpected(key_span)
558                            .with_context(old_span),
559                        );
560                        return None;
561                    }
562                    Item::None => unreachable!(),
563                }
564            }
565        };
566    }
567    Some(table)
568}
569
570fn get_key_span(key: &Key) -> Option<toml_parser::Span> {
571    key.as_repr()
572        .and_then(|r| r.span())
573        .map(|s| toml_parser::Span::new_unchecked(s.start, s.end))
574}
toml_edit/parser/document.rs

toml_edit/parser/
document.rs
