hyper/proto/h1/
decode.rs

1use std::error::Error as StdError;
2use std::fmt;
3use std::io;
4use std::task::{Context, Poll};
5
6use bytes::{BufMut, Bytes, BytesMut};
7use futures_core::ready;
8use http::{HeaderMap, HeaderName, HeaderValue};
9use http_body::Frame;
10
11use super::io::MemRead;
12use super::role::DEFAULT_MAX_HEADERS;
13use super::DecodedLength;
14
15use self::Kind::{Chunked, Eof, Length};
16
17/// Maximum amount of bytes allowed in chunked extensions.
18///
19/// This limit is currentlty applied for the entire body, not per chunk.
20const CHUNKED_EXTENSIONS_LIMIT: u64 = 1024 * 16;
21
22/// Maximum number of bytes allowed for all trailer fields.
23///
24/// TODO: remove this when we land `h1_max_header_size` support.
25const TRAILER_LIMIT: usize = 1024 * 16;
26
27/// Decoders to handle different Transfer-Encodings.
28///
29/// If a message body does not include a Transfer-Encoding, it *should*
30/// include a Content-Length header.
31#[derive(Clone, PartialEq)]
32pub(crate) struct Decoder {
33    kind: Kind,
34}
35
36#[derive(Debug, Clone, PartialEq)]
37enum Kind {
38    /// A Reader used when a Content-Length header is passed with a positive integer.
39    Length(u64),
40    /// A Reader used when Transfer-Encoding is `chunked`.
41    Chunked {
42        state: ChunkedState,
43        chunk_len: u64,
44        extensions_cnt: u64,
45        trailers_buf: Option<BytesMut>,
46        trailers_cnt: usize,
47        h1_max_headers: Option<usize>,
48        h1_max_header_size: Option<usize>,
49    },
50    /// A Reader used for responses that don't indicate a length or chunked.
51    ///
52    /// The bool tracks when EOF is seen on the transport.
53    ///
54    /// Note: This should only used for `Response`s. It is illegal for a
55    /// `Request` to be made with both `Content-Length` and
56    /// `Transfer-Encoding: chunked` missing, as explained from the spec:
57    ///
58    /// > If a Transfer-Encoding header field is present in a response and
59    /// > the chunked transfer coding is not the final encoding, the
60    /// > message body length is determined by reading the connection until
61    /// > it is closed by the server.  If a Transfer-Encoding header field
62    /// > is present in a request and the chunked transfer coding is not
63    /// > the final encoding, the message body length cannot be determined
64    /// > reliably; the server MUST respond with the 400 (Bad Request)
65    /// > status code and then close the connection.
66    Eof(bool),
67}
68
69#[derive(Debug, PartialEq, Clone, Copy)]
70enum ChunkedState {
71    Start,
72    Size,
73    SizeLws,
74    Extension,
75    SizeLf,
76    Body,
77    BodyCr,
78    BodyLf,
79    Trailer,
80    TrailerLf,
81    EndCr,
82    EndLf,
83    End,
84}
85
86impl Decoder {
87    // constructors
88
89    pub(crate) fn length(x: u64) -> Decoder {
90        Decoder {
91            kind: Kind::Length(x),
92        }
93    }
94
95    pub(crate) fn chunked(
96        h1_max_headers: Option<usize>,
97        h1_max_header_size: Option<usize>,
98    ) -> Decoder {
99        Decoder {
100            kind: Kind::Chunked {
101                state: ChunkedState::new(),
102                chunk_len: 0,
103                extensions_cnt: 0,
104                trailers_buf: None,
105                trailers_cnt: 0,
106                h1_max_headers,
107                h1_max_header_size,
108            },
109        }
110    }
111
112    pub(crate) fn eof() -> Decoder {
113        Decoder {
114            kind: Kind::Eof(false),
115        }
116    }
117
118    pub(super) fn new(
119        len: DecodedLength,
120        h1_max_headers: Option<usize>,
121        h1_max_header_size: Option<usize>,
122    ) -> Self {
123        match len {
124            DecodedLength::CHUNKED => Decoder::chunked(h1_max_headers, h1_max_header_size),
125            DecodedLength::CLOSE_DELIMITED => Decoder::eof(),
126            length => Decoder::length(length.danger_len()),
127        }
128    }
129
130    // methods
131
132    pub(crate) fn is_eof(&self) -> bool {
133        matches!(
134            self.kind,
135            Length(0)
136                | Chunked {
137                    state: ChunkedState::End,
138                    ..
139                }
140                | Eof(true)
141        )
142    }
143
144    pub(crate) fn decode<R: MemRead>(
145        &mut self,
146        cx: &mut Context<'_>,
147        body: &mut R,
148    ) -> Poll<Result<Frame<Bytes>, io::Error>> {
149        trace!("decode; state={:?}", self.kind);
150        match &mut self.kind {
151            Length(remaining) => {
152                if *remaining == 0 {
153                    Poll::Ready(Ok(Frame::data(Bytes::new())))
154                } else {
155                    let to_read = usize::try_from(*remaining).unwrap_or(usize::MAX);
156                    let buf = ready!(body.read_mem(cx, to_read))?;
157                    let num = buf.as_ref().len() as u64;
158                    if num > *remaining {
159                        *remaining = 0;
160                    } else if num == 0 {
161                        return Poll::Ready(Err(io::Error::new(
162                            io::ErrorKind::UnexpectedEof,
163                            IncompleteBody,
164                        )));
165                    } else {
166                        *remaining -= num;
167                    }
168                    Poll::Ready(Ok(Frame::data(buf)))
169                }
170            }
171            Chunked {
172                state,
173                chunk_len,
174                extensions_cnt,
175                trailers_buf,
176                trailers_cnt,
177                h1_max_headers,
178                h1_max_header_size,
179            } => {
180                let h1_max_headers = h1_max_headers.unwrap_or(DEFAULT_MAX_HEADERS);
181                let h1_max_header_size = h1_max_header_size.unwrap_or(TRAILER_LIMIT);
182                loop {
183                    let mut buf = None;
184                    // advances the chunked state
185                    *state = ready!(state.step(
186                        cx,
187                        body,
188                        StepArgs {
189                            chunk_size: chunk_len,
190                            extensions_cnt,
191                            chunk_buf: &mut buf,
192                            trailers_buf,
193                            trailers_cnt,
194                            max_headers_cnt: h1_max_headers,
195                            max_headers_bytes: h1_max_header_size,
196                        }
197                    ))?;
198                    if *state == ChunkedState::End {
199                        trace!("end of chunked");
200
201                        if trailers_buf.is_some() {
202                            trace!("found possible trailers");
203
204                            match decode_trailers(
205                                &mut trailers_buf.take().expect("Trailer is None"),
206                                *trailers_cnt,
207                            ) {
208                                Ok(headers) => {
209                                    return Poll::Ready(Ok(Frame::trailers(headers)));
210                                }
211                                Err(e) => {
212                                    return Poll::Ready(Err(e));
213                                }
214                            }
215                        }
216
217                        return Poll::Ready(Ok(Frame::data(Bytes::new())));
218                    }
219                    if let Some(buf) = buf {
220                        return Poll::Ready(Ok(Frame::data(buf)));
221                    }
222                }
223            }
224            Eof(is_eof) => {
225                if *is_eof {
226                    Poll::Ready(Ok(Frame::data(Bytes::new())))
227                } else {
228                    // 8192 chosen because its about 2 packets, there probably
229                    // won't be that much available, so don't have MemReaders
230                    // allocate buffers to big
231                    body.read_mem(cx, 8192).map_ok(|slice| {
232                        *is_eof = slice.is_empty();
233                        Frame::data(slice)
234                    })
235                }
236            }
237        }
238    }
239
240    #[cfg(test)]
241    async fn decode_fut<R: MemRead>(&mut self, body: &mut R) -> Result<Frame<Bytes>, io::Error> {
242        futures_util::future::poll_fn(move |cx| self.decode(cx, body)).await
243    }
244}
245
246impl fmt::Debug for Decoder {
247    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
248        fmt::Debug::fmt(&self.kind, f)
249    }
250}
251
252macro_rules! byte (
253    ($rdr:ident, $cx:expr) => ({
254        let buf = ready!($rdr.read_mem($cx, 1))?;
255        if !buf.is_empty() {
256            buf[0]
257        } else {
258            return Poll::Ready(Err(io::Error::new(io::ErrorKind::UnexpectedEof,
259                                      "unexpected EOF during chunk size line")));
260        }
261    })
262);
263
264macro_rules! or_overflow {
265    ($e:expr) => (
266        match $e {
267            Some(val) => val,
268            None => return Poll::Ready(Err(io::Error::new(
269                io::ErrorKind::InvalidData,
270                "invalid chunk size: overflow",
271            ))),
272        }
273    )
274}
275
276macro_rules! put_u8 {
277    ($trailers_buf:expr, $byte:expr, $limit:expr) => {
278        $trailers_buf.put_u8($byte);
279
280        if $trailers_buf.len() >= $limit {
281            return Poll::Ready(Err(io::Error::new(
282                io::ErrorKind::InvalidData,
283                "chunk trailers bytes over limit",
284            )));
285        }
286    };
287}
288
289struct StepArgs<'args> {
290    chunk_size: &'args mut u64,
291    chunk_buf: &'args mut Option<Bytes>,
292    extensions_cnt: &'args mut u64,
293    trailers_buf: &'args mut Option<BytesMut>,
294    trailers_cnt: &'args mut usize,
295    max_headers_cnt: usize,
296    max_headers_bytes: usize,
297}
298
299impl ChunkedState {
300    fn new() -> ChunkedState {
301        ChunkedState::Start
302    }
303    fn step<R: MemRead>(
304        &self,
305        cx: &mut Context<'_>,
306        body: &mut R,
307        StepArgs {
308            chunk_size,
309            chunk_buf,
310            extensions_cnt,
311            trailers_buf,
312            trailers_cnt,
313            max_headers_cnt,
314            max_headers_bytes,
315        }: StepArgs<'_>,
316    ) -> Poll<Result<ChunkedState, io::Error>> {
317        use self::ChunkedState::{
318            Body, BodyCr, BodyLf, End, EndCr, EndLf, Extension, Size, SizeLf, SizeLws, Start,
319            Trailer, TrailerLf,
320        };
321        match *self {
322            Start => ChunkedState::read_start(cx, body, chunk_size),
323            Size => ChunkedState::read_size(cx, body, chunk_size),
324            SizeLws => ChunkedState::read_size_lws(cx, body),
325            Extension => ChunkedState::read_extension(cx, body, extensions_cnt),
326            SizeLf => ChunkedState::read_size_lf(cx, body, *chunk_size),
327            Body => ChunkedState::read_body(cx, body, chunk_size, chunk_buf),
328            BodyCr => ChunkedState::read_body_cr(cx, body),
329            BodyLf => ChunkedState::read_body_lf(cx, body),
330            Trailer => ChunkedState::read_trailer(cx, body, trailers_buf, max_headers_bytes),
331            TrailerLf => ChunkedState::read_trailer_lf(
332                cx,
333                body,
334                trailers_buf,
335                trailers_cnt,
336                max_headers_cnt,
337                max_headers_bytes,
338            ),
339            EndCr => ChunkedState::read_end_cr(cx, body, trailers_buf, max_headers_bytes),
340            EndLf => ChunkedState::read_end_lf(cx, body, trailers_buf, max_headers_bytes),
341            End => Poll::Ready(Ok(ChunkedState::End)),
342        }
343    }
344
345    fn read_start<R: MemRead>(
346        cx: &mut Context<'_>,
347        rdr: &mut R,
348        size: &mut u64,
349    ) -> Poll<Result<ChunkedState, io::Error>> {
350        trace!("Read chunk start");
351
352        let radix = 16;
353        match byte!(rdr, cx) {
354            b @ b'0'..=b'9' => {
355                *size = or_overflow!(size.checked_mul(radix));
356                *size = or_overflow!(size.checked_add(u64::from(b - b'0')));
357            }
358            b @ b'a'..=b'f' => {
359                *size = or_overflow!(size.checked_mul(radix));
360                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'a')));
361            }
362            b @ b'A'..=b'F' => {
363                *size = or_overflow!(size.checked_mul(radix));
364                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'A')));
365            }
366            _ => {
367                return Poll::Ready(Err(io::Error::new(
368                    io::ErrorKind::InvalidInput,
369                    "Invalid chunk size line: missing size digit",
370                )));
371            }
372        }
373
374        Poll::Ready(Ok(ChunkedState::Size))
375    }
376
377    fn read_size<R: MemRead>(
378        cx: &mut Context<'_>,
379        rdr: &mut R,
380        size: &mut u64,
381    ) -> Poll<Result<ChunkedState, io::Error>> {
382        trace!("Read chunk hex size");
383
384        let radix = 16;
385        match byte!(rdr, cx) {
386            b @ b'0'..=b'9' => {
387                *size = or_overflow!(size.checked_mul(radix));
388                *size = or_overflow!(size.checked_add(u64::from(b - b'0')));
389            }
390            b @ b'a'..=b'f' => {
391                *size = or_overflow!(size.checked_mul(radix));
392                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'a')));
393            }
394            b @ b'A'..=b'F' => {
395                *size = or_overflow!(size.checked_mul(radix));
396                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'A')));
397            }
398            b'\t' | b' ' => return Poll::Ready(Ok(ChunkedState::SizeLws)),
399            b';' => return Poll::Ready(Ok(ChunkedState::Extension)),
400            b'\r' => return Poll::Ready(Ok(ChunkedState::SizeLf)),
401            _ => {
402                return Poll::Ready(Err(io::Error::new(
403                    io::ErrorKind::InvalidInput,
404                    "Invalid chunk size line: Invalid Size",
405                )));
406            }
407        }
408        Poll::Ready(Ok(ChunkedState::Size))
409    }
410    fn read_size_lws<R: MemRead>(
411        cx: &mut Context<'_>,
412        rdr: &mut R,
413    ) -> Poll<Result<ChunkedState, io::Error>> {
414        trace!("read_size_lws");
415        match byte!(rdr, cx) {
416            // LWS can follow the chunk size, but no more digits can come
417            b'\t' | b' ' => Poll::Ready(Ok(ChunkedState::SizeLws)),
418            b';' => Poll::Ready(Ok(ChunkedState::Extension)),
419            b'\r' => Poll::Ready(Ok(ChunkedState::SizeLf)),
420            _ => Poll::Ready(Err(io::Error::new(
421                io::ErrorKind::InvalidInput,
422                "Invalid chunk size linear white space",
423            ))),
424        }
425    }
426    fn read_extension<R: MemRead>(
427        cx: &mut Context<'_>,
428        rdr: &mut R,
429        extensions_cnt: &mut u64,
430    ) -> Poll<Result<ChunkedState, io::Error>> {
431        trace!("read_extension");
432        // We don't care about extensions really at all. Just ignore them.
433        // They "end" at the next CRLF.
434        //
435        // However, some implementations may not check for the CR, so to save
436        // them from themselves, we reject extensions containing plain LF as
437        // well.
438        match byte!(rdr, cx) {
439            b'\r' => Poll::Ready(Ok(ChunkedState::SizeLf)),
440            b'\n' => Poll::Ready(Err(io::Error::new(
441                io::ErrorKind::InvalidData,
442                "invalid chunk extension contains newline",
443            ))),
444            _ => {
445                *extensions_cnt += 1;
446                if *extensions_cnt >= CHUNKED_EXTENSIONS_LIMIT {
447                    Poll::Ready(Err(io::Error::new(
448                        io::ErrorKind::InvalidData,
449                        "chunk extensions over limit",
450                    )))
451                } else {
452                    Poll::Ready(Ok(ChunkedState::Extension))
453                }
454            } // no supported extensions
455        }
456    }
457    fn read_size_lf<R: MemRead>(
458        cx: &mut Context<'_>,
459        rdr: &mut R,
460        size: u64,
461    ) -> Poll<Result<ChunkedState, io::Error>> {
462        trace!("Chunk size is {:?}", size);
463        match byte!(rdr, cx) {
464            b'\n' => {
465                if size == 0 {
466                    Poll::Ready(Ok(ChunkedState::EndCr))
467                } else {
468                    debug!("incoming chunked header: {0:#X} ({0} bytes)", size);
469                    Poll::Ready(Ok(ChunkedState::Body))
470                }
471            }
472            _ => Poll::Ready(Err(io::Error::new(
473                io::ErrorKind::InvalidInput,
474                "Invalid chunk size LF",
475            ))),
476        }
477    }
478
479    fn read_body<R: MemRead>(
480        cx: &mut Context<'_>,
481        rdr: &mut R,
482        rem: &mut u64,
483        buf: &mut Option<Bytes>,
484    ) -> Poll<Result<ChunkedState, io::Error>> {
485        trace!("Chunked read, remaining={:?}", rem);
486
487        // cap remaining bytes at the max capacity of usize
488        let to_read = usize::try_from(*rem).unwrap_or(usize::MAX);
489        let slice = ready!(rdr.read_mem(cx, to_read))?;
490        let count = slice.len();
491
492        if count == 0 {
493            *rem = 0;
494            return Poll::Ready(Err(io::Error::new(
495                io::ErrorKind::UnexpectedEof,
496                IncompleteBody,
497            )));
498        }
499        *buf = Some(slice);
500        *rem -= count as u64;
501
502        if *rem > 0 {
503            Poll::Ready(Ok(ChunkedState::Body))
504        } else {
505            Poll::Ready(Ok(ChunkedState::BodyCr))
506        }
507    }
508    fn read_body_cr<R: MemRead>(
509        cx: &mut Context<'_>,
510        rdr: &mut R,
511    ) -> Poll<Result<ChunkedState, io::Error>> {
512        match byte!(rdr, cx) {
513            b'\r' => Poll::Ready(Ok(ChunkedState::BodyLf)),
514            _ => Poll::Ready(Err(io::Error::new(
515                io::ErrorKind::InvalidInput,
516                "Invalid chunk body CR",
517            ))),
518        }
519    }
520    fn read_body_lf<R: MemRead>(
521        cx: &mut Context<'_>,
522        rdr: &mut R,
523    ) -> Poll<Result<ChunkedState, io::Error>> {
524        match byte!(rdr, cx) {
525            b'\n' => Poll::Ready(Ok(ChunkedState::Start)),
526            _ => Poll::Ready(Err(io::Error::new(
527                io::ErrorKind::InvalidInput,
528                "Invalid chunk body LF",
529            ))),
530        }
531    }
532
533    fn read_trailer<R: MemRead>(
534        cx: &mut Context<'_>,
535        rdr: &mut R,
536        trailers_buf: &mut Option<BytesMut>,
537        h1_max_header_size: usize,
538    ) -> Poll<Result<ChunkedState, io::Error>> {
539        trace!("read_trailer");
540        let byte = byte!(rdr, cx);
541
542        put_u8!(
543            trailers_buf.as_mut().expect("trailers_buf is None"),
544            byte,
545            h1_max_header_size
546        );
547
548        match byte {
549            b'\r' => Poll::Ready(Ok(ChunkedState::TrailerLf)),
550            _ => Poll::Ready(Ok(ChunkedState::Trailer)),
551        }
552    }
553
554    fn read_trailer_lf<R: MemRead>(
555        cx: &mut Context<'_>,
556        rdr: &mut R,
557        trailers_buf: &mut Option<BytesMut>,
558        trailers_cnt: &mut usize,
559        h1_max_headers: usize,
560        h1_max_header_size: usize,
561    ) -> Poll<Result<ChunkedState, io::Error>> {
562        let byte = byte!(rdr, cx);
563        match byte {
564            b'\n' => {
565                if *trailers_cnt >= h1_max_headers {
566                    return Poll::Ready(Err(io::Error::new(
567                        io::ErrorKind::InvalidData,
568                        "chunk trailers count overflow",
569                    )));
570                }
571                *trailers_cnt += 1;
572
573                put_u8!(
574                    trailers_buf.as_mut().expect("trailers_buf is None"),
575                    byte,
576                    h1_max_header_size
577                );
578
579                Poll::Ready(Ok(ChunkedState::EndCr))
580            }
581            _ => Poll::Ready(Err(io::Error::new(
582                io::ErrorKind::InvalidInput,
583                "Invalid trailer end LF",
584            ))),
585        }
586    }
587
588    fn read_end_cr<R: MemRead>(
589        cx: &mut Context<'_>,
590        rdr: &mut R,
591        trailers_buf: &mut Option<BytesMut>,
592        h1_max_header_size: usize,
593    ) -> Poll<Result<ChunkedState, io::Error>> {
594        let byte = byte!(rdr, cx);
595        match byte {
596            b'\r' => {
597                if let Some(trailers_buf) = trailers_buf {
598                    put_u8!(trailers_buf, byte, h1_max_header_size);
599                }
600                Poll::Ready(Ok(ChunkedState::EndLf))
601            }
602            byte => {
603                match trailers_buf {
604                    None => {
605                        // 64 will fit a single Expires header without reallocating
606                        let mut buf = BytesMut::with_capacity(64);
607                        buf.put_u8(byte);
608                        *trailers_buf = Some(buf);
609                    }
610                    Some(trailers_buf) => {
611                        put_u8!(trailers_buf, byte, h1_max_header_size);
612                    }
613                }
614
615                Poll::Ready(Ok(ChunkedState::Trailer))
616            }
617        }
618    }
619    fn read_end_lf<R: MemRead>(
620        cx: &mut Context<'_>,
621        rdr: &mut R,
622        trailers_buf: &mut Option<BytesMut>,
623        h1_max_header_size: usize,
624    ) -> Poll<Result<ChunkedState, io::Error>> {
625        let byte = byte!(rdr, cx);
626        match byte {
627            b'\n' => {
628                if let Some(trailers_buf) = trailers_buf {
629                    put_u8!(trailers_buf, byte, h1_max_header_size);
630                }
631                Poll::Ready(Ok(ChunkedState::End))
632            }
633            _ => Poll::Ready(Err(io::Error::new(
634                io::ErrorKind::InvalidInput,
635                "Invalid chunk end LF",
636            ))),
637        }
638    }
639}
640
641// TODO: disallow Transfer-Encoding, Content-Length, Trailer, etc in trailers ??
642fn decode_trailers(buf: &mut BytesMut, count: usize) -> Result<HeaderMap, io::Error> {
643    let mut trailers = HeaderMap::new();
644    let mut headers = vec![httparse::EMPTY_HEADER; count];
645    let res = httparse::parse_headers(buf, &mut headers);
646    match res {
647        Ok(httparse::Status::Complete((_, headers))) => {
648            for header in headers {
649                use std::convert::TryFrom;
650                let name = match HeaderName::try_from(header.name) {
651                    Ok(name) => name,
652                    Err(_) => {
653                        return Err(io::Error::new(
654                            io::ErrorKind::InvalidInput,
655                            format!("Invalid header name: {header:?}"),
656                        ));
657                    }
658                };
659
660                let value = match HeaderValue::from_bytes(header.value) {
661                    Ok(value) => value,
662                    Err(_) => {
663                        return Err(io::Error::new(
664                            io::ErrorKind::InvalidInput,
665                            format!("Invalid header value: {header:?}"),
666                        ));
667                    }
668                };
669
670                trailers.append(name, value);
671            }
672
673            Ok(trailers)
674        }
675        Ok(httparse::Status::Partial) => Err(io::Error::new(
676            io::ErrorKind::InvalidInput,
677            "Partial header",
678        )),
679        Err(e) => Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
680    }
681}
682
683#[derive(Debug)]
684struct IncompleteBody;
685
686impl fmt::Display for IncompleteBody {
687    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
688        write!(f, "end of file before message length reached")
689    }
690}
691
692impl StdError for IncompleteBody {}
693
694#[cfg(test)]
695mod tests {
696    use super::*;
697    use crate::rt::{Read, ReadBuf};
698    use std::pin::Pin;
699    use std::time::Duration;
700
701    impl MemRead for &[u8] {
702        fn read_mem(&mut self, _: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
703            let n = std::cmp::min(len, self.len());
704            if n > 0 {
705                let (a, b) = self.split_at(n);
706                let buf = Bytes::copy_from_slice(a);
707                *self = b;
708                Poll::Ready(Ok(buf))
709            } else {
710                Poll::Ready(Ok(Bytes::new()))
711            }
712        }
713    }
714
715    impl MemRead for &mut (dyn Read + Unpin) {
716        fn read_mem(&mut self, cx: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
717            let mut v = vec![0; len];
718            let mut buf = ReadBuf::new(&mut v);
719            ready!(Pin::new(self).poll_read(cx, buf.unfilled())?);
720            Poll::Ready(Ok(Bytes::copy_from_slice(buf.filled())))
721        }
722    }
723
724    impl MemRead for Bytes {
725        fn read_mem(&mut self, _: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
726            let n = std::cmp::min(len, self.len());
727            let ret = self.split_to(n);
728            Poll::Ready(Ok(ret))
729        }
730    }
731
732    /*
733    use std::io;
734    use std::io::Write;
735    use super::Decoder;
736    use super::ChunkedState;
737    use futures::{Async, Poll};
738    use bytes::{BytesMut, Bytes};
739    use crate::mock::AsyncIo;
740    */
741
742    #[cfg(not(miri))]
743    #[tokio::test]
744    async fn test_read_chunk_size() {
745        use std::io::ErrorKind::{InvalidData, InvalidInput, UnexpectedEof};
746
747        async fn read(s: &str) -> u64 {
748            let mut state = ChunkedState::new();
749            let rdr = &mut s.as_bytes();
750            let mut size = 0;
751            let mut ext_cnt = 0;
752            let mut trailers_cnt = 0;
753            loop {
754                let result = futures_util::future::poll_fn(|cx| {
755                    state.step(
756                        cx,
757                        rdr,
758                        StepArgs {
759                            chunk_size: &mut size,
760                            extensions_cnt: &mut ext_cnt,
761                            chunk_buf: &mut None,
762                            trailers_buf: &mut None,
763                            trailers_cnt: &mut trailers_cnt,
764                            max_headers_cnt: DEFAULT_MAX_HEADERS,
765                            max_headers_bytes: TRAILER_LIMIT,
766                        },
767                    )
768                })
769                .await;
770                let desc = format!("read_size failed for {:?}", s);
771                state = result.expect(&desc);
772                if state == ChunkedState::Body || state == ChunkedState::EndCr {
773                    break;
774                }
775            }
776            size
777        }
778
779        async fn read_err(s: &str, expected_err: io::ErrorKind) {
780            let mut state = ChunkedState::new();
781            let rdr = &mut s.as_bytes();
782            let mut size = 0;
783            let mut ext_cnt = 0;
784            let mut trailers_cnt = 0;
785            loop {
786                let result = futures_util::future::poll_fn(|cx| {
787                    state.step(
788                        cx,
789                        rdr,
790                        StepArgs {
791                            chunk_size: &mut size,
792                            extensions_cnt: &mut ext_cnt,
793                            chunk_buf: &mut None,
794                            trailers_buf: &mut None,
795                            trailers_cnt: &mut trailers_cnt,
796                            max_headers_cnt: DEFAULT_MAX_HEADERS,
797                            max_headers_bytes: TRAILER_LIMIT,
798                        },
799                    )
800                })
801                .await;
802                state = match result {
803                    Ok(s) => s,
804                    Err(e) => {
805                        assert!(
806                            expected_err == e.kind(),
807                            "Reading {:?}, expected {:?}, but got {:?}",
808                            s,
809                            expected_err,
810                            e.kind()
811                        );
812                        return;
813                    }
814                };
815                if state == ChunkedState::Body || state == ChunkedState::End {
816                    panic!("Was Ok. Expected Err for {:?}", s);
817                }
818            }
819        }
820
821        assert_eq!(1, read("1\r\n").await);
822        assert_eq!(1, read("01\r\n").await);
823        assert_eq!(0, read("0\r\n").await);
824        assert_eq!(0, read("00\r\n").await);
825        assert_eq!(10, read("A\r\n").await);
826        assert_eq!(10, read("a\r\n").await);
827        assert_eq!(255, read("Ff\r\n").await);
828        assert_eq!(255, read("Ff   \r\n").await);
829        // Missing LF or CRLF
830        read_err("F\rF", InvalidInput).await;
831        read_err("F", UnexpectedEof).await;
832        // Missing digit
833        read_err("\r\n\r\n", InvalidInput).await;
834        read_err("\r\n", InvalidInput).await;
835        // Invalid hex digit
836        read_err("X\r\n", InvalidInput).await;
837        read_err("1X\r\n", InvalidInput).await;
838        read_err("-\r\n", InvalidInput).await;
839        read_err("-1\r\n", InvalidInput).await;
840        // Acceptable (if not fully valid) extensions do not influence the size
841        assert_eq!(1, read("1;extension\r\n").await);
842        assert_eq!(10, read("a;ext name=value\r\n").await);
843        assert_eq!(1, read("1;extension;extension2\r\n").await);
844        assert_eq!(1, read("1;;;  ;\r\n").await);
845        assert_eq!(2, read("2; extension...\r\n").await);
846        assert_eq!(3, read("3   ; extension=123\r\n").await);
847        assert_eq!(3, read("3   ;\r\n").await);
848        assert_eq!(3, read("3   ;   \r\n").await);
849        // Invalid extensions cause an error
850        read_err("1 invalid extension\r\n", InvalidInput).await;
851        read_err("1 A\r\n", InvalidInput).await;
852        read_err("1;no CRLF", UnexpectedEof).await;
853        read_err("1;reject\nnewlines\r\n", InvalidData).await;
854        // Overflow
855        read_err("f0000000000000003\r\n", InvalidData).await;
856    }
857
858    #[cfg(not(miri))]
859    #[tokio::test]
860    async fn test_read_sized_early_eof() {
861        let mut bytes = &b"foo bar"[..];
862        let mut decoder = Decoder::length(10);
863        assert_eq!(
864            decoder
865                .decode_fut(&mut bytes)
866                .await
867                .unwrap()
868                .data_ref()
869                .unwrap()
870                .len(),
871            7
872        );
873        let e = decoder.decode_fut(&mut bytes).await.unwrap_err();
874        assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
875    }
876
877    #[cfg(not(miri))]
878    #[tokio::test]
879    async fn test_read_chunked_early_eof() {
880        let mut bytes = &b"\
881            9\r\n\
882            foo bar\
883        "[..];
884        let mut decoder = Decoder::chunked(None, None);
885        assert_eq!(
886            decoder
887                .decode_fut(&mut bytes)
888                .await
889                .unwrap()
890                .data_ref()
891                .unwrap()
892                .len(),
893            7
894        );
895        let e = decoder.decode_fut(&mut bytes).await.unwrap_err();
896        assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
897    }
898
899    #[cfg(not(miri))]
900    #[tokio::test]
901    async fn test_read_chunked_single_read() {
902        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n"[..];
903        let buf = Decoder::chunked(None, None)
904            .decode_fut(&mut mock_buf)
905            .await
906            .expect("decode")
907            .into_data()
908            .expect("unknown frame type");
909        assert_eq!(16, buf.len());
910        let result = String::from_utf8(buf.as_ref().to_vec()).expect("decode String");
911        assert_eq!("1234567890abcdef", &result);
912    }
913
914    #[tokio::test]
915    async fn test_read_chunked_with_missing_zero_digit() {
916        // After reading a valid chunk, the ending is missing a zero.
917        let mut mock_buf = &b"1\r\nZ\r\n\r\n\r\n"[..];
918        let mut decoder = Decoder::chunked(None, None);
919        let buf = decoder
920            .decode_fut(&mut mock_buf)
921            .await
922            .expect("decode")
923            .into_data()
924            .expect("unknown frame type");
925        assert_eq!("Z", buf);
926
927        let err = decoder
928            .decode_fut(&mut mock_buf)
929            .await
930            .expect_err("decode 2");
931        assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
932    }
933
934    #[tokio::test]
935    async fn test_read_chunked_extensions_over_limit() {
936        // construct a chunked body where each individual chunked extension
937        // is totally fine, but combined is over the limit.
938        let per_chunk = super::CHUNKED_EXTENSIONS_LIMIT * 2 / 3;
939        let mut scratch = vec![];
940        for _ in 0..2 {
941            scratch.extend(b"1;");
942            scratch.extend(b"x".repeat(per_chunk as usize));
943            scratch.extend(b"\r\nA\r\n");
944        }
945        scratch.extend(b"0\r\n\r\n");
946        let mut mock_buf = Bytes::from(scratch);
947
948        let mut decoder = Decoder::chunked(None, None);
949        let buf1 = decoder
950            .decode_fut(&mut mock_buf)
951            .await
952            .expect("decode1")
953            .into_data()
954            .expect("unknown frame type");
955        assert_eq!(&buf1[..], b"A");
956
957        let err = decoder
958            .decode_fut(&mut mock_buf)
959            .await
960            .expect_err("decode2");
961        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
962        assert_eq!(err.to_string(), "chunk extensions over limit");
963    }
964
965    #[cfg(not(miri))]
966    #[tokio::test]
967    async fn test_read_chunked_trailer_with_missing_lf() {
968        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\nbad\r\r\n"[..];
969        let mut decoder = Decoder::chunked(None, None);
970        decoder.decode_fut(&mut mock_buf).await.expect("decode");
971        let e = decoder.decode_fut(&mut mock_buf).await.unwrap_err();
972        assert_eq!(e.kind(), io::ErrorKind::InvalidInput);
973    }
974
975    #[cfg(not(miri))]
976    #[tokio::test]
977    async fn test_read_chunked_after_eof() {
978        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n\r\n"[..];
979        let mut decoder = Decoder::chunked(None, None);
980
981        // normal read
982        let buf = decoder
983            .decode_fut(&mut mock_buf)
984            .await
985            .unwrap()
986            .into_data()
987            .expect("unknown frame type");
988        assert_eq!(16, buf.len());
989        let result = String::from_utf8(buf.as_ref().to_vec()).expect("decode String");
990        assert_eq!("1234567890abcdef", &result);
991
992        // eof read
993        let buf = decoder
994            .decode_fut(&mut mock_buf)
995            .await
996            .expect("decode")
997            .into_data()
998            .expect("unknown frame type");
999        assert_eq!(0, buf.len());
1000
1001        // ensure read after eof also returns eof
1002        let buf = decoder
1003            .decode_fut(&mut mock_buf)
1004            .await
1005            .expect("decode")
1006            .into_data()
1007            .expect("unknown frame type");
1008        assert_eq!(0, buf.len());
1009    }
1010
1011    // perform an async read using a custom buffer size and causing a blocking
1012    // read at the specified byte
1013    async fn read_async(mut decoder: Decoder, content: &[u8], block_at: usize) -> String {
1014        let mut outs = Vec::new();
1015
1016        let mut ins = crate::common::io::Compat::new(if block_at == 0 {
1017            tokio_test::io::Builder::new()
1018                .wait(Duration::from_millis(10))
1019                .read(content)
1020                .build()
1021        } else {
1022            tokio_test::io::Builder::new()
1023                .read(&content[..block_at])
1024                .wait(Duration::from_millis(10))
1025                .read(&content[block_at..])
1026                .build()
1027        });
1028
1029        let mut ins = &mut ins as &mut (dyn Read + Unpin);
1030
1031        loop {
1032            let buf = decoder
1033                .decode_fut(&mut ins)
1034                .await
1035                .expect("unexpected decode error")
1036                .into_data()
1037                .expect("unexpected frame type");
1038            if buf.is_empty() {
1039                break; // eof
1040            }
1041            outs.extend(buf.as_ref());
1042        }
1043
1044        String::from_utf8(outs).expect("decode String")
1045    }
1046
1047    // iterate over the different ways that this async read could go.
1048    // tests blocking a read at each byte along the content - The shotgun approach
1049    async fn all_async_cases(content: &str, expected: &str, decoder: Decoder) {
1050        let content_len = content.len();
1051        for block_at in 0..content_len {
1052            let actual = read_async(decoder.clone(), content.as_bytes(), block_at).await;
1053            assert_eq!(expected, &actual) //, "Failed async. Blocking at {}", block_at);
1054        }
1055    }
1056
1057    #[cfg(not(miri))]
1058    #[tokio::test]
1059    async fn test_read_length_async() {
1060        let content = "foobar";
1061        all_async_cases(content, content, Decoder::length(content.len() as u64)).await;
1062    }
1063
1064    #[cfg(not(miri))]
1065    #[tokio::test]
1066    async fn test_read_chunked_async() {
1067        let content = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n";
1068        let expected = "foobar";
1069        all_async_cases(content, expected, Decoder::chunked(None, None)).await;
1070    }
1071
1072    #[cfg(not(miri))]
1073    #[tokio::test]
1074    async fn test_read_eof_async() {
1075        let content = "foobar";
1076        all_async_cases(content, content, Decoder::eof()).await;
1077    }
1078
1079    #[cfg(all(feature = "nightly", not(miri)))]
1080    #[bench]
1081    fn bench_decode_chunked_1kb(b: &mut test::Bencher) {
1082        let rt = new_runtime();
1083
1084        const LEN: usize = 1024;
1085        let mut vec = Vec::new();
1086        vec.extend(format!("{:x}\r\n", LEN).as_bytes());
1087        vec.extend(&[0; LEN][..]);
1088        vec.extend(b"\r\n");
1089        let content = Bytes::from(vec);
1090
1091        b.bytes = LEN as u64;
1092
1093        b.iter(|| {
1094            let mut decoder = Decoder::chunked(None, None);
1095            rt.block_on(async {
1096                let mut raw = content.clone();
1097                let chunk = decoder
1098                    .decode_fut(&mut raw)
1099                    .await
1100                    .unwrap()
1101                    .into_data()
1102                    .unwrap();
1103                assert_eq!(chunk.len(), LEN);
1104            });
1105        });
1106    }
1107
1108    #[cfg(all(feature = "nightly", not(miri)))]
1109    #[bench]
1110    fn bench_decode_length_1kb(b: &mut test::Bencher) {
1111        let rt = new_runtime();
1112
1113        const LEN: usize = 1024;
1114        let content = Bytes::from(&[0; LEN][..]);
1115        b.bytes = LEN as u64;
1116
1117        b.iter(|| {
1118            let mut decoder = Decoder::length(LEN as u64);
1119            rt.block_on(async {
1120                let mut raw = content.clone();
1121                let chunk = decoder
1122                    .decode_fut(&mut raw)
1123                    .await
1124                    .unwrap()
1125                    .into_data()
1126                    .unwrap();
1127                assert_eq!(chunk.len(), LEN);
1128            });
1129        });
1130    }
1131
1132    #[cfg(feature = "nightly")]
1133    fn new_runtime() -> tokio::runtime::Runtime {
1134        tokio::runtime::Builder::new_current_thread()
1135            .enable_all()
1136            .build()
1137            .expect("rt build")
1138    }
1139
1140    #[test]
1141    fn test_decode_trailers() {
1142        let mut buf = BytesMut::new();
1143        buf.extend_from_slice(
1144            b"Expires: Wed, 21 Oct 2015 07:28:00 GMT\r\nX-Stream-Error: failed to decode\r\n\r\n",
1145        );
1146        let headers = decode_trailers(&mut buf, 2).expect("decode_trailers");
1147        assert_eq!(headers.len(), 2);
1148        assert_eq!(
1149            headers.get("Expires").unwrap(),
1150            "Wed, 21 Oct 2015 07:28:00 GMT"
1151        );
1152        assert_eq!(headers.get("X-Stream-Error").unwrap(), "failed to decode");
1153    }
1154
1155    #[test]
1156    fn test_decode_trailers_preserves_duplicate_values() {
1157        let mut buf = BytesMut::new();
1158        buf.extend_from_slice(b"X-Trace: first\r\nX-Trace: second\r\n\r\n");
1159
1160        let headers = decode_trailers(&mut buf, 2).expect("decode_trailers");
1161        let values = headers
1162            .get_all("X-Trace")
1163            .iter()
1164            .map(|value| value.to_str().unwrap())
1165            .collect::<Vec<_>>();
1166
1167        assert_eq!(values, ["first", "second"]);
1168    }
1169
1170    #[tokio::test]
1171    async fn test_trailer_max_headers_enforced() {
1172        let h1_max_headers = 10;
1173        let mut scratch = vec![];
1174        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1175        for i in 0..=h1_max_headers {
1176            scratch.extend(format!("trailer{}: {}\r\n", i, i).as_bytes());
1177        }
1178        scratch.extend(b"\r\n");
1179        let mut mock_buf = Bytes::from(scratch);
1180
1181        let mut decoder = Decoder::chunked(Some(h1_max_headers), None);
1182
1183        // ready chunked body
1184        let buf = decoder
1185            .decode_fut(&mut mock_buf)
1186            .await
1187            .unwrap()
1188            .into_data()
1189            .expect("unknown frame type");
1190        assert_eq!(16, buf.len());
1191
1192        // eof read
1193        let err = decoder
1194            .decode_fut(&mut mock_buf)
1195            .await
1196            .expect_err("trailer fields over limit");
1197        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1198    }
1199
1200    #[tokio::test]
1201    async fn test_trailer_max_headers_allows_exact_limit() {
1202        let h1_max_headers = 10;
1203        let mut scratch = vec![];
1204        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1205        for i in 0..h1_max_headers {
1206            scratch.extend(format!("trailer{}: {}\r\n", i, i).as_bytes());
1207        }
1208        scratch.extend(b"\r\n");
1209        let mut mock_buf = Bytes::from(scratch);
1210
1211        let mut decoder = Decoder::chunked(Some(h1_max_headers), None);
1212
1213        let buf = decoder
1214            .decode_fut(&mut mock_buf)
1215            .await
1216            .unwrap()
1217            .into_data()
1218            .expect("unknown frame type");
1219        assert_eq!(16, buf.len());
1220
1221        let trailers = decoder
1222            .decode_fut(&mut mock_buf)
1223            .await
1224            .expect("trailers at limit")
1225            .into_trailers()
1226            .expect("unknown frame type");
1227        assert_eq!(trailers.len(), h1_max_headers);
1228    }
1229
1230    #[tokio::test]
1231    async fn test_trailer_max_header_size_huge_trailer() {
1232        let max_header_size = 1024;
1233        let mut scratch = vec![];
1234        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1235        scratch.extend(format!("huge_trailer: {}\r\n", "x".repeat(max_header_size)).as_bytes());
1236        scratch.extend(b"\r\n");
1237        let mut mock_buf = Bytes::from(scratch);
1238
1239        let mut decoder = Decoder::chunked(None, Some(max_header_size));
1240
1241        // ready chunked body
1242        let buf = decoder
1243            .decode_fut(&mut mock_buf)
1244            .await
1245            .unwrap()
1246            .into_data()
1247            .expect("unknown frame type");
1248        assert_eq!(16, buf.len());
1249
1250        // eof read
1251        let err = decoder
1252            .decode_fut(&mut mock_buf)
1253            .await
1254            .expect_err("trailers over limit");
1255        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1256    }
1257
1258    #[tokio::test]
1259    async fn test_trailer_max_header_size_many_small_trailers() {
1260        let max_headers = 10;
1261        let header_size = 64;
1262        let mut scratch = vec![];
1263        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1264
1265        for i in 0..max_headers {
1266            scratch.extend(format!("trailer{}: {}\r\n", i, "x".repeat(header_size)).as_bytes());
1267        }
1268
1269        scratch.extend(b"\r\n");
1270        let mut mock_buf = Bytes::from(scratch);
1271
1272        let mut decoder = Decoder::chunked(None, Some(max_headers * header_size));
1273
1274        // ready chunked body
1275        let buf = decoder
1276            .decode_fut(&mut mock_buf)
1277            .await
1278            .unwrap()
1279            .into_data()
1280            .expect("unknown frame type");
1281        assert_eq!(16, buf.len());
1282
1283        // eof read
1284        let err = decoder
1285            .decode_fut(&mut mock_buf)
1286            .await
1287            .expect_err("trailers over limit");
1288        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1289    }
1290}
hyper/proto/h1/decode.rs

hyper/proto/h1/
decode.rs
