hyper/proto/h1/
decode.rs

1use std::error::Error as StdError;
2use std::fmt;
3use std::io;
4use std::task::{Context, Poll};
5
6use bytes::{BufMut, Bytes, BytesMut};
7use futures_core::ready;
8use http::{HeaderMap, HeaderName, HeaderValue};
9use http_body::Frame;
10
11use super::io::MemRead;
12use super::role::DEFAULT_MAX_HEADERS;
13use super::DecodedLength;
14
15use self::Kind::{Chunked, Eof, Length};
16
17/// Maximum amount of bytes allowed in chunked extensions.
18///
19/// This limit is currentlty applied for the entire body, not per chunk.
20const CHUNKED_EXTENSIONS_LIMIT: u64 = 1024 * 16;
21
22/// Maximum number of bytes allowed for all trailer fields.
23///
24/// TODO: remove this when we land `h1_max_header_size` support.
25const TRAILER_LIMIT: usize = 1024 * 16;
26
27/// Decoders to handle different Transfer-Encodings.
28///
29/// If a message body does not include a Transfer-Encoding, it *should*
30/// include a Content-Length header.
31#[derive(Clone, PartialEq)]
32pub(crate) struct Decoder {
33    kind: Kind,
34}
35
36#[derive(Debug, Clone, PartialEq)]
37enum Kind {
38    /// A Reader used when a Content-Length header is passed with a positive integer.
39    Length(u64),
40    /// A Reader used when Transfer-Encoding is `chunked`.
41    Chunked {
42        state: ChunkedState,
43        chunk_len: u64,
44        extensions_cnt: u64,
45        trailers_buf: Option<BytesMut>,
46        trailers_cnt: usize,
47        h1_max_headers: Option<usize>,
48        h1_max_header_size: Option<usize>,
49    },
50    /// A Reader used for responses that don't indicate a length or chunked.
51    ///
52    /// The bool tracks when EOF is seen on the transport.
53    ///
54    /// Note: This should only used for `Response`s. It is illegal for a
55    /// `Request` to be made with both `Content-Length` and
56    /// `Transfer-Encoding: chunked` missing, as explained from the spec:
57    ///
58    /// > If a Transfer-Encoding header field is present in a response and
59    /// > the chunked transfer coding is not the final encoding, the
60    /// > message body length is determined by reading the connection until
61    /// > it is closed by the server.  If a Transfer-Encoding header field
62    /// > is present in a request and the chunked transfer coding is not
63    /// > the final encoding, the message body length cannot be determined
64    /// > reliably; the server MUST respond with the 400 (Bad Request)
65    /// > status code and then close the connection.
66    Eof(bool),
67}
68
69#[derive(Debug, PartialEq, Clone, Copy)]
70enum ChunkedState {
71    Start,
72    Size,
73    SizeLws,
74    Extension,
75    SizeLf,
76    Body,
77    BodyCr,
78    BodyLf,
79    Trailer,
80    TrailerLf,
81    EndCr,
82    EndLf,
83    End,
84}
85
86impl Decoder {
87    // constructors
88
89    pub(crate) fn length(x: u64) -> Decoder {
90        Decoder {
91            kind: Kind::Length(x),
92        }
93    }
94
95    pub(crate) fn chunked(
96        h1_max_headers: Option<usize>,
97        h1_max_header_size: Option<usize>,
98    ) -> Decoder {
99        Decoder {
100            kind: Kind::Chunked {
101                state: ChunkedState::new(),
102                chunk_len: 0,
103                extensions_cnt: 0,
104                trailers_buf: None,
105                trailers_cnt: 0,
106                h1_max_headers,
107                h1_max_header_size,
108            },
109        }
110    }
111
112    pub(crate) fn eof() -> Decoder {
113        Decoder {
114            kind: Kind::Eof(false),
115        }
116    }
117
118    pub(super) fn new(
119        len: DecodedLength,
120        h1_max_headers: Option<usize>,
121        h1_max_header_size: Option<usize>,
122    ) -> Self {
123        match len {
124            DecodedLength::CHUNKED => Decoder::chunked(h1_max_headers, h1_max_header_size),
125            DecodedLength::CLOSE_DELIMITED => Decoder::eof(),
126            length => Decoder::length(length.danger_len()),
127        }
128    }
129
130    // methods
131
132    pub(crate) fn is_eof(&self) -> bool {
133        matches!(
134            self.kind,
135            Length(0)
136                | Chunked {
137                    state: ChunkedState::End,
138                    ..
139                }
140                | Eof(true)
141        )
142    }
143
144    pub(crate) fn decode<R: MemRead>(
145        &mut self,
146        cx: &mut Context<'_>,
147        body: &mut R,
148    ) -> Poll<Result<Frame<Bytes>, io::Error>> {
149        trace!("decode; state={:?}", self.kind);
150        match self.kind {
151            Length(ref mut remaining) => {
152                if *remaining == 0 {
153                    Poll::Ready(Ok(Frame::data(Bytes::new())))
154                } else {
155                    let to_read = usize::try_from(*remaining).unwrap_or(usize::MAX);
156                    let buf = ready!(body.read_mem(cx, to_read))?;
157                    let num = buf.as_ref().len() as u64;
158                    if num > *remaining {
159                        *remaining = 0;
160                    } else if num == 0 {
161                        return Poll::Ready(Err(io::Error::new(
162                            io::ErrorKind::UnexpectedEof,
163                            IncompleteBody,
164                        )));
165                    } else {
166                        *remaining -= num;
167                    }
168                    Poll::Ready(Ok(Frame::data(buf)))
169                }
170            }
171            Chunked {
172                ref mut state,
173                ref mut chunk_len,
174                ref mut extensions_cnt,
175                ref mut trailers_buf,
176                ref mut trailers_cnt,
177                ref h1_max_headers,
178                ref h1_max_header_size,
179            } => {
180                let h1_max_headers = h1_max_headers.unwrap_or(DEFAULT_MAX_HEADERS);
181                let h1_max_header_size = h1_max_header_size.unwrap_or(TRAILER_LIMIT);
182                loop {
183                    let mut buf = None;
184                    // advances the chunked state
185                    *state = ready!(state.step(
186                        cx,
187                        body,
188                        StepArgs {
189                            chunk_size: chunk_len,
190                            extensions_cnt,
191                            chunk_buf: &mut buf,
192                            trailers_buf,
193                            trailers_cnt,
194                            max_headers_cnt: h1_max_headers,
195                            max_headers_bytes: h1_max_header_size,
196                        }
197                    ))?;
198                    if *state == ChunkedState::End {
199                        trace!("end of chunked");
200
201                        if trailers_buf.is_some() {
202                            trace!("found possible trailers");
203
204                            match decode_trailers(
205                                &mut trailers_buf.take().expect("Trailer is None"),
206                                *trailers_cnt,
207                            ) {
208                                Ok(headers) => {
209                                    return Poll::Ready(Ok(Frame::trailers(headers)));
210                                }
211                                Err(e) => {
212                                    return Poll::Ready(Err(e));
213                                }
214                            }
215                        }
216
217                        return Poll::Ready(Ok(Frame::data(Bytes::new())));
218                    }
219                    if let Some(buf) = buf {
220                        return Poll::Ready(Ok(Frame::data(buf)));
221                    }
222                }
223            }
224            Eof(ref mut is_eof) => {
225                if *is_eof {
226                    Poll::Ready(Ok(Frame::data(Bytes::new())))
227                } else {
228                    // 8192 chosen because its about 2 packets, there probably
229                    // won't be that much available, so don't have MemReaders
230                    // allocate buffers to big
231                    body.read_mem(cx, 8192).map_ok(|slice| {
232                        *is_eof = slice.is_empty();
233                        Frame::data(slice)
234                    })
235                }
236            }
237        }
238    }
239
240    #[cfg(test)]
241    async fn decode_fut<R: MemRead>(&mut self, body: &mut R) -> Result<Frame<Bytes>, io::Error> {
242        futures_util::future::poll_fn(move |cx| self.decode(cx, body)).await
243    }
244}
245
246impl fmt::Debug for Decoder {
247    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
248        fmt::Debug::fmt(&self.kind, f)
249    }
250}
251
252macro_rules! byte (
253    ($rdr:ident, $cx:expr) => ({
254        let buf = ready!($rdr.read_mem($cx, 1))?;
255        if !buf.is_empty() {
256            buf[0]
257        } else {
258            return Poll::Ready(Err(io::Error::new(io::ErrorKind::UnexpectedEof,
259                                      "unexpected EOF during chunk size line")));
260        }
261    })
262);
263
264macro_rules! or_overflow {
265    ($e:expr) => (
266        match $e {
267            Some(val) => val,
268            None => return Poll::Ready(Err(io::Error::new(
269                io::ErrorKind::InvalidData,
270                "invalid chunk size: overflow",
271            ))),
272        }
273    )
274}
275
276macro_rules! put_u8 {
277    ($trailers_buf:expr, $byte:expr, $limit:expr) => {
278        $trailers_buf.put_u8($byte);
279
280        if $trailers_buf.len() >= $limit {
281            return Poll::Ready(Err(io::Error::new(
282                io::ErrorKind::InvalidData,
283                "chunk trailers bytes over limit",
284            )));
285        }
286    };
287}
288
289struct StepArgs<'a> {
290    chunk_size: &'a mut u64,
291    chunk_buf: &'a mut Option<Bytes>,
292    extensions_cnt: &'a mut u64,
293    trailers_buf: &'a mut Option<BytesMut>,
294    trailers_cnt: &'a mut usize,
295    max_headers_cnt: usize,
296    max_headers_bytes: usize,
297}
298
299impl ChunkedState {
300    fn new() -> ChunkedState {
301        ChunkedState::Start
302    }
303    fn step<R: MemRead>(
304        &self,
305        cx: &mut Context<'_>,
306        body: &mut R,
307        StepArgs {
308            chunk_size,
309            chunk_buf,
310            extensions_cnt,
311            trailers_buf,
312            trailers_cnt,
313            max_headers_cnt,
314            max_headers_bytes,
315        }: StepArgs<'_>,
316    ) -> Poll<Result<ChunkedState, io::Error>> {
317        use self::ChunkedState::*;
318        match *self {
319            Start => ChunkedState::read_start(cx, body, chunk_size),
320            Size => ChunkedState::read_size(cx, body, chunk_size),
321            SizeLws => ChunkedState::read_size_lws(cx, body),
322            Extension => ChunkedState::read_extension(cx, body, extensions_cnt),
323            SizeLf => ChunkedState::read_size_lf(cx, body, *chunk_size),
324            Body => ChunkedState::read_body(cx, body, chunk_size, chunk_buf),
325            BodyCr => ChunkedState::read_body_cr(cx, body),
326            BodyLf => ChunkedState::read_body_lf(cx, body),
327            Trailer => ChunkedState::read_trailer(cx, body, trailers_buf, max_headers_bytes),
328            TrailerLf => ChunkedState::read_trailer_lf(
329                cx,
330                body,
331                trailers_buf,
332                trailers_cnt,
333                max_headers_cnt,
334                max_headers_bytes,
335            ),
336            EndCr => ChunkedState::read_end_cr(cx, body, trailers_buf, max_headers_bytes),
337            EndLf => ChunkedState::read_end_lf(cx, body, trailers_buf, max_headers_bytes),
338            End => Poll::Ready(Ok(ChunkedState::End)),
339        }
340    }
341
342    fn read_start<R: MemRead>(
343        cx: &mut Context<'_>,
344        rdr: &mut R,
345        size: &mut u64,
346    ) -> Poll<Result<ChunkedState, io::Error>> {
347        trace!("Read chunk start");
348
349        let radix = 16;
350        match byte!(rdr, cx) {
351            b @ b'0'..=b'9' => {
352                *size = or_overflow!(size.checked_mul(radix));
353                *size = or_overflow!(size.checked_add(u64::from(b - b'0')));
354            }
355            b @ b'a'..=b'f' => {
356                *size = or_overflow!(size.checked_mul(radix));
357                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'a')));
358            }
359            b @ b'A'..=b'F' => {
360                *size = or_overflow!(size.checked_mul(radix));
361                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'A')));
362            }
363            _ => {
364                return Poll::Ready(Err(io::Error::new(
365                    io::ErrorKind::InvalidInput,
366                    "Invalid chunk size line: missing size digit",
367                )));
368            }
369        }
370
371        Poll::Ready(Ok(ChunkedState::Size))
372    }
373
374    fn read_size<R: MemRead>(
375        cx: &mut Context<'_>,
376        rdr: &mut R,
377        size: &mut u64,
378    ) -> Poll<Result<ChunkedState, io::Error>> {
379        trace!("Read chunk hex size");
380
381        let radix = 16;
382        match byte!(rdr, cx) {
383            b @ b'0'..=b'9' => {
384                *size = or_overflow!(size.checked_mul(radix));
385                *size = or_overflow!(size.checked_add(u64::from(b - b'0')));
386            }
387            b @ b'a'..=b'f' => {
388                *size = or_overflow!(size.checked_mul(radix));
389                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'a')));
390            }
391            b @ b'A'..=b'F' => {
392                *size = or_overflow!(size.checked_mul(radix));
393                *size = or_overflow!(size.checked_add(u64::from(b + 10 - b'A')));
394            }
395            b'\t' | b' ' => return Poll::Ready(Ok(ChunkedState::SizeLws)),
396            b';' => return Poll::Ready(Ok(ChunkedState::Extension)),
397            b'\r' => return Poll::Ready(Ok(ChunkedState::SizeLf)),
398            _ => {
399                return Poll::Ready(Err(io::Error::new(
400                    io::ErrorKind::InvalidInput,
401                    "Invalid chunk size line: Invalid Size",
402                )));
403            }
404        }
405        Poll::Ready(Ok(ChunkedState::Size))
406    }
407    fn read_size_lws<R: MemRead>(
408        cx: &mut Context<'_>,
409        rdr: &mut R,
410    ) -> Poll<Result<ChunkedState, io::Error>> {
411        trace!("read_size_lws");
412        match byte!(rdr, cx) {
413            // LWS can follow the chunk size, but no more digits can come
414            b'\t' | b' ' => Poll::Ready(Ok(ChunkedState::SizeLws)),
415            b';' => Poll::Ready(Ok(ChunkedState::Extension)),
416            b'\r' => Poll::Ready(Ok(ChunkedState::SizeLf)),
417            _ => Poll::Ready(Err(io::Error::new(
418                io::ErrorKind::InvalidInput,
419                "Invalid chunk size linear white space",
420            ))),
421        }
422    }
423    fn read_extension<R: MemRead>(
424        cx: &mut Context<'_>,
425        rdr: &mut R,
426        extensions_cnt: &mut u64,
427    ) -> Poll<Result<ChunkedState, io::Error>> {
428        trace!("read_extension");
429        // We don't care about extensions really at all. Just ignore them.
430        // They "end" at the next CRLF.
431        //
432        // However, some implementations may not check for the CR, so to save
433        // them from themselves, we reject extensions containing plain LF as
434        // well.
435        match byte!(rdr, cx) {
436            b'\r' => Poll::Ready(Ok(ChunkedState::SizeLf)),
437            b'\n' => Poll::Ready(Err(io::Error::new(
438                io::ErrorKind::InvalidData,
439                "invalid chunk extension contains newline",
440            ))),
441            _ => {
442                *extensions_cnt += 1;
443                if *extensions_cnt >= CHUNKED_EXTENSIONS_LIMIT {
444                    Poll::Ready(Err(io::Error::new(
445                        io::ErrorKind::InvalidData,
446                        "chunk extensions over limit",
447                    )))
448                } else {
449                    Poll::Ready(Ok(ChunkedState::Extension))
450                }
451            } // no supported extensions
452        }
453    }
454    fn read_size_lf<R: MemRead>(
455        cx: &mut Context<'_>,
456        rdr: &mut R,
457        size: u64,
458    ) -> Poll<Result<ChunkedState, io::Error>> {
459        trace!("Chunk size is {:?}", size);
460        match byte!(rdr, cx) {
461            b'\n' => {
462                if size == 0 {
463                    Poll::Ready(Ok(ChunkedState::EndCr))
464                } else {
465                    debug!("incoming chunked header: {0:#X} ({0} bytes)", size);
466                    Poll::Ready(Ok(ChunkedState::Body))
467                }
468            }
469            _ => Poll::Ready(Err(io::Error::new(
470                io::ErrorKind::InvalidInput,
471                "Invalid chunk size LF",
472            ))),
473        }
474    }
475
476    fn read_body<R: MemRead>(
477        cx: &mut Context<'_>,
478        rdr: &mut R,
479        rem: &mut u64,
480        buf: &mut Option<Bytes>,
481    ) -> Poll<Result<ChunkedState, io::Error>> {
482        trace!("Chunked read, remaining={:?}", rem);
483
484        // cap remaining bytes at the max capacity of usize
485        let to_read = usize::try_from(*rem).unwrap_or(usize::MAX);
486        let slice = ready!(rdr.read_mem(cx, to_read))?;
487        let count = slice.len();
488
489        if count == 0 {
490            *rem = 0;
491            return Poll::Ready(Err(io::Error::new(
492                io::ErrorKind::UnexpectedEof,
493                IncompleteBody,
494            )));
495        }
496        *buf = Some(slice);
497        *rem -= count as u64;
498
499        if *rem > 0 {
500            Poll::Ready(Ok(ChunkedState::Body))
501        } else {
502            Poll::Ready(Ok(ChunkedState::BodyCr))
503        }
504    }
505    fn read_body_cr<R: MemRead>(
506        cx: &mut Context<'_>,
507        rdr: &mut R,
508    ) -> Poll<Result<ChunkedState, io::Error>> {
509        match byte!(rdr, cx) {
510            b'\r' => Poll::Ready(Ok(ChunkedState::BodyLf)),
511            _ => Poll::Ready(Err(io::Error::new(
512                io::ErrorKind::InvalidInput,
513                "Invalid chunk body CR",
514            ))),
515        }
516    }
517    fn read_body_lf<R: MemRead>(
518        cx: &mut Context<'_>,
519        rdr: &mut R,
520    ) -> Poll<Result<ChunkedState, io::Error>> {
521        match byte!(rdr, cx) {
522            b'\n' => Poll::Ready(Ok(ChunkedState::Start)),
523            _ => Poll::Ready(Err(io::Error::new(
524                io::ErrorKind::InvalidInput,
525                "Invalid chunk body LF",
526            ))),
527        }
528    }
529
530    fn read_trailer<R: MemRead>(
531        cx: &mut Context<'_>,
532        rdr: &mut R,
533        trailers_buf: &mut Option<BytesMut>,
534        h1_max_header_size: usize,
535    ) -> Poll<Result<ChunkedState, io::Error>> {
536        trace!("read_trailer");
537        let byte = byte!(rdr, cx);
538
539        put_u8!(
540            trailers_buf.as_mut().expect("trailers_buf is None"),
541            byte,
542            h1_max_header_size
543        );
544
545        match byte {
546            b'\r' => Poll::Ready(Ok(ChunkedState::TrailerLf)),
547            _ => Poll::Ready(Ok(ChunkedState::Trailer)),
548        }
549    }
550
551    fn read_trailer_lf<R: MemRead>(
552        cx: &mut Context<'_>,
553        rdr: &mut R,
554        trailers_buf: &mut Option<BytesMut>,
555        trailers_cnt: &mut usize,
556        h1_max_headers: usize,
557        h1_max_header_size: usize,
558    ) -> Poll<Result<ChunkedState, io::Error>> {
559        let byte = byte!(rdr, cx);
560        match byte {
561            b'\n' => {
562                if *trailers_cnt >= h1_max_headers {
563                    return Poll::Ready(Err(io::Error::new(
564                        io::ErrorKind::InvalidData,
565                        "chunk trailers count overflow",
566                    )));
567                }
568                *trailers_cnt += 1;
569
570                put_u8!(
571                    trailers_buf.as_mut().expect("trailers_buf is None"),
572                    byte,
573                    h1_max_header_size
574                );
575
576                Poll::Ready(Ok(ChunkedState::EndCr))
577            }
578            _ => Poll::Ready(Err(io::Error::new(
579                io::ErrorKind::InvalidInput,
580                "Invalid trailer end LF",
581            ))),
582        }
583    }
584
585    fn read_end_cr<R: MemRead>(
586        cx: &mut Context<'_>,
587        rdr: &mut R,
588        trailers_buf: &mut Option<BytesMut>,
589        h1_max_header_size: usize,
590    ) -> Poll<Result<ChunkedState, io::Error>> {
591        let byte = byte!(rdr, cx);
592        match byte {
593            b'\r' => {
594                if let Some(trailers_buf) = trailers_buf {
595                    put_u8!(trailers_buf, byte, h1_max_header_size);
596                }
597                Poll::Ready(Ok(ChunkedState::EndLf))
598            }
599            byte => {
600                match trailers_buf {
601                    None => {
602                        // 64 will fit a single Expires header without reallocating
603                        let mut buf = BytesMut::with_capacity(64);
604                        buf.put_u8(byte);
605                        *trailers_buf = Some(buf);
606                    }
607                    Some(ref mut trailers_buf) => {
608                        put_u8!(trailers_buf, byte, h1_max_header_size);
609                    }
610                }
611
612                Poll::Ready(Ok(ChunkedState::Trailer))
613            }
614        }
615    }
616    fn read_end_lf<R: MemRead>(
617        cx: &mut Context<'_>,
618        rdr: &mut R,
619        trailers_buf: &mut Option<BytesMut>,
620        h1_max_header_size: usize,
621    ) -> Poll<Result<ChunkedState, io::Error>> {
622        let byte = byte!(rdr, cx);
623        match byte {
624            b'\n' => {
625                if let Some(trailers_buf) = trailers_buf {
626                    put_u8!(trailers_buf, byte, h1_max_header_size);
627                }
628                Poll::Ready(Ok(ChunkedState::End))
629            }
630            _ => Poll::Ready(Err(io::Error::new(
631                io::ErrorKind::InvalidInput,
632                "Invalid chunk end LF",
633            ))),
634        }
635    }
636}
637
638// TODO: disallow Transfer-Encoding, Content-Length, Trailer, etc in trailers ??
639fn decode_trailers(buf: &mut BytesMut, count: usize) -> Result<HeaderMap, io::Error> {
640    let mut trailers = HeaderMap::new();
641    let mut headers = vec![httparse::EMPTY_HEADER; count];
642    let res = httparse::parse_headers(buf, &mut headers);
643    match res {
644        Ok(httparse::Status::Complete((_, headers))) => {
645            for header in headers {
646                use std::convert::TryFrom;
647                let name = match HeaderName::try_from(header.name) {
648                    Ok(name) => name,
649                    Err(_) => {
650                        return Err(io::Error::new(
651                            io::ErrorKind::InvalidInput,
652                            format!("Invalid header name: {:?}", &header),
653                        ));
654                    }
655                };
656
657                let value = match HeaderValue::from_bytes(header.value) {
658                    Ok(value) => value,
659                    Err(_) => {
660                        return Err(io::Error::new(
661                            io::ErrorKind::InvalidInput,
662                            format!("Invalid header value: {:?}", &header),
663                        ));
664                    }
665                };
666
667                trailers.append(name, value);
668            }
669
670            Ok(trailers)
671        }
672        Ok(httparse::Status::Partial) => Err(io::Error::new(
673            io::ErrorKind::InvalidInput,
674            "Partial header",
675        )),
676        Err(e) => Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
677    }
678}
679
680#[derive(Debug)]
681struct IncompleteBody;
682
683impl fmt::Display for IncompleteBody {
684    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
685        write!(f, "end of file before message length reached")
686    }
687}
688
689impl StdError for IncompleteBody {}
690
691#[cfg(test)]
692mod tests {
693    use super::*;
694    use crate::rt::{Read, ReadBuf};
695    use std::pin::Pin;
696    use std::time::Duration;
697
698    impl MemRead for &[u8] {
699        fn read_mem(&mut self, _: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
700            let n = std::cmp::min(len, self.len());
701            if n > 0 {
702                let (a, b) = self.split_at(n);
703                let buf = Bytes::copy_from_slice(a);
704                *self = b;
705                Poll::Ready(Ok(buf))
706            } else {
707                Poll::Ready(Ok(Bytes::new()))
708            }
709        }
710    }
711
712    impl MemRead for &mut (dyn Read + Unpin) {
713        fn read_mem(&mut self, cx: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
714            let mut v = vec![0; len];
715            let mut buf = ReadBuf::new(&mut v);
716            ready!(Pin::new(self).poll_read(cx, buf.unfilled())?);
717            Poll::Ready(Ok(Bytes::copy_from_slice(buf.filled())))
718        }
719    }
720
721    impl MemRead for Bytes {
722        fn read_mem(&mut self, _: &mut Context<'_>, len: usize) -> Poll<io::Result<Bytes>> {
723            let n = std::cmp::min(len, self.len());
724            let ret = self.split_to(n);
725            Poll::Ready(Ok(ret))
726        }
727    }
728
729    /*
730    use std::io;
731    use std::io::Write;
732    use super::Decoder;
733    use super::ChunkedState;
734    use futures::{Async, Poll};
735    use bytes::{BytesMut, Bytes};
736    use crate::mock::AsyncIo;
737    */
738
739    #[cfg(not(miri))]
740    #[tokio::test]
741    async fn test_read_chunk_size() {
742        use std::io::ErrorKind::{InvalidData, InvalidInput, UnexpectedEof};
743
744        async fn read(s: &str) -> u64 {
745            let mut state = ChunkedState::new();
746            let rdr = &mut s.as_bytes();
747            let mut size = 0;
748            let mut ext_cnt = 0;
749            let mut trailers_cnt = 0;
750            loop {
751                let result = futures_util::future::poll_fn(|cx| {
752                    state.step(
753                        cx,
754                        rdr,
755                        StepArgs {
756                            chunk_size: &mut size,
757                            extensions_cnt: &mut ext_cnt,
758                            chunk_buf: &mut None,
759                            trailers_buf: &mut None,
760                            trailers_cnt: &mut trailers_cnt,
761                            max_headers_cnt: DEFAULT_MAX_HEADERS,
762                            max_headers_bytes: TRAILER_LIMIT,
763                        },
764                    )
765                })
766                .await;
767                let desc = format!("read_size failed for {:?}", s);
768                state = result.expect(&desc);
769                if state == ChunkedState::Body || state == ChunkedState::EndCr {
770                    break;
771                }
772            }
773            size
774        }
775
776        async fn read_err(s: &str, expected_err: io::ErrorKind) {
777            let mut state = ChunkedState::new();
778            let rdr = &mut s.as_bytes();
779            let mut size = 0;
780            let mut ext_cnt = 0;
781            let mut trailers_cnt = 0;
782            loop {
783                let result = futures_util::future::poll_fn(|cx| {
784                    state.step(
785                        cx,
786                        rdr,
787                        StepArgs {
788                            chunk_size: &mut size,
789                            extensions_cnt: &mut ext_cnt,
790                            chunk_buf: &mut None,
791                            trailers_buf: &mut None,
792                            trailers_cnt: &mut trailers_cnt,
793                            max_headers_cnt: DEFAULT_MAX_HEADERS,
794                            max_headers_bytes: TRAILER_LIMIT,
795                        },
796                    )
797                })
798                .await;
799                state = match result {
800                    Ok(s) => s,
801                    Err(e) => {
802                        assert!(
803                            expected_err == e.kind(),
804                            "Reading {:?}, expected {:?}, but got {:?}",
805                            s,
806                            expected_err,
807                            e.kind()
808                        );
809                        return;
810                    }
811                };
812                if state == ChunkedState::Body || state == ChunkedState::End {
813                    panic!("Was Ok. Expected Err for {:?}", s);
814                }
815            }
816        }
817
818        assert_eq!(1, read("1\r\n").await);
819        assert_eq!(1, read("01\r\n").await);
820        assert_eq!(0, read("0\r\n").await);
821        assert_eq!(0, read("00\r\n").await);
822        assert_eq!(10, read("A\r\n").await);
823        assert_eq!(10, read("a\r\n").await);
824        assert_eq!(255, read("Ff\r\n").await);
825        assert_eq!(255, read("Ff   \r\n").await);
826        // Missing LF or CRLF
827        read_err("F\rF", InvalidInput).await;
828        read_err("F", UnexpectedEof).await;
829        // Missing digit
830        read_err("\r\n\r\n", InvalidInput).await;
831        read_err("\r\n", InvalidInput).await;
832        // Invalid hex digit
833        read_err("X\r\n", InvalidInput).await;
834        read_err("1X\r\n", InvalidInput).await;
835        read_err("-\r\n", InvalidInput).await;
836        read_err("-1\r\n", InvalidInput).await;
837        // Acceptable (if not fully valid) extensions do not influence the size
838        assert_eq!(1, read("1;extension\r\n").await);
839        assert_eq!(10, read("a;ext name=value\r\n").await);
840        assert_eq!(1, read("1;extension;extension2\r\n").await);
841        assert_eq!(1, read("1;;;  ;\r\n").await);
842        assert_eq!(2, read("2; extension...\r\n").await);
843        assert_eq!(3, read("3   ; extension=123\r\n").await);
844        assert_eq!(3, read("3   ;\r\n").await);
845        assert_eq!(3, read("3   ;   \r\n").await);
846        // Invalid extensions cause an error
847        read_err("1 invalid extension\r\n", InvalidInput).await;
848        read_err("1 A\r\n", InvalidInput).await;
849        read_err("1;no CRLF", UnexpectedEof).await;
850        read_err("1;reject\nnewlines\r\n", InvalidData).await;
851        // Overflow
852        read_err("f0000000000000003\r\n", InvalidData).await;
853    }
854
855    #[cfg(not(miri))]
856    #[tokio::test]
857    async fn test_read_sized_early_eof() {
858        let mut bytes = &b"foo bar"[..];
859        let mut decoder = Decoder::length(10);
860        assert_eq!(
861            decoder
862                .decode_fut(&mut bytes)
863                .await
864                .unwrap()
865                .data_ref()
866                .unwrap()
867                .len(),
868            7
869        );
870        let e = decoder.decode_fut(&mut bytes).await.unwrap_err();
871        assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
872    }
873
874    #[cfg(not(miri))]
875    #[tokio::test]
876    async fn test_read_chunked_early_eof() {
877        let mut bytes = &b"\
878            9\r\n\
879            foo bar\
880        "[..];
881        let mut decoder = Decoder::chunked(None, None);
882        assert_eq!(
883            decoder
884                .decode_fut(&mut bytes)
885                .await
886                .unwrap()
887                .data_ref()
888                .unwrap()
889                .len(),
890            7
891        );
892        let e = decoder.decode_fut(&mut bytes).await.unwrap_err();
893        assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
894    }
895
896    #[cfg(not(miri))]
897    #[tokio::test]
898    async fn test_read_chunked_single_read() {
899        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n"[..];
900        let buf = Decoder::chunked(None, None)
901            .decode_fut(&mut mock_buf)
902            .await
903            .expect("decode")
904            .into_data()
905            .expect("unknown frame type");
906        assert_eq!(16, buf.len());
907        let result = String::from_utf8(buf.as_ref().to_vec()).expect("decode String");
908        assert_eq!("1234567890abcdef", &result);
909    }
910
911    #[tokio::test]
912    async fn test_read_chunked_with_missing_zero_digit() {
913        // After reading a valid chunk, the ending is missing a zero.
914        let mut mock_buf = &b"1\r\nZ\r\n\r\n\r\n"[..];
915        let mut decoder = Decoder::chunked(None, None);
916        let buf = decoder
917            .decode_fut(&mut mock_buf)
918            .await
919            .expect("decode")
920            .into_data()
921            .expect("unknown frame type");
922        assert_eq!("Z", buf);
923
924        let err = decoder
925            .decode_fut(&mut mock_buf)
926            .await
927            .expect_err("decode 2");
928        assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
929    }
930
931    #[tokio::test]
932    async fn test_read_chunked_extensions_over_limit() {
933        // construct a chunked body where each individual chunked extension
934        // is totally fine, but combined is over the limit.
935        let per_chunk = super::CHUNKED_EXTENSIONS_LIMIT * 2 / 3;
936        let mut scratch = vec![];
937        for _ in 0..2 {
938            scratch.extend(b"1;");
939            scratch.extend(b"x".repeat(per_chunk as usize));
940            scratch.extend(b"\r\nA\r\n");
941        }
942        scratch.extend(b"0\r\n\r\n");
943        let mut mock_buf = Bytes::from(scratch);
944
945        let mut decoder = Decoder::chunked(None, None);
946        let buf1 = decoder
947            .decode_fut(&mut mock_buf)
948            .await
949            .expect("decode1")
950            .into_data()
951            .expect("unknown frame type");
952        assert_eq!(&buf1[..], b"A");
953
954        let err = decoder
955            .decode_fut(&mut mock_buf)
956            .await
957            .expect_err("decode2");
958        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
959        assert_eq!(err.to_string(), "chunk extensions over limit");
960    }
961
962    #[cfg(not(miri))]
963    #[tokio::test]
964    async fn test_read_chunked_trailer_with_missing_lf() {
965        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\nbad\r\r\n"[..];
966        let mut decoder = Decoder::chunked(None, None);
967        decoder.decode_fut(&mut mock_buf).await.expect("decode");
968        let e = decoder.decode_fut(&mut mock_buf).await.unwrap_err();
969        assert_eq!(e.kind(), io::ErrorKind::InvalidInput);
970    }
971
972    #[cfg(not(miri))]
973    #[tokio::test]
974    async fn test_read_chunked_after_eof() {
975        let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n\r\n"[..];
976        let mut decoder = Decoder::chunked(None, None);
977
978        // normal read
979        let buf = decoder
980            .decode_fut(&mut mock_buf)
981            .await
982            .unwrap()
983            .into_data()
984            .expect("unknown frame type");
985        assert_eq!(16, buf.len());
986        let result = String::from_utf8(buf.as_ref().to_vec()).expect("decode String");
987        assert_eq!("1234567890abcdef", &result);
988
989        // eof read
990        let buf = decoder
991            .decode_fut(&mut mock_buf)
992            .await
993            .expect("decode")
994            .into_data()
995            .expect("unknown frame type");
996        assert_eq!(0, buf.len());
997
998        // ensure read after eof also returns eof
999        let buf = decoder
1000            .decode_fut(&mut mock_buf)
1001            .await
1002            .expect("decode")
1003            .into_data()
1004            .expect("unknown frame type");
1005        assert_eq!(0, buf.len());
1006    }
1007
1008    // perform an async read using a custom buffer size and causing a blocking
1009    // read at the specified byte
1010    async fn read_async(mut decoder: Decoder, content: &[u8], block_at: usize) -> String {
1011        let mut outs = Vec::new();
1012
1013        let mut ins = crate::common::io::Compat::new(if block_at == 0 {
1014            tokio_test::io::Builder::new()
1015                .wait(Duration::from_millis(10))
1016                .read(content)
1017                .build()
1018        } else {
1019            tokio_test::io::Builder::new()
1020                .read(&content[..block_at])
1021                .wait(Duration::from_millis(10))
1022                .read(&content[block_at..])
1023                .build()
1024        });
1025
1026        let mut ins = &mut ins as &mut (dyn Read + Unpin);
1027
1028        loop {
1029            let buf = decoder
1030                .decode_fut(&mut ins)
1031                .await
1032                .expect("unexpected decode error")
1033                .into_data()
1034                .expect("unexpected frame type");
1035            if buf.is_empty() {
1036                break; // eof
1037            }
1038            outs.extend(buf.as_ref());
1039        }
1040
1041        String::from_utf8(outs).expect("decode String")
1042    }
1043
1044    // iterate over the different ways that this async read could go.
1045    // tests blocking a read at each byte along the content - The shotgun approach
1046    async fn all_async_cases(content: &str, expected: &str, decoder: Decoder) {
1047        let content_len = content.len();
1048        for block_at in 0..content_len {
1049            let actual = read_async(decoder.clone(), content.as_bytes(), block_at).await;
1050            assert_eq!(expected, &actual) //, "Failed async. Blocking at {}", block_at);
1051        }
1052    }
1053
1054    #[cfg(not(miri))]
1055    #[tokio::test]
1056    async fn test_read_length_async() {
1057        let content = "foobar";
1058        all_async_cases(content, content, Decoder::length(content.len() as u64)).await;
1059    }
1060
1061    #[cfg(not(miri))]
1062    #[tokio::test]
1063    async fn test_read_chunked_async() {
1064        let content = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n";
1065        let expected = "foobar";
1066        all_async_cases(content, expected, Decoder::chunked(None, None)).await;
1067    }
1068
1069    #[cfg(not(miri))]
1070    #[tokio::test]
1071    async fn test_read_eof_async() {
1072        let content = "foobar";
1073        all_async_cases(content, content, Decoder::eof()).await;
1074    }
1075
1076    #[cfg(all(feature = "nightly", not(miri)))]
1077    #[bench]
1078    fn bench_decode_chunked_1kb(b: &mut test::Bencher) {
1079        let rt = new_runtime();
1080
1081        const LEN: usize = 1024;
1082        let mut vec = Vec::new();
1083        vec.extend(format!("{:x}\r\n", LEN).as_bytes());
1084        vec.extend(&[0; LEN][..]);
1085        vec.extend(b"\r\n");
1086        let content = Bytes::from(vec);
1087
1088        b.bytes = LEN as u64;
1089
1090        b.iter(|| {
1091            let mut decoder = Decoder::chunked(None, None);
1092            rt.block_on(async {
1093                let mut raw = content.clone();
1094                let chunk = decoder
1095                    .decode_fut(&mut raw)
1096                    .await
1097                    .unwrap()
1098                    .into_data()
1099                    .unwrap();
1100                assert_eq!(chunk.len(), LEN);
1101            });
1102        });
1103    }
1104
1105    #[cfg(all(feature = "nightly", not(miri)))]
1106    #[bench]
1107    fn bench_decode_length_1kb(b: &mut test::Bencher) {
1108        let rt = new_runtime();
1109
1110        const LEN: usize = 1024;
1111        let content = Bytes::from(&[0; LEN][..]);
1112        b.bytes = LEN as u64;
1113
1114        b.iter(|| {
1115            let mut decoder = Decoder::length(LEN as u64);
1116            rt.block_on(async {
1117                let mut raw = content.clone();
1118                let chunk = decoder
1119                    .decode_fut(&mut raw)
1120                    .await
1121                    .unwrap()
1122                    .into_data()
1123                    .unwrap();
1124                assert_eq!(chunk.len(), LEN);
1125            });
1126        });
1127    }
1128
1129    #[cfg(feature = "nightly")]
1130    fn new_runtime() -> tokio::runtime::Runtime {
1131        tokio::runtime::Builder::new_current_thread()
1132            .enable_all()
1133            .build()
1134            .expect("rt build")
1135    }
1136
1137    #[test]
1138    fn test_decode_trailers() {
1139        let mut buf = BytesMut::new();
1140        buf.extend_from_slice(
1141            b"Expires: Wed, 21 Oct 2015 07:28:00 GMT\r\nX-Stream-Error: failed to decode\r\n\r\n",
1142        );
1143        let headers = decode_trailers(&mut buf, 2).expect("decode_trailers");
1144        assert_eq!(headers.len(), 2);
1145        assert_eq!(
1146            headers.get("Expires").unwrap(),
1147            "Wed, 21 Oct 2015 07:28:00 GMT"
1148        );
1149        assert_eq!(headers.get("X-Stream-Error").unwrap(), "failed to decode");
1150    }
1151
1152    #[test]
1153    fn test_decode_trailers_preserves_duplicate_values() {
1154        let mut buf = BytesMut::new();
1155        buf.extend_from_slice(b"X-Trace: first\r\nX-Trace: second\r\n\r\n");
1156
1157        let headers = decode_trailers(&mut buf, 2).expect("decode_trailers");
1158        let values = headers
1159            .get_all("X-Trace")
1160            .iter()
1161            .map(|value| value.to_str().unwrap())
1162            .collect::<Vec<_>>();
1163
1164        assert_eq!(values, ["first", "second"]);
1165    }
1166
1167    #[tokio::test]
1168    async fn test_trailer_max_headers_enforced() {
1169        let h1_max_headers = 10;
1170        let mut scratch = vec![];
1171        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1172        for i in 0..=h1_max_headers {
1173            scratch.extend(format!("trailer{}: {}\r\n", i, i).as_bytes());
1174        }
1175        scratch.extend(b"\r\n");
1176        let mut mock_buf = Bytes::from(scratch);
1177
1178        let mut decoder = Decoder::chunked(Some(h1_max_headers), None);
1179
1180        // ready chunked body
1181        let buf = decoder
1182            .decode_fut(&mut mock_buf)
1183            .await
1184            .unwrap()
1185            .into_data()
1186            .expect("unknown frame type");
1187        assert_eq!(16, buf.len());
1188
1189        // eof read
1190        let err = decoder
1191            .decode_fut(&mut mock_buf)
1192            .await
1193            .expect_err("trailer fields over limit");
1194        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1195    }
1196
1197    #[tokio::test]
1198    async fn test_trailer_max_headers_allows_exact_limit() {
1199        let h1_max_headers = 10;
1200        let mut scratch = vec![];
1201        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1202        for i in 0..h1_max_headers {
1203            scratch.extend(format!("trailer{}: {}\r\n", i, i).as_bytes());
1204        }
1205        scratch.extend(b"\r\n");
1206        let mut mock_buf = Bytes::from(scratch);
1207
1208        let mut decoder = Decoder::chunked(Some(h1_max_headers), None);
1209
1210        let buf = decoder
1211            .decode_fut(&mut mock_buf)
1212            .await
1213            .unwrap()
1214            .into_data()
1215            .expect("unknown frame type");
1216        assert_eq!(16, buf.len());
1217
1218        let trailers = decoder
1219            .decode_fut(&mut mock_buf)
1220            .await
1221            .expect("trailers at limit")
1222            .into_trailers()
1223            .expect("unknown frame type");
1224        assert_eq!(trailers.len(), h1_max_headers);
1225    }
1226
1227    #[tokio::test]
1228    async fn test_trailer_max_header_size_huge_trailer() {
1229        let max_header_size = 1024;
1230        let mut scratch = vec![];
1231        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1232        scratch.extend(format!("huge_trailer: {}\r\n", "x".repeat(max_header_size)).as_bytes());
1233        scratch.extend(b"\r\n");
1234        let mut mock_buf = Bytes::from(scratch);
1235
1236        let mut decoder = Decoder::chunked(None, Some(max_header_size));
1237
1238        // ready chunked body
1239        let buf = decoder
1240            .decode_fut(&mut mock_buf)
1241            .await
1242            .unwrap()
1243            .into_data()
1244            .expect("unknown frame type");
1245        assert_eq!(16, buf.len());
1246
1247        // eof read
1248        let err = decoder
1249            .decode_fut(&mut mock_buf)
1250            .await
1251            .expect_err("trailers over limit");
1252        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1253    }
1254
1255    #[tokio::test]
1256    async fn test_trailer_max_header_size_many_small_trailers() {
1257        let max_headers = 10;
1258        let header_size = 64;
1259        let mut scratch = vec![];
1260        scratch.extend(b"10\r\n1234567890abcdef\r\n0\r\n");
1261
1262        for i in 0..max_headers {
1263            scratch.extend(format!("trailer{}: {}\r\n", i, "x".repeat(header_size)).as_bytes());
1264        }
1265
1266        scratch.extend(b"\r\n");
1267        let mut mock_buf = Bytes::from(scratch);
1268
1269        let mut decoder = Decoder::chunked(None, Some(max_headers * header_size));
1270
1271        // ready chunked body
1272        let buf = decoder
1273            .decode_fut(&mut mock_buf)
1274            .await
1275            .unwrap()
1276            .into_data()
1277            .expect("unknown frame type");
1278        assert_eq!(16, buf.len());
1279
1280        // eof read
1281        let err = decoder
1282            .decode_fut(&mut mock_buf)
1283            .await
1284            .expect_err("trailers over limit");
1285        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1286    }
1287}
hyper/proto/h1/decode.rs

hyper/proto/h1/
decode.rs
