base64/engine/
mod.rs

1//! Provides the [Engine] abstraction and out of the box implementations.
2use crate::alphabet::Symbol;
3#[cfg(any(feature = "alloc", test))]
4use crate::chunked_encoder;
5use crate::{
6    encode::{encode_with_padding, EncodeSliceError},
7    encoded_len, DecodeError, DecodeSliceError,
8};
9#[cfg(any(feature = "alloc", test))]
10use alloc::vec::Vec;
11
12#[cfg(any(feature = "alloc", test))]
13use alloc::{string::String, vec};
14
15pub mod general_purpose;
16
17#[cfg(all(
18    feature = "simd-unsafe",
19    any(
20        target_arch = "x86_64",
21        all(target_arch = "aarch64", target_feature = "neon")
22    )
23))]
24pub mod simd;
25
26#[cfg(test)]
27mod naive;
28
29#[cfg(test)]
30mod tests;
31
32pub use general_purpose::{GeneralPurpose, GeneralPurposeConfig, Scalar};
33
34/// The runtime-detected SIMD engine. Requires the `simd-unsafe` feature.
35#[cfg(all(
36    feature = "simd-unsafe",
37    feature = "std",
38    any(
39        target_arch = "x86_64",
40        all(target_arch = "aarch64", target_feature = "neon")
41    )
42))]
43pub use simd::Simd;
44
45/// The AVX2 engine. Requires the `simd-unsafe` feature on an `x86_64` target.
46#[cfg(all(feature = "simd-unsafe", target_arch = "x86_64"))]
47pub use simd::Avx2;
48
49/// The NEON engine. Requires the `simd-unsafe` feature on an `aarch64` target.
50#[cfg(all(
51    feature = "simd-unsafe",
52    target_arch = "aarch64",
53    target_feature = "neon"
54))]
55pub use simd::Neon;
56
57/// An `Engine` provides low-level encoding and decoding operations that all other higher-level parts of the API use. Users of the library will generally not need to implement this.
58///
59/// Different implementations offer different characteristics. The library currently ships with
60/// [`GeneralPurpose`] that offers good speed and works on any CPU, with more choices
61/// coming later, like a constant-time one when side channel resistance is called for, and vendor-specific vectorized ones for more speed.
62///
63/// See [`general_purpose::STANDARD_NO_PAD`] if you just want standard base64. Otherwise, when possible, it's
64/// recommended to store the engine in a `const` so that references to it won't pose any lifetime
65/// issues, and to avoid repeating the cost of engine setup.
66///
67/// Since almost nobody will need to implement `Engine`, docs for internal methods are hidden.
68// When adding an implementation of Engine, include them in the engine test suite:
69// - add an implementation of [engine::tests::EngineWrapper]
70// - add the implementation to the `all_engines` macro
71// All tests run on all engines listed in the macro.
72pub trait Engine: Send + Sync {
73    /// The config type used by this engine
74    type Config: Config;
75    /// The decode estimate used by this engine
76    type DecodeEstimate: DecodeEstimate;
77
78    /// This is not meant to be called directly; it is only for `Engine` implementors.
79    /// See the other `encode*` functions on this trait.
80    ///
81    /// Encode the `input` bytes into the `output` buffer based on the mapping in `encode_table`.
82    ///
83    /// `output` will be long enough to hold the encoded data.
84    ///
85    /// Returns the number of bytes written.
86    ///
87    /// No padding should be written; that is handled separately.
88    ///
89    /// Must not write any bytes into the output slice other than the encoded data.
90    #[doc(hidden)]
91    fn internal_encode(&self, input: &[u8], output: &mut [u8]) -> usize;
92
93    /// This is not meant to be called directly; it is only for `Engine` implementors.
94    ///
95    /// As an optimization to prevent the decoded length from being calculated twice, it is
96    /// sometimes helpful to have a conservative estimate of the decoded size before doing the
97    /// decoding, so this calculation is done separately and passed to [Engine::decode()] as needed.
98    #[doc(hidden)]
99    fn internal_decoded_len_estimate(&self, input_len: usize) -> Self::DecodeEstimate;
100
101    /// This is not meant to be called directly; it is only for `Engine` implementors.
102    /// See the other `decode*` functions on this trait.
103    ///
104    /// Decode `input` base64 bytes into the `output` buffer.
105    ///
106    /// `decode_estimate` is the result of [Engine::internal_decoded_len_estimate()], which is passed in to avoid
107    /// calculating it again (expensive on short inputs).`
108    ///
109    /// Each complete 4-byte chunk of encoded data decodes to 3 bytes of decoded data, but this
110    /// function must also handle the final possibly partial chunk.
111    /// If the input length is not a multiple of 4, or uses padding bytes to reach a multiple of 4,
112    /// the trailing 2 or 3 bytes must decode to 1 or 2 bytes, respectively, as per the
113    /// [RFC](https://tools.ietf.org/html/rfc4648#section-3.5).
114    ///
115    /// Decoding must not write any bytes into the output slice other than the decoded data.
116    ///
117    /// Non-canonical trailing bits in the final symbols or non-canonical padding must be reported as
118    /// errors unless the engine is configured otherwise.
119    #[doc(hidden)]
120    fn internal_decode(
121        &self,
122        input: &[u8],
123        output: &mut [u8],
124        decode_estimate: Self::DecodeEstimate,
125    ) -> Result<DecodeMetadata, DecodeSliceError>;
126
127    /// Returns the config for this engine.
128    fn config(&self) -> &Self::Config;
129
130    /// Encode arbitrary octets as base64 using the provided `Engine`.
131    /// Returns a `String`.
132    ///
133    /// # Example
134    ///
135    /// ```rust
136    /// use base64::{Engine as _, engine::{self, general_purpose}, alphabet};
137    ///
138    /// let b64 = general_purpose::STANDARD.encode(b"hello world~");
139    /// println!("{}", b64);
140    ///
141    /// const CUSTOM_ENGINE: engine::GeneralPurpose =
142    ///     engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
143    ///
144    /// let b64_url = CUSTOM_ENGINE.encode(b"hello internet~");
145    /// ```
146    #[cfg(any(feature = "alloc", test))]
147    #[inline]
148    fn encode<T: AsRef<[u8]>>(&self, input: T) -> String {
149        fn inner<E>(engine: &E, input_bytes: &[u8]) -> String
150        where
151            E: Engine + ?Sized,
152        {
153            let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding())
154                .expect("integer overflow when calculating buffer size");
155
156            let mut buf = vec![0; encoded_size];
157
158            encode_with_padding(input_bytes, &mut buf[..], engine, encoded_size);
159
160            String::from_utf8(buf).expect("Invalid UTF8")
161        }
162
163        inner(self, input.as_ref())
164    }
165
166    /// Encode arbitrary octets as base64 into a supplied `String`.
167    /// Writes into the supplied `String`, which may allocate if its internal buffer isn't big enough.
168    ///
169    /// # Example
170    ///
171    /// ```rust
172    /// use base64::{Engine as _, engine::{self, general_purpose}, alphabet};
173    /// const CUSTOM_ENGINE: engine::GeneralPurpose =
174    ///     engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
175    ///
176    /// fn main() {
177    ///     let mut buf = String::new();
178    ///     general_purpose::STANDARD.encode_string(b"hello world~", &mut buf);
179    ///     println!("{}", buf);
180    ///
181    ///     buf.clear();
182    ///     CUSTOM_ENGINE.encode_string(b"hello internet~", &mut buf);
183    ///     println!("{}", buf);
184    /// }
185    /// ```
186    #[cfg(any(feature = "alloc", test))]
187    #[inline]
188    fn encode_string<T: AsRef<[u8]>>(&self, input: T, output_buf: &mut String) {
189        fn inner<E>(engine: &E, input_bytes: &[u8], output_buf: &mut String)
190        where
191            E: Engine + ?Sized,
192        {
193            let mut sink = chunked_encoder::StringSink::new(output_buf);
194
195            chunked_encoder::ChunkedEncoder::new(engine)
196                .encode(input_bytes, &mut sink)
197                .expect("Writing to a String shouldn't fail");
198        }
199
200        inner(self, input.as_ref(), output_buf);
201    }
202
203    /// Encode arbitrary octets as base64 into a supplied slice.
204    /// Writes into the supplied output buffer.
205    ///
206    /// This is useful if you wish to avoid allocation entirely (e.g. encoding into a stack-resident
207    /// or statically-allocated buffer).
208    ///
209    /// # Example
210    ///
211    #[cfg_attr(feature = "alloc", doc = "```")]
212    #[cfg_attr(not(feature = "alloc"), doc = "```ignore")]
213    /// use base64::{Engine as _, engine::general_purpose};
214    /// let s = b"hello internet!";
215    /// let mut buf = Vec::new();
216    /// // make sure we'll have a slice big enough for base64 + padding
217    /// buf.resize(s.len() * 4 / 3 + 4, 0);
218    ///
219    /// let bytes_written = general_purpose::STANDARD.encode_slice(s, &mut buf).unwrap();
220    ///
221    /// // shorten our vec down to just what was written
222    /// buf.truncate(bytes_written);
223    ///
224    /// assert_eq!(s, general_purpose::STANDARD.decode(&buf).unwrap().as_slice());
225    /// ```
226    #[inline]
227    fn encode_slice<T: AsRef<[u8]>>(
228        &self,
229        input: T,
230        output_buf: &mut [u8],
231    ) -> Result<usize, EncodeSliceError> {
232        fn inner<E>(
233            engine: &E,
234            input_bytes: &[u8],
235            output_buf: &mut [u8],
236        ) -> Result<usize, EncodeSliceError>
237        where
238            E: Engine + ?Sized,
239        {
240            let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding())
241                .expect("usize overflow when calculating buffer size");
242
243            if output_buf.len() < encoded_size {
244                return Err(EncodeSliceError::OutputSliceTooSmall);
245            }
246
247            let b64_output = &mut output_buf[0..encoded_size];
248
249            encode_with_padding(input_bytes, b64_output, engine, encoded_size);
250
251            Ok(encoded_size)
252        }
253
254        inner(self, input.as_ref(), output_buf)
255    }
256
257    /// Decode the input into a new `Vec`.
258    ///
259    /// # Example
260    ///
261    /// ```rust
262    /// use base64::{Engine as _, alphabet, engine::{self, general_purpose}};
263    ///
264    /// let bytes = general_purpose::STANDARD
265    ///     .decode("aGVsbG8gd29ybGR+Cg==").unwrap();
266    /// println!("{:?}", bytes);
267    ///
268    /// // custom engine setup
269    /// let bytes_url = engine::GeneralPurpose::new(
270    ///              &alphabet::URL_SAFE,
271    ///              general_purpose::NO_PAD)
272    ///     .decode("aGVsbG8gaW50ZXJuZXR-Cg").unwrap();
273    /// println!("{:?}", bytes_url);
274    /// ```
275    #[cfg(any(feature = "alloc", test))]
276    #[inline]
277    fn decode<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, DecodeError> {
278        fn inner<E>(engine: &E, input_bytes: &[u8]) -> Result<Vec<u8>, DecodeError>
279        where
280            E: Engine + ?Sized,
281        {
282            let estimate = engine.internal_decoded_len_estimate(input_bytes.len());
283            let mut buffer = vec![0; estimate.decoded_len_estimate()];
284
285            let bytes_written = engine
286                .internal_decode(input_bytes, &mut buffer, estimate)
287                .map_err(|e| match e {
288                    DecodeSliceError::DecodeError(e) => e,
289                    DecodeSliceError::OutputSliceTooSmall => {
290                        unreachable!("Vec is sized conservatively")
291                    }
292                })?
293                .decoded_len;
294
295            buffer.truncate(bytes_written);
296
297            Ok(buffer)
298        }
299
300        inner(self, input.as_ref())
301    }
302
303    /// Decode the `input` into the supplied `buffer`.
304    ///
305    /// Writes into the supplied `Vec`, which may allocate if its internal buffer isn't big enough.
306    /// Returns a `Result` containing an empty tuple, aka `()`.
307    ///
308    /// # Example
309    ///
310    /// ```rust
311    /// use base64::{Engine as _, alphabet, engine::{self, general_purpose}};
312    /// const CUSTOM_ENGINE: engine::GeneralPurpose =
313    ///     engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::PAD);
314    ///
315    /// fn main() {
316    ///     use base64::Engine;
317    ///     let mut buffer = Vec::<u8>::new();
318    ///     // with the default engine
319    ///     general_purpose::STANDARD
320    ///         .decode_vec("aGVsbG8gd29ybGR+Cg==", &mut buffer,).unwrap();
321    ///     println!("{:?}", buffer);
322    ///
323    ///     buffer.clear();
324    ///
325    ///     // with a custom engine
326    ///     CUSTOM_ENGINE.decode_vec(
327    ///         "aGVsbG8gaW50ZXJuZXR-Cg==",
328    ///         &mut buffer,
329    ///     ).unwrap();
330    ///     println!("{:?}", buffer);
331    /// }
332    /// ```
333    #[cfg(any(feature = "alloc", test))]
334    #[inline]
335    fn decode_vec<T: AsRef<[u8]>>(
336        &self,
337        input: T,
338        buffer: &mut Vec<u8>,
339    ) -> Result<(), DecodeError> {
340        fn inner<E>(engine: &E, input_bytes: &[u8], buffer: &mut Vec<u8>) -> Result<(), DecodeError>
341        where
342            E: Engine + ?Sized,
343        {
344            let starting_output_len = buffer.len();
345            let estimate = engine.internal_decoded_len_estimate(input_bytes.len());
346
347            let total_len_estimate = estimate
348                .decoded_len_estimate()
349                .checked_add(starting_output_len)
350                .expect("Overflow when calculating output buffer length");
351
352            buffer.resize(total_len_estimate, 0);
353
354            let buffer_slice = &mut buffer.as_mut_slice()[starting_output_len..];
355
356            let bytes_written = engine
357                .internal_decode(input_bytes, buffer_slice, estimate)
358                .map_err(|e| match e {
359                    DecodeSliceError::DecodeError(e) => e,
360                    DecodeSliceError::OutputSliceTooSmall => {
361                        unreachable!("Vec is sized conservatively")
362                    }
363                })?
364                .decoded_len;
365
366            buffer.truncate(starting_output_len + bytes_written);
367
368            Ok(())
369        }
370
371        inner(self, input.as_ref(), buffer)
372    }
373
374    /// Decode the input into the provided output slice.
375    ///
376    /// Returns the number of bytes written to the slice, or an error if `output` is smaller than
377    /// the estimated decoded length.
378    ///
379    /// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end).
380    ///
381    /// See [`crate::decoded_len_estimate`] for calculating buffer sizes.
382    ///
383    /// See [`Engine::decode_slice_unchecked`] for a version that panics instead of returning an error
384    /// if the output buffer is too small.
385    #[inline]
386    fn decode_slice<T: AsRef<[u8]>>(
387        &self,
388        input: T,
389        output: &mut [u8],
390    ) -> Result<usize, DecodeSliceError> {
391        fn inner<E>(
392            engine: &E,
393            input_bytes: &[u8],
394            output: &mut [u8],
395        ) -> Result<usize, DecodeSliceError>
396        where
397            E: Engine + ?Sized,
398        {
399            engine
400                .internal_decode(
401                    input_bytes,
402                    output,
403                    engine.internal_decoded_len_estimate(input_bytes.len()),
404                )
405                .map(|dm| dm.decoded_len)
406        }
407
408        inner(self, input.as_ref(), output)
409    }
410
411    /// Decode the input into the provided output slice.
412    ///
413    /// Returns the number of bytes written to the slice.
414    ///
415    /// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end).
416    ///
417    /// See [`crate::decoded_len_estimate`] for calculating buffer sizes.
418    ///
419    /// See [`Engine::decode_slice`] for a version that returns an error instead of panicking if the output
420    /// buffer is too small.
421    ///
422    /// # Panics
423    ///
424    /// Panics if the provided output buffer is too small for the decoded data.
425    #[inline]
426    fn decode_slice_unchecked<T: AsRef<[u8]>>(
427        &self,
428        input: T,
429        output: &mut [u8],
430    ) -> Result<usize, DecodeError> {
431        fn inner<E>(engine: &E, input_bytes: &[u8], output: &mut [u8]) -> Result<usize, DecodeError>
432        where
433            E: Engine + ?Sized,
434        {
435            engine
436                .internal_decode(
437                    input_bytes,
438                    output,
439                    engine.internal_decoded_len_estimate(input_bytes.len()),
440                )
441                .map(|dm| dm.decoded_len)
442                .map_err(|e| match e {
443                    DecodeSliceError::DecodeError(e) => e,
444                    DecodeSliceError::OutputSliceTooSmall => {
445                        panic!("Output slice is too small")
446                    }
447                })
448        }
449
450        inner(self, input.as_ref(), output)
451    }
452
453    /// Returns the symbol used for encode padding.
454    ///
455    /// Typically this is `'='`, but weird alphabets may use other values.
456    fn padding(&self) -> Symbol;
457}
458
459/// The minimal level of configuration that engines must support.
460pub trait Config {
461    /// Returns `true` if padding should be added after the encoded output.
462    ///
463    /// Padding is added outside the engine's `encode()` since the engine may be used
464    /// to encode only a chunk of the overall output, so it can't always know when
465    /// the output is "done" and would therefore need padding (if configured).
466    // It could be provided as a separate parameter when encoding, but that feels like
467    // leaking an implementation detail to the user, and it's hopefully more convenient
468    // to have to only pass one thing (the engine) to any part of the API.
469    fn encode_padding(&self) -> bool;
470}
471
472/// The decode estimate used by an engine implementation. Users do not need to interact with this;
473/// it is only for engine implementors.
474///
475/// Implementors may store relevant data here when constructing this to avoid having to calculate
476/// them again during actual decoding.
477pub trait DecodeEstimate {
478    /// Returns a conservative (err on the side of too big) estimate of the decoded length to use
479    /// for pre-allocating buffers, etc.
480    ///
481    /// The estimate must be no larger than the next largest complete triple of decoded bytes.
482    /// That is, the final quad of symbols to decode may be assumed to be complete with no padding.
483    fn decoded_len_estimate(&self) -> usize;
484}
485
486/// Controls how pad bytes are handled when decoding.
487///
488/// Each [Engine] must support at least the behavior indicated by
489/// [`DecodePaddingMode::RequireCanonical`], and may support other modes.
490#[derive(Clone, Copy, Debug, PartialEq, Eq)]
491pub enum DecodePaddingMode {
492    /// Canonical padding is allowed, but any fewer padding bytes than that is also allowed.
493    Indifferent,
494    /// Padding must be canonical (0, 1, or 2 `=` as needed to produce a 4 byte suffix).
495    RequireCanonical,
496    /// Padding must be absent -- for when you want predictable padding, without any wasted bytes.
497    RequireNone,
498}
499
500/// Metadata about the result of a decode operation
501#[derive(PartialEq, Eq, Debug)]
502pub struct DecodeMetadata {
503    /// Number of decoded bytes output
504    pub(crate) decoded_len: usize,
505    /// Offset of the first padding byte in the input, if any
506    pub(crate) padding_offset: Option<usize>,
507}
508
509impl DecodeMetadata {
510    pub(crate) fn new(decoded_bytes: usize, padding_index: Option<usize>) -> Self {
511        Self {
512            decoded_len: decoded_bytes,
513            padding_offset: padding_index,
514        }
515    }
516}
base64/engine/mod.rs

base64/engine/
mod.rs
