net/
http_cache.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5#![deny(missing_docs)]
6
7//! A memory cache implementing the logic specified in <http://tools.ietf.org/html/rfc7234>
8//! and <http://tools.ietf.org/html/rfc7232>.
9
10use std::ops::Bound;
11use std::sync::Arc as StdArc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::{Duration, Instant, SystemTime};
14
15use headers::{
16    CacheControl, ContentRange, Expires, HeaderMapExt, LastModified, Pragma, Range, Vary,
17};
18use http::{HeaderMap, Method, StatusCode, header};
19use log::{debug, error};
20use malloc_size_of::{MallocSizeOf, MallocSizeOfOps};
21use malloc_size_of_derive::MallocSizeOf;
22use net_traits::http_status::HttpStatus;
23use net_traits::request::{CacheMode, Request};
24use net_traits::response::{Response, ResponseBody};
25use net_traits::{CacheEntryDescriptor, FetchMetadata, Metadata, ResourceFetchTiming};
26use parking_lot::Mutex as ParkingLotMutex;
27use quick_cache::sync::{Cache, DefaultLifecycle, PlaceholderGuard};
28use quick_cache::{DefaultHashBuilder, UnitWeighter};
29use servo_arc::Arc;
30use servo_config::pref;
31use servo_url::ServoUrl;
32use tokio::sync::mpsc::{UnboundedSender as TokioSender, unbounded_channel as unbounded};
33use tokio::sync::{OwnedRwLockWriteGuard, RwLock as TokioRwLock};
34
35use crate::fetch::methods::{Data, DoneChannel};
36
37/// The key used to differentiate requests in the cache.
38#[derive(Clone, Eq, Hash, MallocSizeOf, PartialEq)]
39pub struct CacheKey {
40    url: ServoUrl,
41}
42
43impl CacheKey {
44    /// Create a cache-key from a request.
45    pub fn new(request: &Request) -> CacheKey {
46        CacheKey {
47            url: request.current_url(),
48        }
49    }
50
51    /// Create a cache-key from a resolved URL.
52    pub fn from_url(url: ServoUrl) -> CacheKey {
53        CacheKey { url }
54    }
55}
56
57/// A complete cached resource.
58#[derive(Clone, MallocSizeOf)]
59pub struct CachedResource {
60    #[conditional_malloc_size_of]
61    request_headers: Arc<ParkingLotMutex<HeaderMap>>,
62    #[conditional_malloc_size_of]
63    body: Arc<ParkingLotMutex<ResponseBody>>,
64    #[conditional_malloc_size_of]
65    aborted: Arc<AtomicBool>,
66    #[conditional_malloc_size_of]
67    awaiting_body: Arc<ParkingLotMutex<Vec<TokioSender<Data>>>>,
68    metadata: CachedMetadata,
69    location_url: Option<Result<ServoUrl, String>>,
70    status: HttpStatus,
71    url_list: Vec<ServoUrl>,
72    expires: Duration,
73    stale_while_revalidate: Duration,
74    #[conditional_malloc_size_of]
75    revalidating: StdArc<AtomicBool>,
76    last_validated: Instant,
77}
78
79/// Metadata about a loaded resource, such as is obtained from HTTP headers.
80#[derive(Clone, MallocSizeOf)]
81struct CachedMetadata {
82    /// Headers
83    #[conditional_malloc_size_of]
84    pub headers: Arc<ParkingLotMutex<HeaderMap>>,
85    /// Final URL after redirects.
86    pub final_url: ServoUrl,
87    /// MIME type / subtype.
88    pub content_type: Option<String>,
89    /// Character set.
90    pub charset: Option<String>,
91    /// HTTP Status
92    pub status: HttpStatus,
93}
94
95/// Whether a cached response is fresh or requires validation before or after use.
96#[derive(Clone, Copy, Debug, Eq, PartialEq)]
97pub enum ValidationStatus {
98    /// The response is fresh and can be used without any revalidation.
99    Valid,
100    /// The response is stale.
101    Stale {
102        /// Whether the stale response can be served immediately, leaving the
103        /// caller responsible for revalidating it in the background.
104        revalidate_in_background: bool,
105    },
106}
107
108/// Wrapper around a cached response, including information on re-validation needs
109pub(crate) struct CachedResponse {
110    /// The response constructed from the cached resource
111    pub response: Response,
112    /// Whether the stored response is fresh or stale
113    pub validation_status: ValidationStatus,
114    /// Single-flight guard for the background revalidation.
115    pub revalidation_guard: StdArc<AtomicBool>,
116}
117
118type CacheEntry = std::sync::Arc<TokioRwLock<Vec<CachedResource>>>;
119type QuickCache = Cache<CacheKey, CacheEntry, UnitWeighter>;
120type OurLifecycle = DefaultLifecycle<CacheKey, CacheEntry>;
121type QuickCachePlaceeholderGuard<'a> =
122    PlaceholderGuard<'a, CacheKey, CacheEntry, UnitWeighter, DefaultHashBuilder, OurLifecycle>;
123
124/// A simple memory cache.
125/// Elements will be evicted based on the cache heuristic. We weight elements
126/// by the number of entries per given url. We evict currently a whole url.
127/// The cache makes extensive use of `Arc::unwrap_or_clone or` and `Arc::into_inner`
128/// to modify the cached entries. This is ok because `CachedResource` are cheap to clone
129pub struct HttpCache {
130    /// cached responses.
131    entries: QuickCache,
132}
133
134impl MallocSizeOf for HttpCache {
135    fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
136        self.entries
137            .iter()
138            .map(|(_key, entry)| entry.blocking_read().size_of(ops))
139            .sum()
140    }
141}
142
143impl Default for HttpCache {
144    fn default() -> Self {
145        let size = pref!(network_http_cache_size)
146            .try_into()
147            .expect("http_cache_size needs to fit into u64");
148        Self {
149            entries: Cache::new(size),
150        }
151    }
152}
153
154/// Determine if a response is cacheable by default <https://tools.ietf.org/html/rfc7231#section-6.1>
155fn is_cacheable_by_default(status_code: StatusCode) -> bool {
156    matches!(
157        status_code.as_u16(),
158        200 | 203 | 204 | 206 | 300 | 301 | 404 | 405 | 410 | 414 | 501
159    )
160}
161
162/// Determine if a given response is cacheable.
163/// Based on <https://tools.ietf.org/html/rfc7234#section-3>
164fn response_is_cacheable(metadata: &Metadata) -> bool {
165    // TODO: if we determine that this cache should be considered shared:
166    // 1. check for absence of private response directive <https://tools.ietf.org/html/rfc7234#section-5.2.2.6>
167    // 2. check for absence of the Authorization header field.
168    let mut is_cacheable = false;
169    let headers = metadata.headers.as_ref().unwrap();
170    if headers.contains_key(header::EXPIRES) ||
171        headers.contains_key(header::LAST_MODIFIED) ||
172        headers.contains_key(header::ETAG)
173    {
174        is_cacheable = true;
175    }
176    if let Some(ref directive) = headers.typed_get::<CacheControl>() {
177        if directive.no_store() {
178            return false;
179        }
180        if directive.public() ||
181            directive.s_max_age().is_some() ||
182            directive.max_age().is_some() ||
183            directive.no_cache()
184        {
185            // If cache-control is understood, we can use it and ignore pragma.
186            return true;
187        }
188    }
189    if let Some(pragma) = headers.typed_get::<Pragma>() &&
190        pragma.is_no_cache()
191    {
192        return false;
193    }
194    is_cacheable
195}
196
197/// Calculating Age
198/// <https://tools.ietf.org/html/rfc7234#section-4.2.3>
199fn calculate_response_age(response: &Response) -> Duration {
200    // TODO: follow the spec more closely (Date headers, request/response lag, ...)
201    response
202        .headers
203        .get(header::AGE)
204        .and_then(|age_header| age_header.to_str().ok())
205        .and_then(|age_string| age_string.parse::<u64>().ok())
206        .map(Duration::from_secs)
207        .unwrap_or_default()
208}
209
210/// Determine the expiry date from relevant headers,
211/// or uses a heuristic if none are present.
212fn get_response_expiry(response: &Response) -> Duration {
213    // Calculating Freshness Lifetime <https://tools.ietf.org/html/rfc7234#section-4.2.1>
214    let age = calculate_response_age(response);
215    let now = SystemTime::now();
216    if let Some(directives) = response.headers.typed_get::<CacheControl>() {
217        if directives.no_cache() {
218            // Requires validation on first use.
219            return Duration::ZERO;
220        }
221        if let Some(max_age) = directives.max_age().or(directives.s_max_age()) {
222            return max_age.saturating_sub(age);
223        }
224    }
225    match response.headers.typed_get::<Expires>() {
226        Some(expiry) => {
227            // `duration_since` fails if `now` is later than `expiry_time` in which case,
228            // this whole thing return `Duration::ZERO`.
229            let expiry_time: SystemTime = expiry.into();
230            return expiry_time.duration_since(now).unwrap_or(Duration::ZERO);
231        },
232        // Malformed Expires header, shouldn't be used to construct a valid response.
233        None if response.headers.contains_key(header::EXPIRES) => return Duration::ZERO,
234        _ => {},
235    }
236    // Calculating Heuristic Freshness
237    // <https://tools.ietf.org/html/rfc7234#section-4.2.2>
238    if let Some(ref code) = response.status.try_code() {
239        // <https://tools.ietf.org/html/rfc7234#section-5.5.4>
240        // Since presently we do not generate a Warning header field with a 113 warn-code,
241        // 24 hours minus response age is the max for heuristic calculation.
242        let max_heuristic = Duration::from_secs(24 * 60 * 60).saturating_sub(age);
243        let heuristic_freshness = if let Some(last_modified) =
244            // If the response has a Last-Modified header field,
245            // caches are encouraged to use a heuristic expiration value
246            // that is no more than some fraction of the interval since that time.
247            response.headers.typed_get::<LastModified>()
248        {
249            // `time_since_last_modified` will be `Duration::ZERO` if `last_modified` is
250            // after `now`.
251            let last_modified: SystemTime = last_modified.into();
252            let time_since_last_modified = now.duration_since(last_modified).unwrap_or_default();
253
254            // A typical setting of this fraction might be 10%.
255            let raw_heuristic_calc = time_since_last_modified / 10;
256            if raw_heuristic_calc < max_heuristic {
257                raw_heuristic_calc
258            } else {
259                max_heuristic
260            }
261        } else {
262            // Compatible with other browsers.
263            Duration::ZERO
264        };
265        if is_cacheable_by_default(*code) {
266            // Status codes that are cacheable by default can use heuristics to determine freshness.
267            return heuristic_freshness;
268        }
269        // Other status codes can only use heuristic freshness if the public cache directive is present.
270        if let Some(ref directives) = response.headers.typed_get::<CacheControl>() &&
271            directives.public()
272        {
273            return heuristic_freshness;
274        }
275    }
276    // Requires validation upon first use as default.
277    Duration::ZERO
278}
279
280/// The `headers` crate's `CacheControl` does not understand `stale-while-revalidate` directive,
281/// so we need to parse the raw `Cache-Control` header values.
282/// <https://datatracker.ietf.org/doc/html/rfc5861#section-3>
283fn get_stale_while_revalidate(headers: &HeaderMap) -> Duration {
284    for value in headers.get_all(header::CACHE_CONTROL) {
285        let Ok(value) = value.to_str() else {
286            continue;
287        };
288        for directive in value.split(',') {
289            let directive = directive.trim();
290            let Some((name, argument)) = directive.split_once('=') else {
291                continue;
292            };
293            if !name.trim().eq_ignore_ascii_case("stale-while-revalidate") {
294                continue;
295            }
296            // The argument is a number of seconds, optionally quoted.
297            let argument = argument.trim().trim_matches('"');
298            if let Ok(seconds) = argument.parse::<u64>() {
299                return Duration::from_secs(seconds);
300            }
301        }
302    }
303    Duration::ZERO
304}
305
306/// Determine whether the request itself demands revalidation.
307/// <https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.1>
308fn request_demands_revalidation(request: &Request) -> bool {
309    if matches!(
310        request.cache_mode,
311        CacheMode::NoCache | CacheMode::Reload | CacheMode::NoStore
312    ) {
313        return true;
314    }
315    if let Some(directive) = request.headers.typed_get::<CacheControl>() {
316        // The request's `no-store` directive is deliberately *not* treated as
317        // demanding revalidation: https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.1.5
318        // > "does not apply to the already stored response".
319        if directive.no_cache() {
320            return true;
321        }
322
323        if directive.max_age() == Some(Duration::ZERO) {
324            return true;
325        }
326    }
327    false
328}
329
330/// Request Cache-Control Directives
331/// <https://tools.ietf.org/html/rfc7234#section-5.2.1>
332fn get_expiry_adjustment_from_request_headers(request: &Request, expires: Duration) -> Duration {
333    let Some(directive) = request.headers.typed_get::<CacheControl>() else {
334        return expires;
335    };
336
337    if let Some(max_age) = directive.max_stale() {
338        return expires + max_age;
339    }
340
341    match directive.max_age() {
342        Some(max_age) if expires > max_age => return Duration::ZERO,
343        Some(max_age) => return expires - max_age,
344        None => {},
345    };
346
347    if let Some(min_fresh) = directive.min_fresh() {
348        if expires < min_fresh {
349            return Duration::ZERO;
350        }
351        return expires - min_fresh;
352    }
353
354    if directive.no_cache() || directive.no_store() {
355        return Duration::ZERO;
356    }
357
358    expires
359}
360
361/// Create a CachedResponse from a request and a CachedResource.
362fn create_cached_response(
363    request: &Request,
364    cached_resource: &CachedResource,
365    cached_headers: &HeaderMap,
366    done_chan: &mut DoneChannel,
367) -> Option<CachedResponse> {
368    debug!("creating a cached response for {:?}", request.url());
369    if cached_resource.aborted.load(Ordering::Acquire) {
370        return None;
371    }
372    let resource_timing = ResourceFetchTiming::new(request.timing_type());
373    let mut response = Response::new(cached_resource.metadata.final_url.clone(), resource_timing);
374    response.headers = cached_headers.clone();
375    response.body = cached_resource.body.clone();
376    if let ResponseBody::Receiving(_) = *cached_resource.body.lock() {
377        debug!("existing body is in progress");
378        let (done_sender, done_receiver) = unbounded();
379        *done_chan = Some((done_sender.clone(), done_receiver));
380        cached_resource.awaiting_body.lock().push(done_sender);
381    }
382    response
383        .location_url
384        .clone_from(&cached_resource.location_url);
385    response.status.clone_from(&cached_resource.status);
386    response.url_list.clone_from(&cached_resource.url_list);
387    response.referrer = request.referrer.to_url().cloned();
388    response.referrer_policy = request.referrer_policy;
389    response.aborted = cached_resource.aborted.clone();
390
391    let expires = cached_resource.expires;
392    let adjusted_expires = get_expiry_adjustment_from_request_headers(request, expires);
393    let time_since_validated = Instant::now() - cached_resource.last_validated;
394
395    // TODO: take must-revalidate into account <https://tools.ietf.org/html/rfc7234#section-5.2.2.1>
396    // TODO: if this cache is to be considered shared, take proxy-revalidate into account
397    // <https://tools.ietf.org/html/rfc7234#section-5.2.2.7>
398    let has_expired = adjusted_expires <= time_since_validated;
399
400    // - fresh: return immediately, no validation.
401    // - stale:
402    //    - within the stale-while-revalidate window: return immediately + revalidate in the background
403    //    - beyond the stale-while-revalidate window: synchronous validation is required.
404    let stale_for = time_since_validated.saturating_sub(adjusted_expires);
405    let within_stale_while_revalidate_window = stale_for <= cached_resource.stale_while_revalidate;
406    let validation_status = if !has_expired {
407        ValidationStatus::Valid
408    } else {
409        ValidationStatus::Stale {
410            revalidate_in_background: within_stale_while_revalidate_window &&
411                !cached_resource.stale_while_revalidate.is_zero() &&
412                !request_demands_revalidation(request),
413        }
414    };
415
416    let cached_response = CachedResponse {
417        response,
418        validation_status,
419        revalidation_guard: cached_resource.revalidating.clone(),
420    };
421    Some(cached_response)
422}
423
424/// Create a new resource, based on the bytes requested, and an existing resource,
425/// with a status-code of 206.
426fn create_resource_with_bytes_from_resource(
427    bytes: &[u8],
428    resource: &CachedResource,
429) -> CachedResource {
430    CachedResource {
431        request_headers: resource.request_headers.clone(),
432        body: Arc::new(ParkingLotMutex::new(ResponseBody::Done(bytes.to_owned()))),
433        aborted: Arc::new(AtomicBool::new(false)),
434        awaiting_body: Arc::new(ParkingLotMutex::new(vec![])),
435        metadata: resource.metadata.clone(),
436        location_url: resource.location_url.clone(),
437        status: StatusCode::PARTIAL_CONTENT.into(),
438        url_list: resource.url_list.clone(),
439        expires: resource.expires,
440        stale_while_revalidate: resource.stale_while_revalidate,
441        revalidating: resource.revalidating.clone(),
442        last_validated: resource.last_validated,
443    }
444}
445
446/// Support for range requests <https://tools.ietf.org/html/rfc7233>.
447fn handle_range_request(
448    request: &Request,
449    candidates: &[&CachedResource],
450    range_spec: &Range,
451    done_chan: &mut DoneChannel,
452) -> Option<CachedResponse> {
453    let mut complete_cached_resources = candidates
454        .iter()
455        .filter(|resource| resource.status == StatusCode::OK);
456    let partial_cached_resources = candidates
457        .iter()
458        .filter(|resource| resource.status == StatusCode::PARTIAL_CONTENT);
459    if let Some(complete_resource) = complete_cached_resources.next() {
460        // TODO: take the full range spec into account.
461        // If we have a complete resource, take the request range from the body.
462        // When there isn't a complete resource available, we loop over cached partials,
463        // and see if any individual partial response can fulfill the current request for a bytes range.
464        // TODO: combine partials that in combination could satisfy the requested range?
465        // see <https://tools.ietf.org/html/rfc7233#section-4.3>.
466        // TODO: add support for complete and partial resources,
467        // whose body is in the ResponseBody::Receiving state.
468        let body_len = match *complete_resource.body.lock() {
469            ResponseBody::Done(ref body) => body.len(),
470            _ => 0,
471        };
472        let bound = range_spec
473            .satisfiable_ranges(body_len.try_into().unwrap())
474            .next()
475            .unwrap();
476        match bound {
477            (Bound::Included(beginning), Bound::Included(end)) => {
478                if let ResponseBody::Done(ref body) = *complete_resource.body.lock() {
479                    if end == u64::MAX {
480                        // Prevent overflow on the addition below.
481                        return None;
482                    }
483                    let b = beginning as usize;
484                    let e = end as usize + 1;
485                    let requested = body.get(b..e);
486                    if let Some(bytes) = requested {
487                        let new_resource =
488                            create_resource_with_bytes_from_resource(bytes, complete_resource);
489                        let cached_headers = new_resource.metadata.headers.lock();
490                        let cached_response = create_cached_response(
491                            request,
492                            &new_resource,
493                            &cached_headers,
494                            done_chan,
495                        );
496                        if let Some(cached_response) = cached_response {
497                            return Some(cached_response);
498                        }
499                    }
500                }
501            },
502            (Bound::Included(beginning), Bound::Unbounded) => {
503                if let ResponseBody::Done(ref body) = *complete_resource.body.lock() {
504                    let b = beginning as usize;
505                    let requested = body.get(b..);
506                    if let Some(bytes) = requested {
507                        let new_resource =
508                            create_resource_with_bytes_from_resource(bytes, complete_resource);
509                        let cached_headers = new_resource.metadata.headers.lock();
510                        let cached_response = create_cached_response(
511                            request,
512                            &new_resource,
513                            &cached_headers,
514                            done_chan,
515                        );
516                        if let Some(cached_response) = cached_response {
517                            return Some(cached_response);
518                        }
519                    }
520                }
521            },
522            _ => return None,
523        }
524    } else {
525        for partial_resource in partial_cached_resources {
526            let headers = partial_resource.metadata.headers.lock();
527            let content_range = headers.typed_get::<ContentRange>();
528
529            let Some(body_len) = content_range.as_ref().and_then(|range| range.bytes_len()) else {
530                continue;
531            };
532            match range_spec.satisfiable_ranges(body_len - 1).next().unwrap() {
533                (Bound::Included(beginning), Bound::Included(end)) => {
534                    let (res_beginning, res_end) = match content_range {
535                        Some(range) => {
536                            if let Some(bytes_range) = range.bytes_range() {
537                                bytes_range
538                            } else {
539                                continue;
540                            }
541                        },
542                        _ => continue,
543                    };
544                    if res_beginning <= beginning && res_end >= end {
545                        let resource_body = &*partial_resource.body.lock();
546                        let requested = match resource_body {
547                            ResponseBody::Done(body) => {
548                                let b = beginning as usize - res_beginning as usize;
549                                let e = end as usize - res_beginning as usize + 1;
550                                body.get(b..e)
551                            },
552                            _ => continue,
553                        };
554                        if let Some(bytes) = requested {
555                            let new_resource =
556                                create_resource_with_bytes_from_resource(bytes, partial_resource);
557                            let cached_response =
558                                create_cached_response(request, &new_resource, &headers, done_chan);
559                            if let Some(cached_response) = cached_response {
560                                return Some(cached_response);
561                            }
562                        }
563                    }
564                },
565
566                (Bound::Included(beginning), Bound::Unbounded) => {
567                    let (res_beginning, res_end, total) = if let Some(range) = content_range {
568                        match (range.bytes_range(), range.bytes_len()) {
569                            (Some(bytes_range), Some(total)) => {
570                                (bytes_range.0, bytes_range.1, total)
571                            },
572                            _ => continue,
573                        }
574                    } else {
575                        continue;
576                    };
577                    if total == 0 {
578                        // Prevent overflow in the below operations from occuring.
579                        continue;
580                    };
581                    if res_beginning <= beginning && res_end == total - 1 {
582                        let resource_body = &*partial_resource.body.lock();
583                        let requested = match resource_body {
584                            ResponseBody::Done(body) => {
585                                let from_byte = beginning as usize - res_beginning as usize;
586                                body.get(from_byte..)
587                            },
588                            _ => continue,
589                        };
590                        if let Some(bytes) = requested {
591                            if bytes.len() as u64 + beginning < total - 1 {
592                                // Requested range goes beyond the available range.
593                                continue;
594                            }
595                            let new_resource =
596                                create_resource_with_bytes_from_resource(bytes, partial_resource);
597                            let cached_response =
598                                create_cached_response(request, &new_resource, &headers, done_chan);
599                            if let Some(cached_response) = cached_response {
600                                return Some(cached_response);
601                            }
602                        }
603                    }
604                },
605
606                _ => continue,
607            }
608        }
609    }
610
611    None
612}
613
614/// Constructing Responses from Caches.
615/// <https://tools.ietf.org/html/rfc7234#section-4>
616pub(crate) fn construct_response(
617    request: &Request,
618    done_chan: &mut DoneChannel,
619    cache_result: &[CachedResource],
620) -> Option<CachedResponse> {
621    if pref!(network_http_cache_disabled) {
622        return None;
623    }
624
625    // TODO: generate warning headers as appropriate <https://tools.ietf.org/html/rfc7234#section-5.5>
626    debug!("trying to construct cache response for {:?}", request.url());
627    if request.method != Method::GET {
628        // Only Get requests are cached, avoid a url based match for others.
629        debug!("non-GET method, not caching");
630        return None;
631    }
632
633    let resources = cache_result
634        .iter()
635        .filter(|r| !r.aborted.load(Ordering::Relaxed));
636    let mut candidates = vec![];
637    for cached_resource in resources {
638        let mut can_be_constructed = true;
639        let cached_headers = cached_resource.metadata.headers.lock();
640        let original_request_headers = cached_resource.request_headers.lock();
641        if let Some(vary_value) = cached_headers.typed_get::<Vary>() {
642            if vary_value.is_any() {
643                debug!("vary value is any, not caching");
644                can_be_constructed = false
645            } else {
646                // For every header name found in the Vary header of the stored response.
647                // Calculating Secondary Keys with Vary <https://tools.ietf.org/html/rfc7234#section-4.1>
648                for vary_val in vary_value.iter_strs() {
649                    match request.headers.get(vary_val) {
650                        Some(header_data) => {
651                            // If the header is present in the request.
652                            if let Some(original_header_data) =
653                                original_request_headers.get(vary_val)
654                            {
655                                // Check that the value of the nominated header field,
656                                // in the original request, matches the value in the current request.
657                                if original_header_data != header_data {
658                                    debug!("headers don't match, not caching");
659                                    can_be_constructed = false;
660                                    break;
661                                }
662                            }
663                        },
664                        None => {
665                            // If a header field is absent from a request,
666                            // it can only match a stored response if those headers,
667                            // were also absent in the original request.
668                            can_be_constructed = original_request_headers.get(vary_val).is_none();
669                            if !can_be_constructed {
670                                debug!("vary header present, not caching");
671                            }
672                        },
673                    }
674                    if !can_be_constructed {
675                        break;
676                    }
677                }
678            }
679        }
680        if can_be_constructed {
681            candidates.push(cached_resource);
682        }
683    }
684    // Support for range requests
685    if let Some(range_spec) = request.headers.typed_get::<Range>() {
686        return handle_range_request(request, candidates.as_slice(), &range_spec, done_chan);
687    }
688    while let Some(cached_resource) = candidates.pop() {
689        // Not a Range request.
690        // Do not allow 206 responses to be constructed.
691        //
692        // See https://tools.ietf.org/html/rfc7234#section-3.1
693        //
694        // A cache MUST NOT use an incomplete response to answer requests unless the
695        // response has been made complete or the request is partial and
696        // specifies a range that is wholly within the incomplete response.
697        //
698        // TODO: Combining partial content to fulfill a non-Range request
699        // see https://tools.ietf.org/html/rfc7234#section-3.3
700        match cached_resource.status.try_code() {
701            Some(ref code) => {
702                if *code == StatusCode::PARTIAL_CONTENT {
703                    continue;
704                }
705            },
706            None => continue,
707        }
708        // Returning a response that can be constructed
709        // TODO: select the most appropriate one, using a known mechanism from a selecting header field,
710        // or using the Date header to return the most recent one.
711        let cached_headers = cached_resource.metadata.headers.lock();
712        let cached_response =
713            create_cached_response(request, cached_resource, &cached_headers, done_chan);
714        if let Some(cached_response) = cached_response {
715            return Some(cached_response);
716        }
717    }
718    debug!("couldn't find an appropriate response, not caching");
719    // The cache wasn't able to construct anything.
720    None
721}
722
723/// Freshening Stored Responses upon Validation.
724/// <https://tools.ietf.org/html/rfc7234#section-4.3.4>
725pub fn refresh(
726    request: &Request,
727    response: Response,
728    done_chan: &mut DoneChannel,
729    cached_resources: &mut [CachedResource],
730) -> Option<Response> {
731    assert_eq!(response.status, StatusCode::NOT_MODIFIED);
732
733    let cached_resource = cached_resources.iter_mut().next()?;
734
735    let mut constructed_response = if let Some(range_spec) = request.headers.typed_get::<Range>() {
736        handle_range_request(request, &[cached_resource], &range_spec, done_chan)
737            .map(|cached_response| cached_response.response)
738    } else {
739        // done_chan will have been set to Some(..) by http_network_fetch.
740        // If the body is not receiving data, set the done_chan back to None.
741        // Otherwise, create a new dedicated channel to update the consumer.
742        // The response constructed here will replace the 304 one from the network.
743        let in_progress_channel = match &*cached_resource.body.lock() {
744            ResponseBody::Receiving(..) => Some(unbounded()),
745            ResponseBody::Empty | ResponseBody::Done(..) => None,
746        };
747        match in_progress_channel {
748            Some((done_sender, done_receiver)) => {
749                *done_chan = Some((done_sender.clone(), done_receiver));
750                cached_resource.awaiting_body.lock().push(done_sender);
751            },
752            None => *done_chan = None,
753        }
754        // Received a response with 304 status code, in response to a request that matches a cached resource.
755        // 1. update the headers of the cached resource.
756        // 2. return a response, constructed from the cached resource.
757        let resource_timing = ResourceFetchTiming::new(request.timing_type());
758        let mut constructed_response =
759            Response::new(cached_resource.metadata.final_url.clone(), resource_timing);
760
761        constructed_response.body = cached_resource.body.clone();
762
763        constructed_response
764            .status
765            .clone_from(&cached_resource.status);
766        constructed_response.referrer = request.referrer.to_url().cloned();
767        constructed_response.referrer_policy = request.referrer_policy;
768        constructed_response
769            .status
770            .clone_from(&cached_resource.status);
771        constructed_response
772            .url_list
773            .clone_from(&cached_resource.url_list);
774        Some(constructed_response)
775    };
776
777    // Update cached Resource with response and constructed response.
778    if let Some(constructed_response) = constructed_response.as_mut() {
779        // Bracket is to minimize lock duration.
780        {
781            let mut stored_headers = cached_resource.metadata.headers.lock();
782            stored_headers.extend(response.headers);
783            constructed_response.headers = stored_headers.clone();
784        }
785        cached_resource.expires = get_response_expiry(constructed_response);
786        cached_resource.stale_while_revalidate =
787            get_stale_while_revalidate(&constructed_response.headers);
788        cached_resource.last_validated = Instant::now();
789    }
790
791    constructed_response
792}
793
794pub(crate) fn invalidate_cached_resources(cached_resources: &mut [CachedResource]) {
795    for cached_resource in cached_resources.iter_mut() {
796        cached_resource.expires = Duration::ZERO;
797    }
798}
799
800fn resolve_location_url(
801    request: &Request,
802    response: &Response,
803    header_name: header::HeaderName,
804) -> Option<ServoUrl> {
805    response
806        .headers
807        .get(header_name)
808        .and_then(|value| value.to_str().ok())
809        .and_then(|location| request.current_url().join(location).ok())
810}
811
812impl HttpCache {
813    /// Wake-up consumers of cached resources
814    /// whose response body was still receiving data when the resource was constructed,
815    /// and whose response has now either been completed or cancelled.
816    pub(crate) async fn update_awaiting_consumers(&self, request: &Request, response: &Response) {
817        let entry_key = CacheKey::new(request);
818
819        let cached_resources = match self.entries.get(&entry_key) {
820            None => return,
821            Some(resources) => resources,
822        };
823
824        let actual_response = response.actual_response();
825
826        // Ensure we only wake-up consumers of relevant resources,
827        // ie we don't want to wake-up 200 awaiting consumers with a 206.
828        let lock = cached_resources.read().await;
829        let relevant_cached_resources = lock.iter().filter(|resource| {
830            if actual_response.is_network_error() {
831                return *resource.body.lock() == ResponseBody::Empty;
832            }
833            resource.status == actual_response.status
834        });
835
836        for cached_resource in relevant_cached_resources {
837            let mut awaiting_consumers = cached_resource.awaiting_body.lock();
838            if awaiting_consumers.is_empty() {
839                continue;
840            }
841            let to_send = if cached_resource.aborted.load(Ordering::Acquire) {
842                // In the case of an aborted fetch,
843                // wake-up all awaiting consumers.
844                // Each will then start a new network request.
845                // TODO: Wake-up only one consumer, and make it the producer on which others wait.
846                Data::Cancelled
847            } else {
848                match *cached_resource.body.lock() {
849                    ResponseBody::Done(_) | ResponseBody::Empty => Data::Done,
850                    ResponseBody::Receiving(_) => {
851                        continue;
852                    },
853                }
854            };
855            for done_sender in awaiting_consumers.drain(..) {
856                let _ = done_sender.send(to_send.clone());
857            }
858        }
859    }
860
861    /// Returns descriptors for cache entries currently stored in this cache.
862    pub(crate) fn cache_entry_descriptors(&self) -> Vec<CacheEntryDescriptor> {
863        self.entries
864            .iter()
865            .map(|(key, _)| CacheEntryDescriptor::new(key.url.to_string()))
866            .collect()
867    }
868
869    /// Clear the contents of this cache.
870    pub(crate) fn clear(&self) {
871        self.entries.clear();
872    }
873
874    /// Insert a response for `request` into the cache (used by tests that need direct access).
875    pub async fn store(&self, request: &Request, response: &Response) {
876        let guard = self.get_or_guard(CacheKey::new(request)).await;
877        guard.insert(request, response);
878    }
879
880    /// Try to construct a cached response for `request`.
881    pub async fn construct_response(
882        &self,
883        request: &Request,
884        done_chan: &mut DoneChannel,
885    ) -> Option<Response> {
886        let entry = self.entries.get(&CacheKey::new(request))?;
887        let cached_resources = entry.read().await;
888        construct_response(request, done_chan, cached_resources.as_slice())
889            .map(|cached| cached.response)
890    }
891
892    /// Like [`construct_response`](Self::construct_response), but additionally
893    /// reports the [`ValidationStatus`] of the constructed response.
894    #[cfg(feature = "test-util")]
895    pub async fn construct_response_freshness(
896        &self,
897        request: &Request,
898        done_chan: &mut DoneChannel,
899    ) -> Option<ValidationStatus> {
900        let entry = self.entries.get(&CacheKey::new(request))?;
901        let cached_resources = entry.read().await;
902        construct_response(request, done_chan, cached_resources.as_slice())
903            .map(|cached| cached.validation_status)
904    }
905
906    /// Invalidate cache entries referenced by Location/Content-Location headers.
907    pub(crate) async fn invalidate_related_urls(
908        &self,
909        request: &Request,
910        response: &Response,
911        skip_key: &CacheKey,
912    ) {
913        for header_name in &[header::LOCATION, header::CONTENT_LOCATION] {
914            if let Some(location_url) = resolve_location_url(request, response, header_name.clone())
915            {
916                let location_key = CacheKey::from_url(location_url);
917                if &location_key != skip_key {
918                    self.invalidate_entry(&location_key).await;
919                }
920            }
921        }
922    }
923
924    async fn invalidate_entry(&self, key: &CacheKey) {
925        if let Some(entry) = self.entries.get(key) {
926            let mut guarded_resources = entry.write().await;
927            invalidate_cached_resources(guarded_resources.as_mut_slice());
928        }
929    }
930
931    /// If the value exist in the cache, return it. If the value does not exist, return a guard you can use to insert values in the cache.
932    /// If the guard is alive, all other accesses to this function will block.
933    pub async fn get_or_guard(&self, entry_key: CacheKey) -> CachedResourcesOrGuard<'_> {
934        match self.entries.get_value_or_guard_async(&entry_key).await {
935            Ok(val) => CachedResourcesOrGuard::Value(val.write_owned().await),
936            Err(guard) => CachedResourcesOrGuard::Guard(guard),
937        }
938    }
939}
940
941/// Returns an writeable entry into the cache or a guard for insertint an entry
942/// The guard will block other queries to the cache entry in both cases.
943pub enum CachedResourcesOrGuard<'a> {
944    /// The value of the resource in the cache.
945    Value(OwnedRwLockWriteGuard<Vec<CachedResource>>),
946    /// A guard that blocks requests to the cache entry this guard is for.
947    Guard(QuickCachePlaceeholderGuard<'a>),
948}
949
950impl<'a> CachedResourcesOrGuard<'a> {
951    /// Insert into the cache according to http spec
952    pub fn insert(self, request: &Request, response: &Response) {
953        if pref!(network_http_cache_disabled) {
954            return;
955        }
956
957        if request.method != Method::GET {
958            // Only Get requests are cached.
959            return;
960        }
961        if request.headers.contains_key(header::AUTHORIZATION) {
962            // https://tools.ietf.org/html/rfc7234#section-3.1
963            // A shared cache MUST NOT use a cached response
964            // to a request with an Authorization header field
965            //
966            // TODO: unless a cache directive that allows such
967            // responses to be stored is present in the response.
968            return;
969        };
970        let metadata = match response.metadata() {
971            Ok(FetchMetadata::Filtered {
972                filtered: _,
973                unsafe_: metadata,
974            }) |
975            Ok(FetchMetadata::Unfiltered(metadata)) => metadata,
976            _ => return,
977        };
978        if !response_is_cacheable(&metadata) {
979            return;
980        }
981        let expiry = get_response_expiry(response);
982        let stale_while_revalidate = get_stale_while_revalidate(&response.headers);
983        let cacheable_metadata = CachedMetadata {
984            headers: Arc::new(ParkingLotMutex::new(response.headers.clone())),
985            final_url: metadata.final_url,
986            content_type: metadata.content_type.map(|v| v.0.to_string()),
987            charset: metadata.charset,
988            status: metadata.status,
989        };
990        let entry_resource = CachedResource {
991            request_headers: Arc::new(ParkingLotMutex::new(request.headers.clone())),
992            body: response.body.clone(),
993            aborted: response.aborted.clone(),
994            awaiting_body: Arc::new(ParkingLotMutex::new(vec![])),
995            metadata: cacheable_metadata,
996            location_url: response.location_url.clone(),
997            status: response.status.clone(),
998            url_list: response.url_list.clone(),
999            expires: expiry,
1000            stale_while_revalidate,
1001            revalidating: StdArc::new(AtomicBool::new(false)),
1002            last_validated: Instant::now(),
1003        };
1004
1005        match self {
1006            CachedResourcesOrGuard::Value(mut owned_rw_lock_write_guard) => {
1007                owned_rw_lock_write_guard.push(entry_resource);
1008            },
1009            CachedResourcesOrGuard::Guard(placeholder_guard) => {
1010                if placeholder_guard
1011                    .insert(std::sync::Arc::new(TokioRwLock::new(vec![entry_resource])))
1012                    .is_err()
1013                {
1014                    error!("Could not insert into cache");
1015                }
1016            },
1017        }
1018    }
1019
1020    /// If the guard is a value, return it as a mut reference. If the guard is a guard, return None
1021    pub fn try_as_mut(&mut self) -> Option<&mut Vec<CachedResource>> {
1022        match self {
1023            CachedResourcesOrGuard::Value(owned_rw_lock_write_guard) => {
1024                Some(owned_rw_lock_write_guard.as_mut())
1025            },
1026            CachedResourcesOrGuard::Guard(_) => None,
1027        }
1028    }
1029}
net/http_cache.rs

net/
http_cache.rs
