use std::borrow::Cow;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex};
use std::{io, mem, str};
use base64::engine::general_purpose;
use base64::Engine as _;
use content_security_policy as csp;
use crossbeam_channel::Sender;
use devtools_traits::DevtoolsControlMsg;
use headers::{AccessControlExposeHeaders, ContentType, HeaderMapExt};
use http::header::{self, HeaderMap, HeaderName};
use http::{Method, StatusCode};
use ipc_channel::ipc::{self, IpcReceiver};
use log::warn;
use mime::{self, Mime};
use net_traits::filemanager_thread::{FileTokenCheck, RelativePos};
use net_traits::http_status::HttpStatus;
use net_traits::policy_container::{PolicyContainer, RequestPolicyContainer};
use net_traits::request::{
is_cors_safelisted_method, is_cors_safelisted_request_header, BodyChunkRequest,
BodyChunkResponse, CredentialsMode, Destination, Origin, RedirectMode, Referrer, Request,
RequestMode, ResponseTainting, Window,
};
use net_traits::response::{Response, ResponseBody, ResponseType};
use net_traits::{
FetchTaskTarget, NetworkError, ReferrerPolicy, ResourceAttribute, ResourceFetchTiming,
ResourceTimeValue, ResourceTimingType,
};
use rustls::Certificate;
use serde::{Deserialize, Serialize};
use servo_arc::Arc as ServoArc;
use servo_url::ServoUrl;
use tokio::sync::mpsc::{UnboundedReceiver as TokioReceiver, UnboundedSender as TokioSender};
use crate::fetch::cors_cache::CorsCache;
use crate::fetch::headers::determine_nosniff;
use crate::filemanager_thread::FileManager;
use crate::http_loader::{
determine_requests_referrer, http_fetch, set_default_accept, set_default_accept_language,
HttpState,
};
use crate::protocols::ProtocolRegistry;
use crate::subresource_integrity::is_response_integrity_valid;
pub type Target<'a> = &'a mut (dyn FetchTaskTarget + Send);
#[derive(Clone, Deserialize, Serialize)]
pub enum Data {
Payload(Vec<u8>),
Done,
Cancelled,
}
pub struct FetchContext {
pub state: Arc<HttpState>,
pub user_agent: Cow<'static, str>,
pub devtools_chan: Option<Arc<Mutex<Sender<DevtoolsControlMsg>>>>,
pub filemanager: Arc<Mutex<FileManager>>,
pub file_token: FileTokenCheck,
pub cancellation_listener: Arc<Mutex<CancellationListener>>,
pub timing: ServoArc<Mutex<ResourceFetchTiming>>,
pub protocols: Arc<ProtocolRegistry>,
}
pub struct CancellationListener {
cancel_chan: Option<IpcReceiver<()>>,
cancelled: bool,
}
impl CancellationListener {
pub fn new(cancel_chan: Option<IpcReceiver<()>>) -> Self {
Self {
cancel_chan,
cancelled: false,
}
}
pub fn cancelled(&mut self) -> bool {
if let Some(ref cancel_chan) = self.cancel_chan {
if self.cancelled {
true
} else if cancel_chan.try_recv().is_ok() {
self.cancelled = true;
true
} else {
false
}
} else {
false
}
}
}
pub type DoneChannel = Option<(TokioSender<Data>, TokioReceiver<Data>)>;
pub async fn fetch(request: &mut Request, target: Target<'_>, context: &FetchContext) {
context
.timing
.lock()
.unwrap()
.set_attribute(ResourceAttribute::FetchStart);
context
.timing
.lock()
.unwrap()
.set_attribute(ResourceAttribute::StartTime(ResourceTimeValue::FetchStart));
fetch_with_cors_cache(request, &mut CorsCache::default(), target, context).await;
}
pub async fn fetch_with_cors_cache(
request: &mut Request,
cache: &mut CorsCache,
target: Target<'_>,
context: &FetchContext,
) {
if request.window == Window::Client {
} else {
request.window = Window::NoWindow;
}
if request.origin == Origin::Client {
unimplemented!()
}
if let RequestPolicyContainer::Client = request.policy_container {
request.policy_container =
RequestPolicyContainer::PolicyContainer(PolicyContainer::default());
}
set_default_accept(request);
set_default_accept_language(&mut request.headers);
if request.is_subresource_request() {
}
main_fetch(request, cache, false, false, target, &mut None, context).await;
}
pub fn should_request_be_blocked_by_csp(
request: &Request,
policy_container: &PolicyContainer,
) -> csp::CheckResult {
let origin = match &request.origin {
Origin::Client => return csp::CheckResult::Allowed,
Origin::Origin(origin) => origin,
};
let csp_request = csp::Request {
url: request.url().into_url(),
origin: origin.clone().into_url_origin(),
redirect_count: request.redirect_count,
destination: request.destination,
initiator: csp::Initiator::None,
nonce: String::new(),
integrity_metadata: request.integrity_metadata.clone(),
parser_metadata: csp::ParserMetadata::None,
};
policy_container
.csp_list
.as_ref()
.map(|c| c.should_request_be_blocked(&csp_request).0)
.unwrap_or(csp::CheckResult::Allowed)
}
pub async fn main_fetch(
request: &mut Request,
cache: &mut CorsCache,
cors_flag: bool,
recursive_flag: bool,
target: Target<'_>,
done_chan: &mut DoneChannel,
context: &FetchContext,
) -> Response {
let mut response = None;
if let Some(ref details) = request.crash {
response = Some(Response::network_error(NetworkError::Crash(
details.clone(),
)));
}
if request.local_urls_only &&
!matches!(
request.current_url().scheme(),
"about" | "blob" | "data" | "filesystem"
)
{
response = Some(Response::network_error(NetworkError::Internal(
"Non-local scheme".into(),
)));
}
let policy_container = match &request.policy_container {
RequestPolicyContainer::Client => PolicyContainer::default(),
RequestPolicyContainer::PolicyContainer(container) => container.to_owned(),
};
if should_request_be_blocked_by_csp(request, &policy_container) == csp::CheckResult::Blocked {
warn!("Request blocked by CSP");
response = Some(Response::network_error(NetworkError::Internal(
"Blocked by Content-Security-Policy".into(),
)))
}
if should_be_blocked_due_to_bad_port(&request.current_url()) {
response = Some(Response::network_error(NetworkError::Internal(
"Request attempted on bad port".into(),
)));
}
if request.referrer_policy == ReferrerPolicy::EmptyString {
request.referrer_policy = policy_container.get_referrer_policy();
}
let referrer_url = match mem::replace(&mut request.referrer, Referrer::NoReferrer) {
Referrer::NoReferrer => None,
Referrer::ReferrerUrl(referrer_source) | Referrer::Client(referrer_source) => {
request.headers.remove(header::REFERER);
determine_requests_referrer(
request.referrer_policy,
referrer_source,
request.current_url(),
)
},
};
request.referrer = referrer_url.map_or(Referrer::NoReferrer, Referrer::ReferrerUrl);
context
.state
.hsts_list
.read()
.unwrap()
.apply_hsts_rules(request.current_url_mut());
let current_url = request.current_url();
let current_scheme = current_url.scheme();
let mut response = match response {
Some(res) => res,
None => {
let same_origin = if let Origin::Origin(ref origin) = request.origin {
*origin == current_url.origin()
} else {
false
};
if (same_origin && !cors_flag) ||
current_scheme == "chrome" ||
context.protocols.is_fetchable(current_scheme) ||
matches!(
request.mode,
RequestMode::Navigate | RequestMode::WebSocket { .. }
)
{
request.response_tainting = ResponseTainting::Basic;
scheme_fetch(request, cache, target, done_chan, context).await
} else if request.mode == RequestMode::SameOrigin {
Response::network_error(NetworkError::Internal("Cross-origin response".into()))
} else if request.mode == RequestMode::NoCors {
if request.redirect_mode != RedirectMode::Follow {
Response::network_error(NetworkError::Internal(
"NoCors requests must follow redirects".into(),
))
} else {
request.response_tainting = ResponseTainting::Opaque;
scheme_fetch(request, cache, target, done_chan, context).await
}
} else if !matches!(current_scheme, "http" | "https") {
Response::network_error(NetworkError::Internal("Non-http scheme".into()))
} else if request.use_cors_preflight ||
(request.unsafe_request &&
(!is_cors_safelisted_method(&request.method) ||
request.headers.iter().any(|(name, value)| {
!is_cors_safelisted_request_header(&name, &value)
})))
{
request.response_tainting = ResponseTainting::CorsTainting;
let response = http_fetch(
request, cache, true, true, false, target, done_chan, context,
)
.await;
if response.is_network_error() {
}
response
} else {
request.response_tainting = ResponseTainting::CorsTainting;
http_fetch(
request, cache, true, false, false, target, done_chan, context,
)
.await
}
},
};
if recursive_flag {
return response;
}
let mut response = if !response.is_network_error() && response.internal_response.is_none() {
if request.response_tainting == ResponseTainting::CorsTainting {
let header_names: Option<Vec<HeaderName>> = response
.headers
.typed_get::<AccessControlExposeHeaders>()
.map(|v| v.iter().collect());
match header_names {
Some(ref list)
if request.credentials_mode != CredentialsMode::Include &&
list.iter().any(|header| header == "*") =>
{
response.cors_exposed_header_name_list = response
.headers
.iter()
.map(|(name, _)| name.as_str().to_owned())
.collect();
},
Some(list) => {
response.cors_exposed_header_name_list =
list.iter().map(|h| h.as_str().to_owned()).collect();
},
_ => (),
}
}
let response_type = match request.response_tainting {
ResponseTainting::Basic => ResponseType::Basic,
ResponseTainting::CorsTainting => ResponseType::Cors,
ResponseTainting::Opaque => ResponseType::Opaque,
};
response.to_filtered(response_type)
} else {
response
};
let internal_error = {
let response_is_network_error = response.is_network_error();
let should_replace_with_nosniff_error = !response_is_network_error &&
should_be_blocked_due_to_nosniff(request.destination, &response.headers);
let should_replace_with_mime_type_error = !response_is_network_error &&
should_be_blocked_due_to_mime_type(request.destination, &response.headers);
let mut network_error_response = response
.get_network_error()
.cloned()
.map(Response::network_error);
let internal_response = if let Some(error_response) = network_error_response.as_mut() {
error_response
} else {
response.actual_response_mut()
};
if internal_response.url_list.is_empty() {
internal_response.url_list.clone_from(&request.url_list)
}
let blocked_error_response;
let internal_response = if should_replace_with_nosniff_error {
blocked_error_response =
Response::network_error(NetworkError::Internal("Blocked by nosniff".into()));
&blocked_error_response
} else if should_replace_with_mime_type_error {
blocked_error_response =
Response::network_error(NetworkError::Internal("Blocked by mime type".into()));
&blocked_error_response
} else {
internal_response
};
let not_network_error = !response_is_network_error && !internal_response.is_network_error();
if not_network_error &&
(is_null_body_status(&internal_response.status) ||
matches!(request.method, Method::HEAD | Method::CONNECT))
{
let mut body = internal_response.body.lock().unwrap();
*body = ResponseBody::Empty;
}
internal_response.get_network_error().cloned()
};
let mut response = if let Some(error) = internal_error {
Response::network_error(error)
} else {
response
};
let mut response_loaded = false;
let mut response = if !response.is_network_error() && !request.integrity_metadata.is_empty() {
wait_for_response(request, &mut response, target, done_chan).await;
response_loaded = true;
let integrity_metadata = &request.integrity_metadata;
if response.termination_reason.is_none() &&
!is_response_integrity_valid(integrity_metadata, &response)
{
Response::network_error(NetworkError::Internal(
"Subresource integrity validation failed".into(),
))
} else {
response
}
} else {
response
};
if request.synchronous {
target.process_response(request, &response);
if !response_loaded {
wait_for_response(request, &mut response, target, done_chan).await;
}
target.process_response_eof(request, &response);
return response;
}
if request.body.is_some() && matches!(current_scheme, "http" | "https") {
target.process_request_body(request);
target.process_request_eof(request);
}
target.process_response(request, &response);
if !response_loaded {
wait_for_response(request, &mut response, target, done_chan).await;
}
target.process_response_eof(request, &response);
if let Ok(http_cache) = context.state.http_cache.write() {
http_cache.update_awaiting_consumers(request, &response);
}
response
}
async fn wait_for_response(
request: &Request,
response: &mut Response,
target: Target<'_>,
done_chan: &mut DoneChannel,
) {
if let Some(ref mut ch) = *done_chan {
loop {
match ch.1.recv().await {
Some(Data::Payload(vec)) => {
target.process_response_chunk(request, vec);
},
Some(Data::Done) => {
break;
},
Some(Data::Cancelled) => {
response.aborted.store(true, Ordering::Release);
break;
},
_ => {
panic!("fetch worker should always send Done before terminating");
},
}
}
} else {
let body = response.actual_response().body.lock().unwrap();
if let ResponseBody::Done(ref vec) = *body {
target.process_response_chunk(request, vec.clone());
} else {
assert_eq!(*body, ResponseBody::Empty)
}
}
}
pub enum RangeRequestBounds {
Final(RelativePos),
Pending(u64),
}
impl RangeRequestBounds {
pub fn get_final(&self, len: Option<u64>) -> Result<RelativePos, &'static str> {
match self {
RangeRequestBounds::Final(pos) => {
if let Some(len) = len {
if pos.start <= len as i64 {
return Ok(pos.clone());
}
}
Err("Tried to process RangeRequestBounds::Final without len")
},
RangeRequestBounds::Pending(offset) => Ok(RelativePos::from_opts(
if let Some(len) = len {
Some((len - u64::min(len, *offset)) as i64)
} else {
Some(0)
},
None,
)),
}
}
}
fn create_blank_reply(url: ServoUrl, timing_type: ResourceTimingType) -> Response {
let mut response = Response::new(url, ResourceFetchTiming::new(timing_type));
response
.headers
.typed_insert(ContentType::from(mime::TEXT_HTML_UTF_8));
*response.body.lock().unwrap() = ResponseBody::Done(vec![]);
response.status = HttpStatus::default();
response
}
fn handle_allowcert_request(request: &mut Request, context: &FetchContext) -> io::Result<()> {
let error = |string| Err(io::Error::new(io::ErrorKind::Other, string));
let body = match request.body.as_mut() {
Some(body) => body,
None => return error("No body found"),
};
let stream = body.take_stream();
let stream = stream.lock().unwrap();
let (body_chan, body_port) = ipc::channel().unwrap();
let _ = stream.send(BodyChunkRequest::Connect(body_chan));
let _ = stream.send(BodyChunkRequest::Chunk);
let body_bytes = match body_port.recv().ok() {
Some(BodyChunkResponse::Chunk(bytes)) => bytes,
_ => return error("Certificate not sent in a single chunk"),
};
let split_idx = match body_bytes.iter().position(|b| *b == b'&') {
Some(split_idx) => split_idx,
None => return error("Could not find ampersand in data"),
};
let (secret, cert_base64) = body_bytes.split_at(split_idx);
let secret = str::from_utf8(secret).ok().and_then(|s| s.parse().ok());
if secret != Some(*net_traits::PRIVILEGED_SECRET) {
return error("Invalid secret sent. Ignoring request");
}
let cert_bytes = match general_purpose::STANDARD_NO_PAD.decode(&cert_base64[1..]) {
Ok(bytes) => bytes,
Err(_) => return error("Could not decode certificate base64"),
};
context
.state
.override_manager
.add_override(&Certificate(cert_bytes));
Ok(())
}
async fn scheme_fetch(
request: &mut Request,
cache: &mut CorsCache,
target: Target<'_>,
done_chan: &mut DoneChannel,
context: &FetchContext,
) -> Response {
let url = request.current_url();
let scheme = url.scheme();
match scheme {
"about" if url.path() == "blank" => create_blank_reply(url, request.timing_type()),
"chrome" if url.path() == "allowcert" => {
if let Err(error) = handle_allowcert_request(request, context) {
warn!("Could not handle allowcert request: {error}");
}
create_blank_reply(url, request.timing_type())
},
"http" | "https" => {
http_fetch(
request, cache, false, false, false, target, done_chan, context,
)
.await
},
_ => match context.protocols.get(scheme) {
Some(handler) => handler.load(request, done_chan, context).await,
None => Response::network_error(NetworkError::Internal("Unexpected scheme".into())),
},
}
}
fn is_null_body_status(status: &HttpStatus) -> bool {
matches!(
status.try_code(),
Some(StatusCode::SWITCHING_PROTOCOLS) |
Some(StatusCode::NO_CONTENT) |
Some(StatusCode::RESET_CONTENT) |
Some(StatusCode::NOT_MODIFIED)
)
}
pub fn should_be_blocked_due_to_nosniff(
destination: Destination,
response_headers: &HeaderMap,
) -> bool {
if !determine_nosniff(response_headers) {
return false;
}
let content_type_header = response_headers.typed_get::<ContentType>();
#[inline]
fn is_javascript_mime_type(mime_type: &Mime) -> bool {
let javascript_mime_types: [Mime; 16] = [
"application/ecmascript".parse().unwrap(),
"application/javascript".parse().unwrap(),
"application/x-ecmascript".parse().unwrap(),
"application/x-javascript".parse().unwrap(),
"text/ecmascript".parse().unwrap(),
"text/javascript".parse().unwrap(),
"text/javascript1.0".parse().unwrap(),
"text/javascript1.1".parse().unwrap(),
"text/javascript1.2".parse().unwrap(),
"text/javascript1.3".parse().unwrap(),
"text/javascript1.4".parse().unwrap(),
"text/javascript1.5".parse().unwrap(),
"text/jscript".parse().unwrap(),
"text/livescript".parse().unwrap(),
"text/x-ecmascript".parse().unwrap(),
"text/x-javascript".parse().unwrap(),
];
javascript_mime_types
.iter()
.any(|mime| mime.type_() == mime_type.type_() && mime.subtype() == mime_type.subtype())
}
match content_type_header {
Some(ref ct) if destination.is_script_like() => {
!is_javascript_mime_type(&ct.clone().into())
},
Some(ref ct) if destination == Destination::Style => {
let m: mime::Mime = ct.clone().into();
m.type_() != mime::TEXT && m.subtype() != mime::CSS
},
None if destination == Destination::Style || destination.is_script_like() => true,
_ => false,
}
}
fn should_be_blocked_due_to_mime_type(
destination: Destination,
response_headers: &HeaderMap,
) -> bool {
let mime_type: mime::Mime = match response_headers.typed_get::<ContentType>() {
Some(header) => header.into(),
None => return false,
};
destination.is_script_like() &&
match mime_type.type_() {
mime::AUDIO | mime::VIDEO | mime::IMAGE => true,
mime::TEXT if mime_type.subtype() == mime::CSV => true,
_ => false,
}
}
pub fn should_be_blocked_due_to_bad_port(url: &ServoUrl) -> bool {
let scheme = url.scheme();
let port = if let Some(port) = url.port() {
port
} else {
return false;
};
if scheme == "ftp" && (port == 20 || port == 21) {
return false;
}
if is_network_scheme(scheme) && is_bad_port(port) {
return true;
}
false
}
fn is_network_scheme(scheme: &str) -> bool {
scheme == "ftp" || scheme == "http" || scheme == "https"
}
fn is_bad_port(port: u16) -> bool {
static BAD_PORTS: [u16; 78] = [
1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, 53, 69, 77, 79, 87, 95, 101,
102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 137, 139, 143, 161, 179, 389,
427, 465, 512, 513, 514, 515, 526, 530, 531, 532, 540, 548, 554, 556, 563, 587, 601, 636,
993, 995, 1719, 1720, 1723, 2049, 3659, 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667,
6668, 6669, 6697, 10080,
];
BAD_PORTS.binary_search(&port).is_ok()
}