From 567eae670455b60c3384b227bb97b53b340c0df5 Mon Sep 17 00:00:00 2001 From: Justin Chase Date: Thu, 22 Jul 2021 10:21:48 -0500 Subject: [PATCH] Support native certs --- Cargo.lock | 64 +++++++++++++-- cli/Cargo.toml | 2 + cli/file_fetcher.rs | 12 ++- cli/flags.rs | 4 + cli/http_util.rs | 39 +++------- cli/main.rs | 4 +- cli/program_state.rs | 58 ++++++++++++-- cli/standalone.rs | 18 ++++- cli/tests/integration/mod.rs | 4 +- cli/tools/standalone.rs | 2 + core/Cargo.toml | 5 ++ core/http.rs | 117 ++++++++++++++++++++++++++++ core/lib.rs | 5 ++ extensions/fetch/Cargo.toml | 1 + extensions/fetch/lib.rs | 75 ++++-------------- extensions/net/Cargo.toml | 3 - extensions/net/lib.rs | 12 +-- extensions/net/ops_tls.rs | 125 ++++++++++-------------------- extensions/websocket/Cargo.toml | 2 - extensions/websocket/lib.rs | 31 +++----- runtime/examples/hello_runtime.rs | 2 +- runtime/web_worker.rs | 9 ++- runtime/worker.rs | 14 ++-- 23 files changed, 375 insertions(+), 233 deletions(-) create mode 100644 core/http.rs diff --git a/Cargo.lock b/Cargo.lock index 1b27a5a712d4e2..890e24ad51610f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,6 +588,7 @@ dependencies = [ "rand 0.8.4", "regex", "ring", + "rustls-native-certs", "rustyline", "rustyline-derive", "semver-parser 0.10.2", @@ -608,6 +609,7 @@ dependencies = [ "trust-dns-server", "uuid", "walkdir", + "webpki-roots", "winapi 0.3.9", "winres", ] @@ -650,12 +652,17 @@ dependencies = [ "log", "parking_lot", "pin-project", + "reqwest", + "rustls", + "rustls-native-certs", "rusty_v8", "serde", "serde_json", "serde_v8", "tokio", "url", + "webpki", + "webpki-roots", ] [[package]] @@ -701,6 +708,7 @@ dependencies = [ "deno_core", "deno_web", "http", + "lazy_static", "reqwest", "serde", "tokio", @@ -750,13 +758,10 @@ dependencies = [ "deno_core", "lazy_static", "log", - "rustls", "serde", "tokio", "trust-dns-proto", "trust-dns-resolver", - "webpki", - "webpki-roots", ] [[package]] @@ -870,8 +875,6 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-tungstenite", - "webpki", - "webpki-roots", ] [[package]] @@ -2342,6 +2345,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + [[package]] name = "os_pipe" version = "0.9.2" @@ -2956,6 +2965,18 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "rusty_v8" version = "0.25.3" @@ -3017,6 +3038,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -3039,6 +3070,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2daf06dafd24a6..930f5442c1dfd2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -75,6 +75,7 @@ pin-project = "1.0.7" regex = "1.4.3" rand = { version = "0.8.4", features = [ "small_rng" ] } ring = "0.16.20" +rustls-native-certs = "0.5.0" rustyline = { version = "8.2.0", default-features = false } rustyline-derive = "0.4.0" semver-parser = "0.10.2" @@ -91,6 +92,7 @@ tokio = { version = "1.8.1", features = ["full"] } tokio-rustls = "0.22.0" uuid = { version = "0.8.2", features = ["v4", "serde"] } walkdir = "2.3.2" +webpki-roots = "0.21.1" [target.'cfg(windows)'.dependencies] fwdansi = "1.1.0" diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 61cf1dae68f9f9..bd386017062d9a 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -3,7 +3,6 @@ use crate::auth_tokens::AuthTokens; use crate::colors; use crate::http_cache::HttpCache; -use crate::http_util::create_http_client; use crate::http_util::fetch_once; use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceResult; @@ -17,7 +16,9 @@ use deno_core::error::uri_error; use deno_core::error::AnyError; use deno_core::futures; use deno_core::futures::future::FutureExt; +use deno_core::http::create_http_client; use deno_core::parking_lot::Mutex; +use deno_core::rustls::RootCertStore; use deno_core::ModuleSpecifier; use deno_runtime::deno_fetch::reqwest; use deno_runtime::deno_web::BlobStore; @@ -220,7 +221,7 @@ impl FileFetcher { http_cache: HttpCache, cache_setting: CacheSetting, allow_remote: bool, - ca_data: Option>, + root_store: Option, blob_store: BlobStore, ) -> Result { Ok(Self { @@ -229,7 +230,12 @@ impl FileFetcher { cache: Default::default(), cache_setting, http_cache, - http_client: create_http_client(get_user_agent(), ca_data)?, + http_client: create_http_client( + get_user_agent(), + root_store, + None, + None, + )?, blob_store, }) } diff --git a/cli/flags.rs b/cli/flags.rs index 69df8f6fec1899..f861aa5b9a3f98 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -137,6 +137,7 @@ pub struct Flags { pub allow_read: Option>, pub allow_run: Option>, pub allow_write: Option>, + pub ca_stores: Option>, pub ca_file: Option, pub cache_blocklist: Vec, pub cached_only: bool, @@ -264,6 +265,9 @@ static ENV_VARIABLES_HELP: &str = r#"ENVIRONMENT VARIABLES: hostnames to use when fetching remote modules from private repositories (e.g. "abcde12345@deno.land;54321edcba@github.com") + DENO_TLS_CA_STORE Comma-seperated list of order dependent certificate stores + (system, mozilla) + (defaults to mozilla) DENO_CERT Load certificate authority from PEM encoded file DENO_DIR Set the cache directory DENO_INSTALL_ROOT Set deno install's output directory diff --git a/cli/http_util.rs b/cli/http_util.rs index a199f20c8e67fb..a178f56c4258c9 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -1,46 +1,18 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. - use crate::auth_tokens::AuthToken; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::url::Url; -use deno_runtime::deno_fetch::reqwest; -use deno_runtime::deno_fetch::reqwest::header::HeaderMap; use deno_runtime::deno_fetch::reqwest::header::HeaderValue; use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH; use deno_runtime::deno_fetch::reqwest::header::LOCATION; -use deno_runtime::deno_fetch::reqwest::header::USER_AGENT; -use deno_runtime::deno_fetch::reqwest::redirect::Policy; use deno_runtime::deno_fetch::reqwest::Client; use deno_runtime::deno_fetch::reqwest::StatusCode; use log::debug; use std::collections::HashMap; -/// Create new instance of async reqwest::Client. This client supports -/// proxies and doesn't follow redirects. -pub fn create_http_client( - user_agent: String, - ca_data: Option>, -) -> Result { - let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, user_agent.parse().unwrap()); - let mut builder = Client::builder() - .redirect(Policy::none()) - .default_headers(headers) - .use_rustls_tls(); - - if let Some(ca_data) = ca_data { - let cert = reqwest::Certificate::from_pem(&ca_data)?; - builder = builder.add_root_certificate(cert); - } - - builder - .build() - .map_err(|e| generic_error(format!("Unable to build http client: {}", e))) -} - /// Construct the next uri based on base uri and location header fragment /// See fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { @@ -168,10 +140,11 @@ pub async fn fetch_once( mod tests { use super::*; use crate::version; + use deno_core::http::create_http_client; use std::fs::read; fn create_test_client(ca_data: Option>) -> Client { - create_http_client("test_client".to_string(), ca_data).unwrap() + create_http_client("test_client".to_string(), None, ca_data, None).unwrap() } #[tokio::test] @@ -362,6 +335,7 @@ mod tests { let client = create_http_client( version::get_user_agent(), + None, Some( read( test_util::root_path() @@ -371,6 +345,7 @@ mod tests { ) .unwrap(), ), + None, ) .unwrap(); let result = fetch_once(FetchOnceArgs { @@ -400,6 +375,7 @@ mod tests { .unwrap(); let client = create_http_client( version::get_user_agent(), + None, Some( read( test_util::root_path() @@ -409,6 +385,7 @@ mod tests { ) .unwrap(), ), + None, ) .unwrap(); let result = fetch_once(FetchOnceArgs { @@ -437,6 +414,7 @@ mod tests { let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); let client = create_http_client( version::get_user_agent(), + None, Some( read( test_util::root_path() @@ -446,6 +424,7 @@ mod tests { ) .unwrap(), ), + None, ) .unwrap(); let result = fetch_once(FetchOnceArgs { @@ -488,6 +467,7 @@ mod tests { .unwrap(); let client = create_http_client( version::get_user_agent(), + None, Some( read( test_util::root_path() @@ -497,6 +477,7 @@ mod tests { ) .unwrap(), ), + None, ) .unwrap(); let result = fetch_once(FetchOnceArgs { diff --git a/cli/main.rs b/cli/main.rs index f697724376f865..ae7c56473df485 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -109,7 +109,7 @@ fn create_web_worker_callback( .log_level .map_or(false, |l| l == log::Level::Debug), unstable: program_state.flags.unstable, - ca_data: program_state.ca_data.clone(), + root_store: program_state.root_store.clone(), user_agent: version::get_user_agent(), seed: program_state.flags.seed, module_loader, @@ -188,7 +188,7 @@ pub fn create_main_worker( .log_level .map_or(false, |l| l == log::Level::Debug), unstable: program_state.flags.unstable, - ca_data: program_state.ca_data.clone(), + root_store: program_state.root_store.clone(), user_agent: version::get_user_agent(), seed: program_state.flags.seed, js_error_create_fn: Some(js_error_create_fn), diff --git a/cli/program_state.rs b/cli/program_state.rs index 264b2272997c80..72e2b6c06dd142 100644 --- a/cli/program_state.rs +++ b/cli/program_state.rs @@ -27,15 +27,18 @@ use deno_core::error::AnyError; use deno_core::error::Context; use deno_core::parking_lot::Mutex; use deno_core::resolve_url; +use deno_core::rustls::RootCertStore; use deno_core::url::Url; use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use log::debug; use log::warn; +use rustls_native_certs::load_native_certs; use std::collections::HashMap; use std::collections::HashSet; use std::env; -use std::fs::read; +use std::fs::File; +use std::io::BufReader; use std::sync::Arc; /// This structure represents state of single "deno" program. @@ -53,7 +56,7 @@ pub struct ProgramState { pub maybe_config_file: Option, pub maybe_import_map: Option, pub maybe_inspector_server: Option>, - pub ca_data: Option>, + pub root_store: Option, pub blob_store: BlobStore, pub broadcast_channel: InMemoryBroadcastChannel, pub shared_array_buffer_store: SharedArrayBufferStore, @@ -65,11 +68,50 @@ impl ProgramState { let dir = deno_dir::DenoDir::new(custom_root)?; let deps_cache_location = dir.root.join("deps"); let http_cache = http_cache::HttpCache::new(&deps_cache_location); + + let default_ca_stores = vec!["mozilla".to_string()]; + let supported_ca_stores = vec!["mozilla".to_string(), "system".to_string()]; + + let mut root_store = RootCertStore::empty(); + let ca_stores: Vec = flags + .ca_stores + .clone() + .or_else(|| { + let env_ca_store = env::var("DENO_TLS_CA_STORE"); + match env_ca_store { + Ok(arg) => Some(arg.split(',').map(|s| s.to_string()).collect()), + _ => None, + } + }) + .or_else(|| Some(default_ca_stores.clone())) + .unwrap() + .iter() + .filter(|store| supported_ca_stores.contains(&store)) + .map(|s| s.to_string()) + .collect(); + + for store in ca_stores.iter() { + match store.as_str() { + "mozilla" => { + root_store.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); + } + "system" => { + let roots = load_native_certs() + .expect("could not load platform certs") + .roots; + root_store.roots.extend(roots); + } + _ => (), + } + } + + // Should this do a permissions check? let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok()); - let ca_data = match &ca_file { - Some(ca_file) => Some(read(ca_file).context("Failed to open ca file")?), - None => None, - }; + if let Some(ca_file) = ca_file { + let certfile = File::open(&ca_file)?; + let mut reader = BufReader::new(certfile); + root_store.add_pem_file(&mut reader).unwrap(); + } let cache_usage = if flags.cached_only { CacheSetting::Only @@ -89,7 +131,7 @@ impl ProgramState { http_cache, cache_usage, !flags.no_remote, - ca_data.clone(), + Some(root_store.clone()), blob_store.clone(), )?; @@ -148,7 +190,7 @@ impl ProgramState { maybe_config_file, maybe_import_map, maybe_inspector_server, - ca_data, + root_store: Some(root_store.clone()), blob_store, broadcast_channel, shared_array_buffer_store, diff --git a/cli/standalone.rs b/cli/standalone.rs index c8918563f9df5c..58ecdbc2595d15 100644 --- a/cli/standalone.rs +++ b/cli/standalone.rs @@ -13,6 +13,7 @@ use deno_core::error::uri_error; use deno_core::error::AnyError; use deno_core::error::Context; use deno_core::futures::FutureExt; +use deno_core::http::create_default_root_store; use deno_core::located_script_name; use deno_core::resolve_url; use deno_core::serde::Deserialize; @@ -34,6 +35,8 @@ use std::cell::RefCell; use std::convert::TryInto; use std::env::current_exe; use std::fs::File; +use std::io::BufReader; +use std::io::Cursor; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; @@ -51,6 +54,7 @@ pub struct Metadata { pub location: Option, pub v8_flags: Vec, pub log_level: Option, + pub ca_stores: Option>, pub ca_data: Option>, } @@ -201,6 +205,7 @@ fn metadata_to_flags(metadata: &Metadata) -> Flags { allow_write: permissions.allow_write, v8_flags: metadata.v8_flags.clone(), log_level: metadata.log_level, + ca_stores: metadata.ca_stores.clone(), ..Default::default() } } @@ -227,13 +232,24 @@ pub async fn run( .collect::>(), ); + let mut root_store = program_state + .root_store + .clone() + .or_else(|| Some(create_default_root_store())) + .unwrap(); + + if let Some(cert) = metadata.ca_data { + let reader = &mut BufReader::new(Cursor::new(cert)); + root_store.add_pem_file(reader).unwrap(); + } + let options = WorkerOptions { apply_source_maps: false, args: metadata.argv, debug_flag: metadata.log_level.map_or(false, |l| l == log::Level::Debug), user_agent: version::get_user_agent(), unstable: metadata.unstable, - ca_data: metadata.ca_data, + root_store: Some(root_store), seed: metadata.seed, js_error_create_fn: None, create_web_worker_cb, diff --git a/cli/tests/integration/mod.rs b/cli/tests/integration/mod.rs index c11d26dc9cf85e..a8fc75b754e3f9 100644 --- a/cli/tests/integration/mod.rs +++ b/cli/tests/integration/mod.rs @@ -1,9 +1,9 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. use crate::itest; +use deno_core::rustls; use deno_core::url; -use deno_runtime::deno_net::ops_tls::rustls; -use deno_runtime::deno_net::ops_tls::webpki; +use deno_core::webpki; use deno_runtime::deno_net::ops_tls::TlsStream; use std::fs; use std::io::BufReader; diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index 259c61222a0857..2c057ff624b4c1 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -100,6 +100,7 @@ pub fn create_standalone_binary( permissions: flags.clone().into(), v8_flags: flags.v8_flags.clone(), log_level: flags.log_level, + ca_stores: flags.ca_stores, ca_data, }; let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec(); @@ -205,6 +206,7 @@ pub fn compile_to_runtime_flags( allow_read: flags.allow_read, allow_run: flags.allow_run, allow_write: flags.allow_write, + ca_stores: flags.ca_stores, ca_file: flags.ca_file, cache_blocklist: vec![], cached_only: false, diff --git a/core/Cargo.toml b/core/Cargo.toml index 5c30f5a4df76e0..713568122ac161 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -22,11 +22,16 @@ libc = "0.2.98" log = "0.4.14" parking_lot = "0.11.1" pin-project = "1.0.7" +reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] } +rustls = "0.19.0" +rustls-native-certs = "0.5.0" rusty_v8 = "0.25.3" serde = { version = "1.0.126", features = ["derive"] } serde_json = { version = "1.0.64", features = ["preserve_order"] } serde_v8 = { version = "0.8.0" } url = { version = "2.2.2", features = ["serde"] } +webpki = "0.21.4" +webpki-roots = "0.21.1" [[example]] name = "http_bench_json_ops" diff --git a/core/http.rs b/core/http.rs new file mode 100644 index 00000000000000..3624a6d197e31f --- /dev/null +++ b/core/http.rs @@ -0,0 +1,117 @@ +pub use reqwest; +pub use rustls; +pub use webpki; + +use crate::error::generic_error; +use crate::error::AnyError; +use crate::parking_lot::Mutex; + +use reqwest::Client; +use rustls::ClientConfig; +use rustls::RootCertStore; +use rustls::StoresClientSessions; +// use rustls_native_certs::load_native_certs; +use reqwest::header::HeaderMap; +use reqwest::header::USER_AGENT; +use reqwest::redirect::Policy; +use serde::Deserialize; +use std::collections::HashMap; +use std::io::BufReader; +use std::io::Cursor; +use std::sync::Arc; + +#[derive(Deserialize, Default, Debug, Clone)] +#[serde(rename_all = "camelCase")] +#[serde(default)] +pub struct Proxy { + pub url: String, + pub basic_auth: Option, +} + +#[derive(Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct BasicAuth { + pub username: String, + pub password: String, +} + +lazy_static::lazy_static! { + static ref CLIENT_SESSION_MEMORY_CACHE: Arc = + Arc::new(ClientSessionMemoryCache::default()); +} + +#[derive(Default)] +struct ClientSessionMemoryCache(Mutex, Vec>>); + +impl StoresClientSessions for ClientSessionMemoryCache { + fn get(&self, key: &[u8]) -> Option> { + self.0.lock().get(key).cloned() + } + + fn put(&self, key: Vec, value: Vec) -> bool { + let mut sessions = self.0.lock(); + // TODO(bnoordhuis) Evict sessions LRU-style instead of arbitrarily. + while sessions.len() >= 1024 { + let key = sessions.keys().next().unwrap().clone(); + sessions.remove(&key); + } + sessions.insert(key, value); + true + } +} + +pub fn create_default_root_store() -> RootCertStore { + let mut root_store = RootCertStore::empty(); + // todo: Consider also loading the system keychain here + root_store.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); + root_store +} + +pub fn create_client_config( + root_store: Option, + ca_data: Option>, +) -> Result { + let mut tls_config = ClientConfig::new(); + tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone()); + tls_config.root_store = root_store + .or_else(|| Some(create_default_root_store())) + .unwrap(); + + // If a custom cert is specified, add it to the store + if let Some(cert) = ca_data { + let reader = &mut BufReader::new(Cursor::new(cert)); + tls_config.root_store.add_pem_file(reader).unwrap(); + } + + Ok(tls_config) +} + +/// Create new instance of async reqwest::Client. This client supports +/// proxies and doesn't follow redirects. +pub fn create_http_client( + user_agent: String, + root_store: Option, + ca_data: Option>, + proxy: Option, +) -> Result { + let tls_config = create_client_config(root_store, ca_data)?; + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, user_agent.parse().unwrap()); + let mut builder = Client::builder() + .redirect(Policy::none()) + .default_headers(headers) + .use_preconfigured_tls(tls_config); + + if let Some(proxy) = proxy { + let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?; + if let Some(basic_auth) = &proxy.basic_auth { + reqwest_proxy = + reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password); + } + builder = builder.proxy(reqwest_proxy); + } + + builder + .build() + .map_err(|e| generic_error(format!("Unable to build http client: {}", e))) +} diff --git a/core/lib.rs b/core/lib.rs index a9f6d5d8bd220f..429425d01dc22e 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -6,6 +6,7 @@ pub mod error; mod extensions; mod flags; mod gotham_state; +pub mod http; mod inspector; mod module_specifier; mod modules; @@ -19,6 +20,7 @@ mod runtime; // Re-exports pub use futures; pub use parking_lot; +pub use rustls; pub use rusty_v8 as v8; pub use serde; pub use serde_json; @@ -26,6 +28,7 @@ pub use serde_v8; pub use serde_v8::Buffer as ZeroCopyBuf; pub use serde_v8::ByteString; pub use url; +pub use webpki; pub use crate::async_cancel::CancelFuture; pub use crate::async_cancel::CancelHandle; @@ -41,6 +44,8 @@ pub use crate::async_cell::AsyncRefFuture; pub use crate::async_cell::RcLike; pub use crate::async_cell::RcRef; pub use crate::flags::v8_set_flags; +pub use crate::http::create_client_config; +pub use crate::http::create_http_client; pub use crate::inspector::InspectorSessionProxy; pub use crate::inspector::JsRuntimeInspector; pub use crate::inspector::LocalInspectorSession; diff --git a/extensions/fetch/Cargo.toml b/extensions/fetch/Cargo.toml index 4cc4fee2ce5f44..ba24d0c8801b51 100644 --- a/extensions/fetch/Cargo.toml +++ b/extensions/fetch/Cargo.toml @@ -19,6 +19,7 @@ data-url = "0.1.0" deno_core = { version = "0.95.0", path = "../../core" } deno_web = { version = "0.44.0", path = "../web" } http = "0.2.4" +lazy_static = "1.4.0" reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] } serde = { version = "1.0.126", features = ["derive"] } tokio = { version = "1.8.1", features = ["full"] } diff --git a/extensions/fetch/lib.rs b/extensions/fetch/lib.rs index f870c58dc70df1..952f71a9f2f2d6 100644 --- a/extensions/fetch/lib.rs +++ b/extensions/fetch/lib.rs @@ -1,16 +1,18 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. use deno_core::error::bad_resource_id; -use deno_core::error::generic_error; use deno_core::error::null_opbuf; use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::futures::Future; use deno_core::futures::Stream; use deno_core::futures::StreamExt; +use deno_core::http::create_http_client; +use deno_core::http::Proxy; use deno_core::include_js_files; use deno_core::op_async; use deno_core::op_sync; +use deno_core::rustls::RootCertStore; use deno_core::url::Url; use deno_core::AsyncRefCell; use deno_core::ByteString; @@ -28,12 +30,9 @@ use deno_core::ZeroCopyBuf; use data_url::DataUrl; use deno_web::BlobStore; use http::header::CONTENT_LENGTH; -use reqwest::header::HeaderMap; use reqwest::header::HeaderName; use reqwest::header::HeaderValue; use reqwest::header::HOST; -use reqwest::header::USER_AGENT; -use reqwest::redirect::Policy; use reqwest::Body; use reqwest::Client; use reqwest::Method; @@ -58,7 +57,7 @@ pub use reqwest; // Re-export reqwest pub fn init( user_agent: String, - ca_data: Option>, + root_store: Option, proxy: Option, ) -> Extension { Extension::builder() @@ -82,12 +81,17 @@ pub fn init( ]) .state(move |state| { state.put::({ - create_http_client(user_agent.clone(), ca_data.clone(), proxy.clone()) - .unwrap() + create_http_client( + user_agent.clone(), + root_store.clone(), + None, + proxy.clone(), + ) + .unwrap() }); state.put::(HttpClientDefaults { - ca_data: ca_data.clone(), user_agent: user_agent.clone(), + root_store: root_store.clone(), proxy: proxy.clone(), }); Ok(()) @@ -97,7 +101,7 @@ pub fn init( pub struct HttpClientDefaults { pub user_agent: String, - pub ca_data: Option>, + pub root_store: Option, pub proxy: Option, } @@ -492,26 +496,12 @@ impl HttpClientResource { #[serde(rename_all = "camelCase")] #[serde(default)] pub struct CreateHttpClientOptions { + ca_stores: Option>, ca_file: Option, ca_data: Option, proxy: Option, } -#[derive(Deserialize, Default, Debug, Clone)] -#[serde(rename_all = "camelCase")] -#[serde(default)] -pub struct Proxy { - pub url: String, - pub basic_auth: Option, -} - -#[derive(Deserialize, Default, Debug, Clone)] -#[serde(default)] -pub struct BasicAuth { - pub username: String, - pub password: String, -} - pub fn op_create_http_client( state: &mut OpState, args: CreateHttpClientOptions, @@ -532,12 +522,12 @@ where } let defaults = state.borrow::(); - let cert_data = get_cert_data(args.ca_file.as_deref(), args.ca_data.as_deref())?; let client = create_http_client( defaults.user_agent.clone(), - cert_data.or_else(|| defaults.ca_data.clone()), + defaults.root_store.clone(), + cert_data, args.proxy, ) .unwrap(); @@ -560,36 +550,3 @@ fn get_cert_data( Ok(None) } } - -/// Create new instance of async reqwest::Client. This client supports -/// proxies and doesn't follow redirects. -pub fn create_http_client( - user_agent: String, - ca_data: Option>, - proxy: Option, -) -> Result { - let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, user_agent.parse().unwrap()); - let mut builder = Client::builder() - .redirect(Policy::none()) - .default_headers(headers) - .use_rustls_tls(); - - if let Some(ca_data) = ca_data { - let cert = reqwest::Certificate::from_pem(&ca_data)?; - builder = builder.add_root_certificate(cert); - } - - if let Some(proxy) = proxy { - let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?; - if let Some(basic_auth) = &proxy.basic_auth { - reqwest_proxy = - reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password); - } - builder = builder.proxy(reqwest_proxy); - } - - builder - .build() - .map_err(|e| generic_error(format!("Unable to build http client: {}", e))) -} diff --git a/extensions/net/Cargo.toml b/extensions/net/Cargo.toml index 9c6d5d161ed13e..e9c48c05c7babf 100644 --- a/extensions/net/Cargo.toml +++ b/extensions/net/Cargo.toml @@ -18,10 +18,7 @@ deno_core = { version = "0.95.0", path = "../../core" } log = "0.4.14" lazy_static = "1.4.0" -rustls = "0.19.0" serde = { version = "1.0.126", features = ["derive"] } tokio = { version = "1.8.1", features = ["full"] } -webpki = "0.21.4" -webpki-roots = "0.21.1" trust-dns-proto = "0.20.3" trust-dns-resolver = { version = "0.20.3", features = ["tokio-runtime", "serde-config"] } diff --git a/extensions/net/lib.rs b/extensions/net/lib.rs index 11d0b4493682ec..1d82ce8503e9bd 100644 --- a/extensions/net/lib.rs +++ b/extensions/net/lib.rs @@ -9,6 +9,7 @@ pub mod resolve_addr; use deno_core::error::AnyError; use deno_core::include_js_files; +use deno_core::rustls::RootCertStore; use deno_core::Extension; use deno_core::OpState; use std::cell::RefCell; @@ -90,20 +91,17 @@ pub fn get_unstable_declaration() -> PathBuf { #[derive(Clone)] pub struct DefaultTlsOptions { - pub ca_data: Option>, + pub root_store: Option, } pub fn init( - ca_data: Option>, + root_store: Option, unstable: bool, ) -> Extension { let mut ops_to_register = vec![]; ops_to_register.extend(io::init()); ops_to_register.extend(ops::init::

()); ops_to_register.extend(ops_tls::init::

()); - - let default_tls_options = DefaultTlsOptions { ca_data }; - Extension::builder() .js(include_js_files!( prefix "deno:extensions/net", @@ -113,7 +111,9 @@ pub fn init( )) .ops(ops_to_register) .state(move |state| { - state.put(default_tls_options.clone()); + state.put(DefaultTlsOptions { + root_store: root_store.clone(), + }); state.put(UnstableChecker { unstable }); Ok(()) }) diff --git a/extensions/net/ops_tls.rs b/extensions/net/ops_tls.rs index a082f7f620392e..4d87543f6bcec8 100644 --- a/extensions/net/ops_tls.rs +++ b/extensions/net/ops_tls.rs @@ -1,8 +1,5 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. -pub use rustls; -pub use webpki; - use crate::io::TcpStreamResource; use crate::io::TlsStreamResource; use crate::ops::IpAddr; @@ -27,9 +24,22 @@ use deno_core::futures::task::Poll; use deno_core::futures::task::RawWaker; use deno_core::futures::task::RawWakerVTable; use deno_core::futures::task::Waker; +use deno_core::http::create_client_config; use deno_core::op_async; use deno_core::op_sync; use deno_core::parking_lot::Mutex; +use deno_core::rustls::internal::pemfile::certs; +use deno_core::rustls::internal::pemfile::pkcs8_private_keys; +use deno_core::rustls::internal::pemfile::rsa_private_keys; +use deno_core::rustls::Certificate; +use deno_core::rustls::ClientConfig; +use deno_core::rustls::ClientSession; +use deno_core::rustls::NoClientAuth; +use deno_core::rustls::PrivateKey; +use deno_core::rustls::ServerConfig; +use deno_core::rustls::ServerSession; +use deno_core::rustls::Session; +use deno_core::webpki::DNSNameRef; use deno_core::AsyncRefCell; use deno_core::CancelHandle; use deno_core::CancelTryFuture; @@ -41,27 +51,13 @@ use deno_core::ResourceId; use io::Error; use io::Read; use io::Write; -use rustls::internal::pemfile::certs; -use rustls::internal::pemfile::pkcs8_private_keys; -use rustls::internal::pemfile::rsa_private_keys; -use rustls::Certificate; -use rustls::ClientConfig; -use rustls::ClientSession; -use rustls::NoClientAuth; -use rustls::PrivateKey; -use rustls::ServerConfig; -use rustls::ServerSession; -use rustls::Session; -use rustls::StoresClientSessions; use serde::Deserialize; use std::borrow::Cow; use std::cell::RefCell; -use std::collections::HashMap; use std::convert::From; use std::fs::File; use std::io; use std::io::BufReader; -use std::io::Cursor; use std::io::ErrorKind; use std::ops::Deref; use std::ops::DerefMut; @@ -76,32 +72,6 @@ use tokio::io::ReadBuf; use tokio::net::TcpListener; use tokio::net::TcpStream; use tokio::task::spawn_local; -use webpki::DNSNameRef; - -lazy_static::lazy_static! { - static ref CLIENT_SESSION_MEMORY_CACHE: Arc = - Arc::new(ClientSessionMemoryCache::default()); -} - -#[derive(Default)] -struct ClientSessionMemoryCache(Mutex, Vec>>); - -impl StoresClientSessions for ClientSessionMemoryCache { - fn get(&self, key: &[u8]) -> Option> { - self.0.lock().get(key).cloned() - } - - fn put(&self, key: Vec, value: Vec) -> bool { - let mut sessions = self.0.lock(); - // TODO(bnoordhuis) Evict sessions LRU-style instead of arbitrarily. - while sessions.len() >= 1024 { - let key = sessions.keys().next().unwrap().clone(); - sessions.remove(&key); - } - sessions.insert(key, value); - true - } -} #[derive(Debug)] enum TlsSession { @@ -703,8 +673,6 @@ where n => n, }; let cert_file = args.cert_file.as_deref(); - - let default_tls_options; { super::check_unstable2(&state, "Deno.startTls"); let mut s = state.borrow_mut(); @@ -713,12 +681,25 @@ where if let Some(path) = cert_file { permissions.check_read(Path::new(path))?; } - default_tls_options = s.borrow::().clone(); } + let ca_data = match cert_file { + Some(path) => { + let mut buf = Vec::new(); + File::open(path)?.read_to_end(&mut buf)?; + Some(buf) + } + _ => None, + }; + let hostname_dns = DNSNameRef::try_from_ascii_str(hostname) .map_err(|_| invalid_hostname(hostname))?; + let root_store = state + .borrow() + .borrow::() + .root_store + .clone(); let resource_rc = state .borrow_mut() .resource_table @@ -732,22 +713,7 @@ where let local_addr = tcp_stream.local_addr()?; let remote_addr = tcp_stream.peer_addr()?; - let mut tls_config = ClientConfig::new(); - tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone()); - tls_config - .root_store - .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); - if let Some(ca_data) = default_tls_options.ca_data { - let reader = &mut Cursor::new(ca_data); - tls_config.root_store.add_pem_file(reader).unwrap(); - }; - if let Some(path) = cert_file { - let key_file = File::open(path)?; - let reader = &mut BufReader::new(key_file); - tls_config.root_store.add_pem_file(reader).unwrap(); - } - let tls_config = Arc::new(tls_config); - + let tls_config = Arc::new(create_client_config(root_store, ca_data)?); let tls_stream = TlsStream::new_client_side(tcp_stream, &tls_config, hostname_dns); @@ -786,8 +752,6 @@ where }; let port = args.port; let cert_file = args.cert_file.as_deref(); - - let default_tls_options; { let mut s = state.borrow_mut(); let permissions = s.borrow_mut::(); @@ -795,9 +759,22 @@ where if let Some(path) = cert_file { permissions.check_read(Path::new(path))?; } - default_tls_options = s.borrow::().clone(); } + let ca_data = match cert_file { + Some(path) => { + let mut buf = Vec::new(); + File::open(path)?.read_to_end(&mut buf)?; + Some(buf) + } + _ => None, + }; + + let root_store = state + .borrow() + .borrow::() + .root_store + .clone(); let hostname_dns = DNSNameRef::try_from_ascii_str(hostname) .map_err(|_| invalid_hostname(hostname))?; @@ -808,23 +785,7 @@ where let tcp_stream = TcpStream::connect(connect_addr).await?; let local_addr = tcp_stream.local_addr()?; let remote_addr = tcp_stream.peer_addr()?; - - let mut tls_config = ClientConfig::new(); - tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone()); - tls_config - .root_store - .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); - if let Some(ca_data) = default_tls_options.ca_data { - let reader = &mut Cursor::new(ca_data); - tls_config.root_store.add_pem_file(reader).unwrap(); - }; - if let Some(path) = cert_file { - let key_file = File::open(path)?; - let reader = &mut BufReader::new(key_file); - tls_config.root_store.add_pem_file(reader).unwrap(); - } - let tls_config = Arc::new(tls_config); - + let tls_config = Arc::new(create_client_config(root_store, ca_data)?); let tls_stream = TlsStream::new_client_side(tcp_stream, &tls_config, hostname_dns); diff --git a/extensions/websocket/Cargo.toml b/extensions/websocket/Cargo.toml index 818f8135a52a4b..9294829c6ec161 100644 --- a/extensions/websocket/Cargo.toml +++ b/extensions/websocket/Cargo.toml @@ -21,5 +21,3 @@ tokio = { version = "1.8.1", features = ["full"] } tokio-rustls = "0.22.0" tokio-tungstenite = { version = "0.14.0", features = ["rustls-tls"] } hyper = { version = "0.14.9" } -webpki = "0.21.4" -webpki-roots = "0.21.1" diff --git a/extensions/websocket/lib.rs b/extensions/websocket/lib.rs index f5bf15c79b97c3..f92cac0a6da6b3 100644 --- a/extensions/websocket/lib.rs +++ b/extensions/websocket/lib.rs @@ -9,10 +9,12 @@ use deno_core::futures::stream::SplitSink; use deno_core::futures::stream::SplitStream; use deno_core::futures::SinkExt; use deno_core::futures::StreamExt; +use deno_core::http::create_client_config; use deno_core::include_js_files; use deno_core::op_async; use deno_core::op_sync; use deno_core::url; +use deno_core::webpki::DNSNameRef; use deno_core::AsyncRefCell; use deno_core::CancelFuture; use deno_core::CancelHandle; @@ -28,25 +30,23 @@ use serde::Deserialize; use serde::Serialize; use std::borrow::Cow; use std::cell::RefCell; -use std::io::BufReader; -use std::io::Cursor; use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; use tokio::net::TcpStream; -use tokio_rustls::{rustls::ClientConfig, TlsConnector}; +use tokio_rustls::rustls::RootCertStore; +use tokio_rustls::TlsConnector; use tokio_tungstenite::tungstenite::{ handshake::client::Response, protocol::frame::coding::CloseCode, protocol::CloseFrame, Message, }; use tokio_tungstenite::MaybeTlsStream; use tokio_tungstenite::{client_async, WebSocketStream}; -use webpki::DNSNameRef; pub use tokio_tungstenite; // Re-export tokio_tungstenite #[derive(Clone)] -pub struct WsCaData(pub Vec); +pub struct WsRootStore(pub Option); #[derive(Clone)] pub struct WsUserAgent(pub String); @@ -197,7 +197,7 @@ where ); } - let ws_ca_data = state.borrow().try_borrow::().cloned(); + let root_store = state.borrow().borrow::().0.clone(); let user_agent = state.borrow().borrow::().0.clone(); let uri: Uri = args.url.parse()?; let mut request = Request::builder().method(Method::GET).uri(&uri); @@ -221,17 +221,8 @@ where let socket: MaybeTlsStream = match uri.scheme_str() { Some("ws") => MaybeTlsStream::Plain(tcp_socket), Some("wss") => { - let mut config = ClientConfig::new(); - config - .root_store - .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); - - if let Some(ws_ca_data) = ws_ca_data { - let reader = &mut BufReader::new(Cursor::new(ws_ca_data.0)); - config.root_store.add_pem_file(reader).unwrap(); - } - - let tls_connector = TlsConnector::from(Arc::new(config)); + let tls_config = create_client_config(root_store, None)?; + let tls_connector = TlsConnector::from(Arc::new(tls_config)); let dnsname = DNSNameRef::try_from_ascii_str(domain) .map_err(|_| invalid_hostname(domain))?; let tls_socket = tls_connector.connect(dnsname, tcp_socket).await?; @@ -385,7 +376,7 @@ pub async fn op_ws_next_event( pub fn init( user_agent: String, - ca_data: Option>, + root_store: Option, ) -> Extension { Extension::builder() .js(include_js_files!( @@ -404,9 +395,7 @@ pub fn init( ]) .state(move |state| { state.put::(WsUserAgent(user_agent.clone())); - if let Some(ca_data) = ca_data.clone() { - state.put::(WsCaData(ca_data)); - } + state.put::(WsRootStore(root_store.clone())); Ok(()) }) .build() diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs index 7078090373df38..560c863d064739 100644 --- a/runtime/examples/hello_runtime.rs +++ b/runtime/examples/hello_runtime.rs @@ -27,7 +27,7 @@ async fn main() -> Result<(), AnyError> { args: vec![], debug_flag: false, unstable: false, - ca_data: None, + root_store: None, user_agent: "hello_runtime".to_string(), seed: None, js_error_create_fn: None, diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index bcbf71692901d0..48bc85bba69e0d 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -14,6 +14,7 @@ use deno_core::futures::future::poll_fn; use deno_core::futures::future::FutureExt; use deno_core::futures::stream::StreamExt; use deno_core::located_script_name; +use deno_core::rustls::RootCertStore; use deno_core::serde::Deserialize; use deno_core::serde::Serialize; use deno_core::serde_json; @@ -252,7 +253,7 @@ pub struct WebWorkerOptions { pub args: Vec, pub debug_flag: bool, pub unstable: bool, - pub ca_data: Option>, + pub root_store: Option, pub user_agent: String, pub seed: Option, pub module_loader: Rc, @@ -299,12 +300,12 @@ impl WebWorker { deno_web::init(options.blob_store.clone(), Some(main_module.clone())), deno_fetch::init::( options.user_agent.clone(), - options.ca_data.clone(), + options.root_store.clone(), None, ), deno_websocket::init::( options.user_agent.clone(), - options.ca_data.clone(), + options.root_store.clone(), ), deno_broadcast_channel::init( options.broadcast_channel.clone(), @@ -333,7 +334,7 @@ impl WebWorker { ops::fs_events::init(), ops::fs::init(), deno_net::init::( - options.ca_data.clone(), + options.root_store.clone(), options.unstable, ), ops::os::init(), diff --git a/runtime/worker.rs b/runtime/worker.rs index 18ba348ff5ef61..907487e0f37eb5 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -10,6 +10,7 @@ use deno_core::error::AnyError; use deno_core::futures::stream::StreamExt; use deno_core::futures::Future; use deno_core::located_script_name; +use deno_core::rustls::RootCertStore; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::url::Url; @@ -50,7 +51,7 @@ pub struct WorkerOptions { pub args: Vec, pub debug_flag: bool, pub unstable: bool, - pub ca_data: Option>, + pub root_store: Option, pub user_agent: String, pub seed: Option, pub module_loader: Rc, @@ -99,12 +100,12 @@ impl MainWorker { deno_web::init(options.blob_store.clone(), options.location.clone()), deno_fetch::init::( options.user_agent.clone(), - options.ca_data.clone(), + options.root_store.clone(), None, ), deno_websocket::init::( options.user_agent.clone(), - options.ca_data.clone(), + options.root_store.clone(), ), deno_webstorage::init(options.origin_storage_dir.clone()), deno_crypto::init(options.seed), @@ -123,7 +124,10 @@ impl MainWorker { ops::fs::init(), ops::io::init(), ops::io::init_stdio(), - deno_net::init::(options.ca_data.clone(), options.unstable), + deno_net::init::( + options.root_store.clone(), + options.unstable, + ), ops::os::init(), ops::permissions::init(), ops::plugin::init(), @@ -292,7 +296,7 @@ mod tests { args: vec![], debug_flag: false, unstable: false, - ca_data: None, + root_store: None, seed: None, js_error_create_fn: None, create_web_worker_cb: Arc::new(|_| unreachable!()),