diff --git a/.gitignore b/.gitignore index abc8cf51..f5636a63 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ dhat-heap.json .vscode .idea .cover -bleeper.user.toml \ No newline at end of file +bleeper.user.toml +pingora-proxy/tests/keys/server.crt +pingora-proxy/tests/utils/conf/keys/server.crt \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 2fdc570e..37b6228e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "pingora-lru", "pingora-openssl", "pingora-boringssl", + "pingora-rustls", "pingora-runtime", "pingora-ketama", "pingora-load-balancing", diff --git a/pingora-cache/Cargo.toml b/pingora-cache/Cargo.toml index 2dc3f2f7..253b311f 100644 --- a/pingora-cache/Cargo.toml +++ b/pingora-cache/Cargo.toml @@ -67,3 +67,4 @@ harness = false default = ["openssl"] openssl = ["pingora-core/openssl"] boringssl = ["pingora-core/boringssl"] +rustls = ["pingora-core/rustls"] \ No newline at end of file diff --git a/pingora-core/Cargo.toml b/pingora-core/Cargo.toml index c13823c3..75d14bdb 100644 --- a/pingora-core/Cargo.toml +++ b/pingora-core/Cargo.toml @@ -20,8 +20,9 @@ path = "src/lib.rs" [dependencies] pingora-runtime = { version = "0.3.0", path = "../pingora-runtime" } -pingora-openssl = { version = "0.3.0", path = "../pingora-openssl", optional = true } pingora-boringssl = { version = "0.3.0", path = "../pingora-boringssl", optional = true } +pingora-openssl = { version = "0.3.0", path = "../pingora-openssl", optional = true } +pingora-rustls = { version = "0.3.0", path = "../pingora-rustls", optional = true } pingora-pool = { version = "0.3.0", path = "../pingora-pool" } pingora-error = { version = "0.3.0", path = "../pingora-error" } pingora-timeout = { version = "0.3.0", path = "../pingora-timeout" } @@ -68,6 +69,7 @@ openssl-probe = "0.1" tokio-test = "0.4" zstd = "0" httpdate = "1" +x509-parser = { version = "0.16.0", optional = true } [dev-dependencies] matches = "0.1" @@ -81,4 +83,5 @@ jemallocator = "0.5" default = ["openssl"] openssl = ["pingora-openssl"] boringssl = ["pingora-boringssl"] -patched_http1 = [] +rustls = ["pingora-rustls", "dep:x509-parser"] +patched_http1 = [] \ No newline at end of file diff --git a/pingora-core/src/connectors/mod.rs b/pingora-core/src/connectors/mod.rs index 9bfd91b8..38da0cd9 100644 --- a/pingora-core/src/connectors/mod.rs +++ b/pingora-core/src/connectors/mod.rs @@ -14,26 +14,29 @@ //! Connecting to servers -pub mod http; -mod l4; -mod offload; -mod tls; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use futures::future::FutureExt; +use log::{debug, error, warn}; +use parking_lot::RwLock; +use tokio::io::AsyncReadExt; +use tokio::sync::Mutex; + +use offload::OffloadRuntime; +use pingora_error::{ErrorType::*, OrErr, Result}; +use pingora_pool::{ConnectionMeta, ConnectionPool}; +use crate::connectors::tls::{do_connect, Connector}; use crate::protocols::Stream; use crate::server::configuration::ServerConf; -use crate::tls::ssl::SslConnector; use crate::upstreams::peer::{Peer, ALPN}; -use l4::connect as l4_connect; -use log::{debug, error, warn}; -use offload::OffloadRuntime; -use parking_lot::RwLock; -use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use pingora_pool::{ConnectionMeta, ConnectionPool}; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::sync::Mutex; +pub mod http; +mod l4; +mod offload; +mod tls; /// The options to configure a [TransportConnector] #[derive(Clone)] @@ -123,7 +126,7 @@ impl ConnectorOptions { /// [TransportConnector] provides APIs to connect to servers via TCP or TLS with connection reuse pub struct TransportConnector { - tls_ctx: tls::Connector, + tls_ctx: Connector, connection_pool: Arc>>>, offload: Option, bind_to_v4: Vec, @@ -149,7 +152,7 @@ impl TransportConnector { .as_ref() .map_or_else(Vec::new, |o| o.bind_to_v6.clone()); TransportConnector { - tls_ctx: tls::Connector::new(options), + tls_ctx: Connector::new(options), connection_pool: Arc::new(ConnectionPool::new(pool_size)), offload: offload.map(|v| OffloadRuntime::new(v.0, v.1)), bind_to_v4, @@ -171,11 +174,13 @@ impl TransportConnector { let stream = if let Some(rt) = rt { let peer = peer.clone(); let tls_ctx = self.tls_ctx.clone(); - rt.spawn(async move { do_connect(&peer, bind_to, alpn_override, &tls_ctx.ctx).await }) - .await - .or_err(InternalError, "offload runtime failure")?? + rt.spawn( + async move { do_connect(&peer, bind_to, alpn_override, tls_ctx.context()).await }, + ) + .await + .or_err(InternalError, "offload runtime failure")?? } else { - do_connect(peer, bind_to, alpn_override, &self.tls_ctx.ctx).await? + do_connect(peer, bind_to, alpn_override, self.tls_ctx.context()).await? }; Ok(stream) @@ -268,46 +273,6 @@ impl TransportConnector { } } -// Perform the actual L4 and tls connection steps while respecting the peer's -// connection timeout if there is one -async fn do_connect( - peer: &P, - bind_to: Option, - alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result { - // Create the future that does the connections, but don't evaluate it until - // we decide if we need a timeout or not - let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx); - - match peer.total_connection_timeout() { - Some(t) => match pingora_timeout::timeout(t, connect_future).await { - Ok(res) => res, - Err(_) => Error::e_explain( - ConnectTimedout, - format!("connecting to server {peer}, total-connection timeout {t:?}"), - ), - }, - None => connect_future.await, - } -} - -// Perform the actual L4 and tls connection steps with no timeout -async fn do_connect_inner( - peer: &P, - bind_to: Option, - alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result { - let stream = l4_connect(peer, bind_to).await?; - if peer.tls() { - let tls_stream = tls::connect(stream, peer, alpn_override, tls_ctx).await?; - Ok(Box::new(tls_stream)) - } else { - Ok(Box::new(stream)) - } -} - struct PreferredHttpVersion { // TODO: shard to avoid the global lock versions: RwLock>, // @@ -337,9 +302,6 @@ impl PreferredHttpVersion { } } -use futures::future::FutureExt; -use tokio::io::AsyncReadExt; - /// Test whether a stream is already closed or not reusable (server sent unexpected data) fn test_reusable_stream(stream: &mut Stream) -> bool { let mut buf = [0; 1]; @@ -365,13 +327,14 @@ fn test_reusable_stream(stream: &mut Stream) -> bool { #[cfg(test)] mod tests { + use tokio::io::AsyncWriteExt; + use tokio::net::UnixListener; + use pingora_error::ErrorType; - use super::*; - use crate::tls::ssl::SslMethod; use crate::upstreams::peer::BasicPeer; - use tokio::io::AsyncWriteExt; - use tokio::net::UnixListener; + + use super::*; // 192.0.2.1 is effectively a black hole const BLACK_HOLE: &str = "192.0.2.1:79"; @@ -475,8 +438,8 @@ mod tests { /// This assumes that the connection will fail to on the peer and returns /// the decomposed error type and message async fn get_do_connect_failure_with_peer(peer: &BasicPeer) -> (ErrorType, String) { - let ssl_connector = SslConnector::builder(SslMethod::tls()).unwrap().build(); - let stream = do_connect(peer, None, None, &ssl_connector).await; + let connector = Connector::new(None); + let stream = do_connect(peer, None, None, connector.context()).await; match stream { Ok(_) => panic!("should throw an error"), Err(e) => ( @@ -509,6 +472,7 @@ mod tests { } #[tokio::test] + #[ignore] async fn test_do_connect_without_total_timeout() { let peer = BasicPeer::new(BLACK_HOLE); let (etype, context) = get_do_connect_failure_with_peer(&peer).await; diff --git a/pingora-core/src/connectors/tls.rs b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs similarity index 74% rename from pingora-core/src/connectors/tls.rs rename to pingora-core/src/connectors/tls/boringssl_openssl/mod.rs index 8a6bd635..a16c61c5 100644 --- a/pingora-core/src/connectors/tls.rs +++ b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs @@ -12,13 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use log::debug; -use pingora_error::{Error, ErrorType::*, OrErr, Result}; +//! BoringSSL & OpenSSL TLS connector specific implementation + +use std::any::Any; use std::sync::{Arc, Once}; -use super::ConnectorOptions; -use crate::protocols::ssl::client::handshake; -use crate::protocols::ssl::SslStream; +use log::debug; + +use pingora_error::ErrorType::{ConnectTimedout, InternalError}; +use pingora_error::{Error, OrErr, Result}; + +use crate::connectors::ConnectorOptions; +use crate::listeners::ALPN; +use crate::protocols::tls::boringssl_openssl::client::handshake; +use crate::protocols::tls::TlsStream; use crate::protocols::IO; use crate::tls::ext::{ add_host, clear_error_stack, ssl_add_chain_cert, ssl_set_groups_list, @@ -29,7 +36,10 @@ use crate::tls::ext::{ use crate::tls::ssl::SslCurve; use crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion}; use crate::tls::x509::store::X509StoreBuilder; -use crate::upstreams::peer::{Peer, ALPN}; +use crate::upstreams::peer::Peer; +use crate::utils::tls::boringssl_openssl::{der_to_private_key, der_to_x509}; + +use super::{replace_leftmost_underscore, Connector, TlsConnectorContext}; const CIPHER_LIST: &str = "AES-128-GCM-SHA256\ :AES-256-GCM-SHA384\ @@ -60,6 +70,7 @@ const SIGALG_LIST: &str = "ECDSA_SECP256R1_SHA256\ :RSA_PKCS1_SHA512\ :RSA_PKCS1_SHA1\ :ECDSA_SECP521R1_SHA512"; + /** * Enabled curves for ECDHE (signature key exchange). * As of 4/10/2023, the only addition to boringssl's defaults is SECP521R1. @@ -83,13 +94,17 @@ fn init_ssl_cert_env_vars() { INIT_CA_ENV.call_once(openssl_probe::init_ssl_cert_env_vars); } -#[derive(Clone)] -pub struct Connector { - pub(crate) ctx: Arc, // Arc to support clone -} +pub(super) struct TlsConnectorCtx(SslConnector); + +impl TlsConnectorContext for TlsConnectorCtx { + fn as_any(&self) -> &dyn Any { + self + } -impl Connector { - pub fn new(options: Option) -> Self { + fn build_connector(options: Option) -> Connector + where + Self: Sized, + { let mut builder = SslConnector::builder(SslMethod::tls()).unwrap(); // TODO: make these conf // Set supported ciphers. @@ -142,49 +157,24 @@ impl Connector { } Connector { - ctx: Arc::new(builder.build()), + ctx: Arc::new(TlsConnectorCtx(builder.build())), } } } -/* - OpenSSL considers underscores in hostnames non-compliant. - We replace the underscore in the leftmost label as we must support these - hostnames for wildcard matches and we have not patched OpenSSL. - - https://github.com/openssl/openssl/issues/12566 - - > The labels must follow the rules for ARPANET host names. They must - > start with a letter, end with a letter or digit, and have as interior - > characters only letters, digits, and hyphen. There are also some - > restrictions on the length. Labels must be 63 characters or less. - - https://datatracker.ietf.org/doc/html/rfc1034#section-3.5 -*/ -fn replace_leftmost_underscore(sni: &str) -> Option { - // wildcard is only leftmost label - if let Some((leftmost, rest)) = sni.split_once('.') { - // if not a subdomain or leftmost does not contain underscore return - if !rest.contains('.') || !leftmost.contains('_') { - return None; - } - // we have a subdomain, replace underscores - let leftmost = leftmost.replace('_', "-"); - return Some(format!("{leftmost}.{rest}")); - } - None -} - -pub(crate) async fn connect( +pub(super) async fn connect( stream: T, peer: &P, alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result> + tls_ctx: &Arc, +) -> Result> where T: IO, P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, { - let mut ssl_conf = tls_ctx.configure().unwrap(); + let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); + let mut ssl_conf = ctx.0.configure().unwrap(); ssl_set_renegotiate_mode_freely(&mut ssl_conf); @@ -192,8 +182,9 @@ where // TODO: store X509Store in the peer directly if let Some(ca_list) = peer.get_ca() { let mut store_builder = X509StoreBuilder::new().unwrap(); - for ca in &***ca_list { - store_builder.add_cert(ca.clone()).unwrap(); + for ca in &**ca_list { + let cert = der_to_x509(ca)?; + store_builder.add_cert(cert).unwrap(); } ssl_set_verify_cert_store(&mut ssl_conf, &store_builder.build()) .or_err(InternalError, "failed to load cert store")?; @@ -202,16 +193,17 @@ where // Set up client cert/key if let Some(key_pair) = peer.get_client_cert_key() { debug!("setting client cert and key"); - ssl_use_certificate(&mut ssl_conf, key_pair.leaf()) - .or_err(InternalError, "invalid client cert")?; - ssl_use_private_key(&mut ssl_conf, key_pair.key()) - .or_err(InternalError, "invalid client key")?; + let leaf = der_to_x509(key_pair.leaf())?; + ssl_use_certificate(&mut ssl_conf, &leaf).or_err(InternalError, "invalid client cert")?; + let key = der_to_private_key(key_pair.key())?; + ssl_use_private_key(&mut ssl_conf, &key).or_err(InternalError, "invalid client key")?; let intermediates = key_pair.intermediates(); if !intermediates.is_empty() { debug!("adding intermediate certificates for mTLS chain"); for int in intermediates { - ssl_add_chain_cert(&mut ssl_conf, int) + let cert = der_to_x509(int)?; + ssl_add_chain_cert(&mut ssl_conf, &cert) .or_err(InternalError, "invalid intermediate client cert")?; } } @@ -283,41 +275,3 @@ where None => connect_future.await, } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_replace_leftmost_underscore() { - let none_cases = [ - "", - "some", - "some.com", - "1.1.1.1:5050", - "dog.dot.com", - "dog.d_t.com", - "dog.dot.c_m", - "d_g.com", - "_", - "dog.c_m", - ]; - - for case in none_cases { - assert!(replace_leftmost_underscore(case).is_none(), "{}", case); - } - - assert_eq!( - Some("bb-b.some.com".to_string()), - replace_leftmost_underscore("bb_b.some.com") - ); - assert_eq!( - Some("a-a-a.some.com".to_string()), - replace_leftmost_underscore("a_a_a.some.com") - ); - assert_eq!( - Some("-.some.com".to_string()), - replace_leftmost_underscore("_.some.com") - ); - } -} diff --git a/pingora-core/src/connectors/tls/mod.rs b/pingora-core/src/connectors/tls/mod.rs new file mode 100644 index 00000000..9ede8049 --- /dev/null +++ b/pingora-core/src/connectors/tls/mod.rs @@ -0,0 +1,174 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; +use std::net::SocketAddr; +use std::sync::Arc; + +use pingora_error::ErrorType::ConnectTimedout; +use pingora_error::{Error, Result}; + +use crate::connectors::l4::connect as l4_connect; +#[cfg(not(feature = "rustls"))] +use crate::connectors::tls::boringssl_openssl::connect as tls_connect; +#[cfg(not(feature = "rustls"))] +use crate::connectors::tls::boringssl_openssl::TlsConnectorCtx; +#[cfg(feature = "rustls")] +use crate::connectors::tls::rustls::connect as tls_connect; +#[cfg(feature = "rustls")] +use crate::connectors::tls::rustls::TlsConnectorCtx; +use crate::protocols::Stream; +use crate::upstreams::peer::{Peer, ALPN}; + +use super::ConnectorOptions; + +#[cfg(not(feature = "rustls"))] +pub(crate) mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; + +#[derive(Clone)] +pub struct Connector { + ctx: Arc, // Arc to support clone +} + +impl Connector { + pub fn new(options: Option) -> Self { + TlsConnectorCtx::build_connector(options) + } + + pub fn context(&self) -> &Arc { + &self.ctx + } +} + +pub trait TlsConnectorContext { + fn as_any(&self) -> &dyn Any; + + fn build_connector(options: Option) -> Connector + where + Self: Sized; +} + +pub(super) async fn do_connect( + peer: &P, + bind_to: Option, + alpn_override: Option, + tls_ctx: &Arc, +) -> Result +where + P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, +{ + // Create the future that does the connections, but don't evaluate it until + // we decide if we need a timeout or not + let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx); + + match peer.total_connection_timeout() { + Some(t) => match pingora_timeout::timeout(t, connect_future).await { + Ok(res) => res, + Err(_) => Error::e_explain( + ConnectTimedout, + format!("connecting to server {peer}, total-connection timeout {t:?}"), + ), + }, + None => connect_future.await, + } +} + +async fn do_connect_inner( + peer: &P, + bind_to: Option, + alpn_override: Option, + tls_ctx: &Arc, +) -> Result +where + P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, +{ + let stream = l4_connect(peer, bind_to).await?; + if peer.tls() { + let tls_stream = tls_connect(stream, peer, alpn_override, tls_ctx).await?; + Ok(Box::new(tls_stream)) + } else { + Ok(Box::new(stream)) + } +} + +/* + OpenSSL considers underscores in hostnames non-compliant. + We replace the underscore in the leftmost label as we must support these + hostnames for wildcard matches and we have not patched OpenSSL. + + https://github.com/openssl/openssl/issues/12566 + + > The labels must follow the rules for ARPANET host names. They must + > start with a letter, end with a letter or digit, and have as interior + > characters only letters, digits, and hyphen. There are also some + > restrictions on the length. Labels must be 63 characters or less. + - https://datatracker.ietf.org/doc/html/rfc1034#section-3.5 +*/ +pub(crate) fn replace_leftmost_underscore(sni: &str) -> Option { + // wildcard is only leftmost label + if let Some((leftmost, rest)) = sni.split_once('.') { + // if not a subdomain or leftmost does not contain underscore return + if !rest.contains('.') || !leftmost.contains('_') { + return None; + } + // we have a subdomain, replace underscores + let leftmost = leftmost.replace('_', "-"); + return Some(format!("{leftmost}.{rest}")); + } + None +} + +#[cfg(test)] +mod tests { + #[test] + fn test_replace_leftmost_underscore() { + let none_cases = [ + "", + "some", + "some.com", + "1.1.1.1:5050", + "dog.dot.com", + "dog.d_t.com", + "dog.dot.c_m", + "d_g.com", + "_", + "dog.c_m", + ]; + + for case in none_cases { + assert!( + super::replace_leftmost_underscore(case).is_none(), + "{}", + case + ); + } + + assert_eq!( + Some("bb-b.some.com".to_string()), + super::replace_leftmost_underscore("bb_b.some.com") + ); + assert_eq!( + Some("a-a-a.some.com".to_string()), + super::replace_leftmost_underscore("a_a_a.some.com") + ); + assert_eq!( + Some("-.some.com".to_string()), + super::replace_leftmost_underscore("_.some.com") + ); + } +} diff --git a/pingora-core/src/connectors/tls/rustls/mod.rs b/pingora-core/src/connectors/tls/rustls/mod.rs new file mode 100644 index 00000000..9953e7a3 --- /dev/null +++ b/pingora-core/src/connectors/tls/rustls/mod.rs @@ -0,0 +1,230 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS connector specific implementation + +use std::any::Any; +use std::sync::Arc; + +use log::debug; + +use pingora_error::ErrorType::{ConnectTimedout, InvalidCert}; +use pingora_error::{Error, OrErr, Result}; +use pingora_rustls::version; +use pingora_rustls::CertificateDer; +use pingora_rustls::ClientConfig; +use pingora_rustls::ClientConfig as RusTlsClientConfig; +use pingora_rustls::PrivateKeyDer; +use pingora_rustls::RootCertStore; +use pingora_rustls::TlsConnector as RusTlsConnector; +use pingora_rustls::{ + load_ca_file_into_store, load_certs_key_file, load_platform_certs_incl_env_into_store, +}; + +use crate::connectors::ConnectorOptions; +use crate::listeners::ALPN; +use crate::protocols::tls::rustls::client::handshake; +use crate::protocols::tls::TlsStream; +use crate::protocols::IO; +use crate::upstreams::peer::Peer; + +use super::{replace_leftmost_underscore, Connector, TlsConnectorContext}; + +pub(super) struct TlsConnectorCtx { + config: RusTlsClientConfig, + ca_certs: RootCertStore, +} +impl TlsConnectorContext for TlsConnectorCtx { + fn as_any(&self) -> &dyn Any { + self + } + + fn build_connector(options: Option) -> Connector + where + Self: Sized, + { + // NOTE: Rustls only supports TLS 1.2 & 1.3 + + // TODO: currently using Rustls defaults + // - support SSLKEYLOGFILE + // - set supported ciphers/algorithms/curves + // - add options for CRL/OCSP validation + + let (ca_certs, certs_key) = { + let mut ca_certs = RootCertStore::empty(); + let mut certs_key = None; + + if let Some(conf) = options.as_ref() { + if let Some(ca_file_path) = conf.ca_file.as_ref() { + load_ca_file_into_store(ca_file_path, &mut ca_certs); + } else { + load_platform_certs_incl_env_into_store(&mut ca_certs); + } + if let Some((cert, key)) = conf.cert_key_file.as_ref() { + certs_key = load_certs_key_file(&cert, &key); + } + // TODO: support SSLKEYLOGFILE + } else { + load_platform_certs_incl_env_into_store(&mut ca_certs); + } + + (ca_certs, certs_key) + }; + + // TODO: WebPkiServerVerifier for CRL/OCSP validation + let builder = + ClientConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_root_certificates(ca_certs.clone()); + + let config = match certs_key { + Some((certs, key)) => { + match builder.with_client_auth_cert(certs.clone(), key.clone_key()) { + Ok(config) => config, + Err(err) => { + // TODO: is there a viable alternative to the panic? + // falling back to no client auth... does not seem to be reasonable. + panic!( + "{}", + format!("Failed to configure client auth cert/key. Error: {}", err) + ); + } + } + } + None => builder.with_no_client_auth(), + }; + + Connector { + ctx: Arc::new(TlsConnectorCtx { config, ca_certs }), + } + } +} + +pub(super) async fn connect( + stream: T, + peer: &P, + alpn_override: Option, + tls_ctx: &Arc, +) -> Result> +where + T: IO, + P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, +{ + let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); + let mut config = ctx.config.clone(); + + // TODO: setup CA/verify cert store from peer + // looks like the fields are always None + // peer.get_ca() + + let key_pair = peer.get_client_cert_key(); + let updated_config: Option = match key_pair { + None => None, + Some(key_arc) => { + debug!("setting client cert and key"); + + let mut cert_chain = vec![]; + debug!("adding leaf certificate to mTLS cert chain"); + cert_chain.push(key_arc.leaf().to_owned()); + + debug!("adding intermediate certificates to mTLS cert chain"); + key_arc + .intermediates() + .to_owned() + .iter() + .map(|i| i.to_vec()) + .for_each(|i| cert_chain.push(i)); + + let certs: Vec = cert_chain + .into_iter() + .map(|c| c.as_slice().to_owned().into()) + .collect(); + let private_key: PrivateKeyDer = + key_arc.key().as_slice().to_owned().try_into().unwrap(); + + let builder = ClientConfig::builder_with_protocol_versions(&vec![ + &version::TLS12, + &version::TLS13, + ]) + .with_root_certificates(ctx.ca_certs.clone()); + + let updated_config = builder + .with_client_auth_cert(certs, private_key) + .explain_err(InvalidCert, |e| { + format!( + "Failed to use peer cert/key to update Rustls config: {:?}", + e + ) + })?; + Some(updated_config) + } + }; + + if let Some(alpn) = alpn_override.as_ref().or(peer.get_alpn()) { + config.alpn_protocols = alpn.to_wire_protocols(); + } + + // TODO: curve setup from peer + // - second key share from peer, currently only used in boringssl with PQ features + + let tls_conn = if let Some(cfg) = updated_config { + RusTlsConnector::from(Arc::new(cfg)) + } else { + RusTlsConnector::from(Arc::new(config)) + }; + + // TODO: for consistent behaviour between TLS providers some additions are required + // - allowing to disable verification + // - the validation/replace logic would need adjustments to match the boringssl/openssl behaviour + // implementing a custom certificate_verifier could be used to achieve matching behaviour + //let d_conf = config.dangerous(); + //d_conf.set_certificate_verifier(...); + + let mut domain = peer.sni().to_string(); + if peer.sni().is_empty() { + // use ip in case SNI is not present + // TODO: disable validation + domain = peer.address().as_inet().unwrap().ip().to_string() + } + if peer.verify_cert() { + if peer.verify_hostname() { + // TODO: streamline logic with replacing first underscore within TLS implementations + if let Some(sni_s) = replace_leftmost_underscore(peer.sni()) { + domain = sni_s; + } + if let Some(alt_cn) = peer.alternative_cn() { + if !alt_cn.is_empty() { + domain = alt_cn.to_string(); + // TODO: streamline logic with replacing first underscore within TLS implementations + if let Some(alt_cn_s) = replace_leftmost_underscore(alt_cn) { + domain = alt_cn_s; + } + } + } + } + } + + let connect_future = handshake(&tls_conn, &domain, stream); + + match peer.connection_timeout() { + Some(t) => match pingora_timeout::timeout(t, connect_future).await { + Ok(res) => res, + Err(_) => Error::e_explain( + ConnectTimedout, + format!("connecting to server {}, timeout {:?}", peer, t), + ), + }, + None => connect_future.await, + } +} diff --git a/pingora-core/src/lib.rs b/pingora-core/src/lib.rs index 3435fe58..a4d131f7 100644 --- a/pingora-core/src/lib.rs +++ b/pingora-core/src/lib.rs @@ -54,12 +54,23 @@ pub use pingora_error::{ErrorType::*, *}; // If both openssl and boringssl are enabled, prefer boringssl. // This is to make sure that boringssl can override the default openssl feature // when this crate is used indirectly by other crates. -#[cfg(feature = "boringssl")] +#[cfg(all(not(feature = "rustls"), feature = "boringssl"))] pub use pingora_boringssl as tls; -#[cfg(all(not(feature = "boringssl"), feature = "openssl"))] +#[cfg(all( + not(feature = "rustls"), + not(feature = "boringssl"), + feature = "openssl" +))] pub use pingora_openssl as tls; +#[cfg(all( + not(feature = "boringssl"), + not(feature = "openssl"), + feature = "rustls" +))] +pub use pingora_rustls as tls; + pub mod prelude { pub use crate::server::configuration::Opt; pub use crate::server::Server; diff --git a/pingora-core/src/listeners/mod.rs b/pingora-core/src/listeners/mod.rs index cc8edb87..025ae932 100644 --- a/pingora-core/src/listeners/mod.rs +++ b/pingora-core/src/listeners/mod.rs @@ -14,22 +14,21 @@ //! The listening endpoints (TCP and TLS) and their configurations. -mod l4; -mod tls; - -use crate::protocols::Stream; -use crate::server::ListenFds; - -use pingora_error::Result; use std::{fs::Permissions, sync::Arc}; use l4::{ListenerEndpoint, Stream as L4Stream}; -use tls::Acceptor; - -pub use crate::protocols::ssl::server::TlsAccept; pub use l4::{ServerAddress, TcpSocketOptions}; +use pingora_error::Result; +use tls::Acceptor; pub use tls::{TlsSettings, ALPN}; +pub use crate::protocols::tls::server::TlsAccept; +use crate::protocols::Stream; +use crate::server::ListenFds; + +mod l4; +pub mod tls; + struct TransportStackBuilder { l4: ServerAddress, tls: Option, @@ -82,7 +81,7 @@ pub(crate) struct UninitializedStream { impl UninitializedStream { pub async fn handshake(self) -> Result { if let Some(tls) = self.tls { - let tls_stream = tls.tls_handshake(self.l4).await?; + let tls_stream = tls.handshake(self.l4).await?; Ok(Box::new(tls_stream)) } else { Ok(Box::new(self.l4)) @@ -182,11 +181,12 @@ impl Listeners { #[cfg(test)] mod test { - use super::*; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; use tokio::time::{sleep, Duration}; + use super::*; + #[tokio::test] async fn test_listen_tcp() { let addr1 = "127.0.0.1:7101"; diff --git a/pingora-core/src/listeners/tls.rs b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs similarity index 50% rename from pingora-core/src/listeners/tls.rs rename to pingora-core/src/listeners/tls/boringssl_openssl/mod.rs index 1dd63d46..b5d0556c 100644 --- a/pingora-core/src/listeners/tls.rs +++ b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs @@ -12,60 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -use log::debug; -use pingora_error::{ErrorType, OrErr, Result}; -use std::ops::{Deref, DerefMut}; +//! BoringSSL & OpenSSL listener specific implementation -use crate::protocols::ssl::{ - server::{handshake, handshake_with_callback, TlsAcceptCallbacks}, - SslStream, -}; -use crate::protocols::IO; +use crate::listeners::{TlsSettings, ALPN}; use crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; +use pingora_error::{ErrorType, OrErr, Result}; +use std::ops::{Deref, DerefMut}; -pub use crate::protocols::ssl::ALPN; - -pub const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); - -pub(crate) struct Acceptor { - ssl_acceptor: SslAcceptor, - callbacks: Option, -} - -/// The TLS settings of a listening endpoint -pub struct TlsSettings { - accept_builder: SslAcceptorBuilder, - callbacks: Option, -} - -impl From for TlsSettings { - fn from(settings: SslAcceptorBuilder) -> Self { - TlsSettings { - accept_builder: settings, - callbacks: None, - } - } -} +const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); -impl Deref for TlsSettings { - type Target = SslAcceptorBuilder; +pub struct TlsAcceptorBuil(SslAcceptorBuilder); +pub(super) struct TlsAcc(pub(super) SslAcceptor); - fn deref(&self) -> &Self::Target { - &self.accept_builder +impl TlsAcceptorBuil { + pub(super) fn build(self) -> TlsAcc { + TlsAcc(SslAcceptorBuilder::build(self.0)) } -} -impl DerefMut for TlsSettings { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.accept_builder + pub(super) fn set_alpn(&mut self, alpn: ALPN) { + match alpn { + ALPN::H2H1 => self.0.set_alpn_select_callback(alpn::prefer_h2), + ALPN::H1 => self.0.set_alpn_select_callback(alpn::h1_only), + ALPN::H2 => self.0.set_alpn_select_callback(alpn::h2_only), + } } -} -impl TlsSettings { - /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29) - /// server side TLS settings. Users can adjust the TLS settings after this object is created. - /// Return error if the provided certificate and private key are invalid or not found. - pub fn intermediate(cert_path: &str, key_path: &str) -> Result { + pub(super) fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where + Self: Sized, + { let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( TLS_CONF_ERR, "fail to create mozilla_intermediate_v5 Acceptor", @@ -78,64 +53,32 @@ impl TlsSettings { .or_err_with(TLS_CONF_ERR, || { format!("fail to read cert file {cert_path}") })?; - Ok(TlsSettings { - accept_builder, - callbacks: None, - }) + Ok(TlsAcceptorBuil(accept_builder)) } - /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks] - /// is needed to provide the certificate during the TLS handshake. - pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result { + pub(super) fn acceptor_with_callbacks() -> Result + where + Self: Sized, + { let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( TLS_CONF_ERR, "fail to create mozilla_intermediate_v5 Acceptor", )?; - Ok(TlsSettings { - accept_builder, - callbacks: Some(callbacks), - }) - } - - /// Enable HTTP/2 support for this endpoint, which is default off. - /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed - pub fn enable_h2(&mut self) { - self.set_alpn(ALPN::H2H1); - } - - /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details - pub fn set_alpn(&mut self, alpn: ALPN) { - match alpn { - ALPN::H2H1 => self - .accept_builder - .set_alpn_select_callback(alpn::prefer_h2), - ALPN::H1 => self.accept_builder.set_alpn_select_callback(alpn::h1_only), - ALPN::H2 => self.accept_builder.set_alpn_select_callback(alpn::h2_only), - } - } - - pub(crate) fn build(self) -> Acceptor { - Acceptor { - ssl_acceptor: self.accept_builder.build(), - callbacks: self.callbacks, - } + Ok(TlsAcceptorBuil(accept_builder)) } } -impl Acceptor { - pub async fn tls_handshake(&self, stream: S) -> Result> { - debug!("new ssl session"); - // TODO: be able to offload this handshake in a thread pool - if let Some(cb) = self.callbacks.as_ref() { - handshake_with_callback(&self.ssl_acceptor, stream, cb).await - } else { - handshake(&self.ssl_acceptor, stream).await +impl From for TlsSettings { + fn from(settings: SslAcceptorBuilder) -> Self { + TlsSettings { + accept_builder: TlsAcceptorBuil(settings), + callbacks: None, } } } mod alpn { - use super::*; + use crate::protocols::ALPN; use crate::tls::ssl::{select_next_proto, AlpnError, SslRef}; // A standard implementation provided by the SSL lib is used below @@ -161,3 +104,17 @@ mod alpn { } } } + +impl Deref for TlsAcceptorBuil { + type Target = SslAcceptorBuilder; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TlsAcceptorBuil { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs new file mode 100644 index 00000000..9340a151 --- /dev/null +++ b/pingora-core/src/listeners/tls/mod.rs @@ -0,0 +1,127 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::protocols::tls::server::TlsAcceptCallbacks; +use crate::protocols::tls::TlsStream; +pub use crate::protocols::tls::ALPN; +use crate::protocols::IO; +#[cfg(not(feature = "rustls"))] +use crate::{ + listeners::tls::boringssl_openssl::{TlsAcc, TlsAcceptorBuil}, + protocols::tls::boringssl_openssl::server::{handshake, handshake_with_callback}, +}; +#[cfg(feature = "rustls")] +use crate::{ + listeners::tls::rustls::{TlsAcc, TlsAcceptorBuil}, + protocols::tls::rustls::server::{handshake, handshake_with_callback}, +}; +use log::debug; +use pingora_error::Result; +use std::ops::{Deref, DerefMut}; + +#[cfg(not(feature = "rustls"))] +pub(crate) mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; + +pub struct Acceptor { + tls_acceptor: TlsAcc, + callbacks: Option, +} + +/// The TLS settings of a listening endpoint +pub struct TlsSettings { + accept_builder: TlsAcceptorBuil, + callbacks: Option, +} + +// NOTE: keeping trait for documentation purpose +// switched to direct implementations to eliminate redirections in within the call-graph +// the below trait needs to be implemented for TlsAcceptorBuil +pub trait TlsAcceptorBuilder { + type TlsAcceptor; // TlsAcc + fn build(self) -> Self::TlsAcceptor; + fn set_alpn(&mut self, alpn: ALPN); + fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where + Self: Sized; + fn acceptor_with_callbacks() -> Result + where + Self: Sized; +} + +impl TlsSettings { + /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29) + /// server side TLS settings. Users can adjust the TLS settings after this object is created. + /// Return error if the provided certificate and private key are invalid or not found. + pub fn intermediate(cert_path: &str, key_path: &str) -> Result { + Ok(TlsSettings { + accept_builder: TlsAcceptorBuil::acceptor_intermediate(cert_path, key_path)?, + callbacks: None, + }) + } + + /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks] + /// is needed to provide the certificate during the TLS handshake. + pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result { + Ok(TlsSettings { + accept_builder: TlsAcceptorBuil::acceptor_with_callbacks()?, + callbacks: Some(callbacks), + }) + } + + /// Enable HTTP/2 support for this endpoint, which is default off. + /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed + pub fn enable_h2(&mut self) { + self.set_alpn(ALPN::H2H1); + } + + /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details + pub fn set_alpn(&mut self, alpn: ALPN) { + self.accept_builder.set_alpn(alpn); + } + + pub(crate) fn build(self) -> Acceptor { + Acceptor { + tls_acceptor: self.accept_builder.build(), + callbacks: self.callbacks, + } + } +} + +impl Acceptor { + pub async fn handshake(&self, stream: S) -> Result> { + debug!("new tls session"); + // TODO: be able to offload this handshake in a thread pool + if let Some(cb) = self.callbacks.as_ref() { + handshake_with_callback(&self.tls_acceptor.0, stream, cb).await + } else { + handshake(&self.tls_acceptor.0, stream).await + } + } +} + +impl Deref for TlsSettings { + type Target = TlsAcceptorBuil; + + fn deref(&self) -> &Self::Target { + &self.accept_builder + } +} + +impl DerefMut for TlsSettings { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.accept_builder + } +} diff --git a/pingora-core/src/listeners/tls/rustls/mod.rs b/pingora-core/src/listeners/tls/rustls/mod.rs new file mode 100644 index 00000000..2e36913c --- /dev/null +++ b/pingora-core/src/listeners/tls/rustls/mod.rs @@ -0,0 +1,89 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS listener specific implementation + +use std::sync::Arc; + +use pingora_error::ErrorType::InternalError; +use pingora_error::{Error, ErrorSource, ImmutStr, OrErr, Result}; +use pingora_rustls::load_certs_key_file; +use pingora_rustls::ServerConfig; +use pingora_rustls::{version, TlsAcceptor as RusTlsAcceptor}; + +use crate::listeners::ALPN; + +pub struct TlsAcceptorBuil { + alpn_protocols: Option>>, + cert_path: String, + key_path: String, +} + +pub(super) struct TlsAcc(pub(super) RusTlsAcceptor); + +impl TlsAcceptorBuil { + pub(super) fn build(self) -> TlsAcc { + let (certs, key) = load_certs_key_file(&self.cert_path, &self.key_path).expect( + format!( + "Failed to load provided certificates \"{}\" or key \"{}\".", + self.cert_path, self.key_path + ) + .as_str(), + ); + + let mut config = + ServerConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_no_client_auth() + .with_single_cert(certs, key) + .explain_err(InternalError, |e| { + format!("Failed to create server listener config: {}", e) + }) + .unwrap(); + + if let Some(alpn_protocols) = self.alpn_protocols { + config.alpn_protocols = alpn_protocols; + } + + TlsAcc(RusTlsAcceptor::from(Arc::new(config))) + } + pub(super) fn set_alpn(&mut self, alpn: ALPN) { + self.alpn_protocols = Some(alpn.to_wire_protocols()); + } + + pub(super) fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where + Self: Sized, + { + Ok(TlsAcceptorBuil { + alpn_protocols: None, + cert_path: cert_path.to_string(), + key_path: key_path.to_string(), + }) + } + + pub(super) fn acceptor_with_callbacks() -> Result + where + Self: Sized, + { + // TODO: verify if/how callback in handshake can be done using Rustls + Err(Error::create( + InternalError, + ErrorSource::Internal, + Some(ImmutStr::from( + "Certificate callbacks are not supported with feature \"rustls\".", + )), + None, + )) + } +} diff --git a/pingora-core/src/protocols/digest.rs b/pingora-core/src/protocols/digest.rs index e080a89a..56ec3725 100644 --- a/pingora-core/src/protocols/digest.rs +++ b/pingora-core/src/protocols/digest.rs @@ -22,7 +22,7 @@ use once_cell::sync::OnceCell; use super::l4::ext::{get_recv_buf, get_tcp_info, TCP_INFO}; use super::l4::socket::SocketAddr; use super::raw_connect::ProxyDigest; -use super::ssl::digest::SslDigest; +use super::tls::SslDigest; /// The information can be extracted from a connection #[derive(Clone, Debug, Default)] diff --git a/pingora-core/src/protocols/http/server.rs b/pingora-core/src/protocols/http/server.rs index c6479e7a..56ae2c94 100644 --- a/pingora-core/src/protocols/http/server.rs +++ b/pingora-core/src/protocols/http/server.rs @@ -53,7 +53,7 @@ impl Session { /// else with the session. /// - `Ok(true)`: successful /// - `Ok(false)`: client exit without sending any bytes. This is normal on reused connection. - /// In this case the user should give up this session. + /// In this case the user should give up this session. pub async fn read_request(&mut self) -> Result { match self { Self::H1(s) => { diff --git a/pingora-core/src/protocols/mod.rs b/pingora-core/src/protocols/mod.rs index 32695e1c..d212bd0f 100644 --- a/pingora-core/src/protocols/mod.rs +++ b/pingora-core/src/protocols/mod.rs @@ -18,14 +18,14 @@ mod digest; pub mod http; pub mod l4; pub mod raw_connect; -pub mod ssl; +pub mod tls; pub use digest::{ Digest, GetProxyDigest, GetSocketDigest, GetTimingDigest, ProtoDigest, SocketDigest, TimingDigest, }; pub use l4::ext::TcpKeepalive; -pub use ssl::ALPN; +pub use tls::ALPN; use async_trait::async_trait; use std::fmt::Debug; @@ -48,19 +48,19 @@ pub trait UniqueID { /// Interface to get TLS info pub trait Ssl { /// Return the TLS info if the connection is over TLS + #[cfg(not(feature = "rustls"))] fn get_ssl(&self) -> Option<&crate::tls::ssl::SslRef> { None } /// Return the [`ssl::SslDigest`] for logging - fn get_ssl_digest(&self) -> Option> { + fn get_ssl_digest(&self) -> Option> { None } /// Return selected ALPN if any fn selected_alpn_proto(&self) -> Option { - let ssl = self.get_ssl()?; - ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + None } } diff --git a/pingora-core/src/protocols/ssl/client.rs b/pingora-core/src/protocols/ssl/client.rs deleted file mode 100644 index 9edf29b2..00000000 --- a/pingora-core/src/protocols/ssl/client.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! TLS client specific implementation - -use super::SslStream; -use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{ - GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest, TimingDigest, IO, -}; -use crate::tls::{ssl, ssl::ConnectConfiguration, ssl_sys::X509_V_ERR_INVALID_CALL}; - -use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use std::sync::Arc; -use std::time::Duration; - -/// Perform the TLS handshake for the given connection with the given configuration -pub async fn handshake( - conn_config: ConnectConfiguration, - domain: &str, - io: S, -) -> Result> { - let ssl = conn_config - .into_ssl(domain) - .explain_err(TLSHandshakeFailure, |e| format!("ssl config error: {e}"))?; - let mut stream = SslStream::new(ssl, io) - .explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}"))?; - let handshake_result = stream.connect().await; - match handshake_result { - Ok(()) => Ok(stream), - Err(e) => { - let context = format!("TLS connect() failed: {e}, SNI: {domain}"); - match e.code() { - ssl::ErrorCode::SSL => { - // Unify the return type of `verify_result` for openssl - #[cfg(not(feature = "boringssl"))] - fn verify_result(stream: SslStream) -> Result<(), i32> { - match stream.ssl().verify_result().as_raw() { - crate::tls::ssl_sys::X509_V_OK => Ok(()), - e => Err(e), - } - } - - // Unify the return type of `verify_result` for boringssl - #[cfg(feature = "boringssl")] - fn verify_result(stream: SslStream) -> Result<(), i32> { - stream.ssl().verify_result().map_err(|e| e.as_raw()) - } - - match verify_result(stream) { - Ok(()) => Error::e_explain(TLSHandshakeFailure, context), - // X509_V_ERR_INVALID_CALL in case verify result was never set - Err(X509_V_ERR_INVALID_CALL) => { - Error::e_explain(TLSHandshakeFailure, context) - } - _ => Error::e_explain(InvalidCert, context), - } - } - /* likely network error, but still mark as TLS error */ - _ => Error::e_explain(TLSHandshakeFailure, context), - } - } - } -} - -impl GetTimingDigest for SslStream -where - S: GetTimingDigest, -{ - fn get_timing_digest(&self) -> Vec> { - let mut ts_vec = self.get_ref().get_timing_digest(); - ts_vec.push(Some(self.timing.clone())); - ts_vec - } - fn get_read_pending_time(&self) -> Duration { - self.get_ref().get_read_pending_time() - } - - fn get_write_pending_time(&self) -> Duration { - self.get_ref().get_write_pending_time() - } -} - -impl GetProxyDigest for SslStream -where - S: GetProxyDigest, -{ - fn get_proxy_digest(&self) -> Option> { - self.get_ref().get_proxy_digest() - } -} - -impl GetSocketDigest for SslStream -where - S: GetSocketDigest, -{ - fn get_socket_digest(&self) -> Option> { - self.get_ref().get_socket_digest() - } - fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - self.get_mut().set_socket_digest(socket_digest) - } -} diff --git a/pingora-core/src/protocols/ssl/digest.rs b/pingora-core/src/protocols/ssl/digest.rs deleted file mode 100644 index 3cdb7aa0..00000000 --- a/pingora-core/src/protocols/ssl/digest.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! TLS information from the TLS connection - -use crate::tls::{hash::MessageDigest, ssl::SslRef}; -use crate::utils; - -/// The TLS connection information -#[derive(Clone, Debug)] -pub struct SslDigest { - /// The cipher used - pub cipher: &'static str, - /// The TLS version of this connection - pub version: &'static str, - /// The organization of the peer's certificate - pub organization: Option, - /// The serial number of the peer's certificate - pub serial_number: Option, - /// The digest of the peer's certificate - pub cert_digest: Vec, -} - -impl SslDigest { - pub fn from_ssl(ssl: &SslRef) -> Self { - let cipher = match ssl.current_cipher() { - Some(c) => c.name(), - None => "", - }; - - let (cert_digest, org, sn) = match ssl.peer_certificate() { - Some(cert) => { - let cert_digest = match cert.digest(MessageDigest::sha256()) { - Ok(c) => c.as_ref().to_vec(), - Err(_) => Vec::new(), - }; - ( - cert_digest, - utils::get_organization(&cert), - utils::get_serial(&cert).ok(), - ) - } - None => (Vec::new(), None, None), - }; - - SslDigest { - cipher, - version: ssl.version_str(), - organization: org, - serial_number: sn, - cert_digest, - } - } -} diff --git a/pingora-core/src/protocols/ssl/mod.rs b/pingora-core/src/protocols/ssl/mod.rs deleted file mode 100644 index f1ce8b95..00000000 --- a/pingora-core/src/protocols/ssl/mod.rs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The TLS layer implementations - -pub mod client; -pub mod digest; -pub mod server; - -use crate::protocols::digest::TimingDigest; -use crate::protocols::{Ssl, UniqueID}; -use crate::tls::{self, ssl, tokio_ssl::SslStream as InnerSsl}; -use log::warn; -use pingora_error::{ErrorType::*, OrErr, Result}; -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::SystemTime; -use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; - -pub use digest::SslDigest; - -/// The TLS connection -#[derive(Debug)] -pub struct SslStream { - ssl: InnerSsl, - digest: Option>, - timing: TimingDigest, -} - -impl SslStream -where - T: AsyncRead + AsyncWrite + std::marker::Unpin, -{ - /// Create a new TLS connection from the given `stream` - /// - /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS - /// handshake after. - pub fn new(ssl: ssl::Ssl, stream: T) -> Result { - let ssl = InnerSsl::new(ssl, stream) - .explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}"))?; - - Ok(SslStream { - ssl, - digest: None, - timing: Default::default(), - }) - } - - /// Connect to the remote TLS server as a client - pub async fn connect(&mut self) -> Result<(), ssl::Error> { - Self::clear_error(); - Pin::new(&mut self.ssl).connect().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = Some(Arc::new(SslDigest::from_ssl(self.ssl()))); - Ok(()) - } - - /// Finish the TLS handshake from client as a server - pub async fn accept(&mut self) -> Result<(), ssl::Error> { - Self::clear_error(); - Pin::new(&mut self.ssl).accept().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = Some(Arc::new(SslDigest::from_ssl(self.ssl()))); - Ok(()) - } - - #[inline] - fn clear_error() { - let errs = tls::error::ErrorStack::get(); - if !errs.errors().is_empty() { - warn!("Clearing dirty TLS error stack: {}", errs); - } - } -} - -impl SslStream { - pub fn ssl_digest(&self) -> Option> { - self.digest.clone() - } -} - -use std::ops::{Deref, DerefMut}; - -impl Deref for SslStream { - type Target = InnerSsl; - - fn deref(&self) -> &Self::Target { - &self.ssl - } -} - -impl DerefMut for SslStream { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.ssl - } -} - -impl AsyncRead for SslStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_read(cx, buf) - } -} - -impl AsyncWrite for SslStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context, - buf: &[u8], - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_write(cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_shutdown(cx) - } - - fn poll_write_vectored( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_write_vectored(cx, bufs) - } - - fn is_write_vectored(&self) -> bool { - true - } -} - -impl UniqueID for SslStream -where - T: UniqueID, -{ - fn id(&self) -> i32 { - self.ssl.get_ref().id() - } -} - -impl Ssl for SslStream { - fn get_ssl(&self) -> Option<&ssl::SslRef> { - Some(self.ssl()) - } - - fn get_ssl_digest(&self) -> Option> { - self.ssl_digest() - } -} - -/// The protocol for Application-Layer Protocol Negotiation -#[derive(Hash, Clone, Debug)] -pub enum ALPN { - /// Prefer HTTP/1.1 only - H1, - /// Prefer HTTP/2 only - H2, - /// Prefer HTTP/2 over HTTP/1.1 - H2H1, -} - -impl std::fmt::Display for ALPN { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ALPN::H1 => write!(f, "H1"), - ALPN::H2 => write!(f, "H2"), - ALPN::H2H1 => write!(f, "H2H1"), - } - } -} - -impl ALPN { - /// Create a new ALPN according to the `max` and `min` version constraints - pub fn new(max: u8, min: u8) -> Self { - if max == 1 { - ALPN::H1 - } else if min == 2 { - ALPN::H2 - } else { - ALPN::H2H1 - } - } - - /// Return the max http version this [`ALPN`] allows - pub fn get_max_http_version(&self) -> u8 { - match self { - ALPN::H1 => 1, - _ => 2, - } - } - - /// Return the min http version this [`ALPN`] allows - pub fn get_min_http_version(&self) -> u8 { - match self { - ALPN::H2 => 2, - _ => 1, - } - } - - pub(crate) fn to_wire_preference(&self) -> &[u8] { - // https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_alpn_select_cb.html - // "vector of nonempty, 8-bit length-prefixed, byte strings" - match self { - Self::H1 => b"\x08http/1.1", - Self::H2 => b"\x02h2", - Self::H2H1 => b"\x02h2\x08http/1.1", - } - } - - pub(crate) fn from_wire_selected(raw: &[u8]) -> Option { - match raw { - b"http/1.1" => Some(Self::H1), - b"h2" => Some(Self::H2), - _ => None, - } - } -} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs new file mode 100644 index 00000000..37d2160b --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs @@ -0,0 +1,44 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS client specific implementation + +use pingora_error::{Error, ErrorType::*, OrErr, Result}; + +use crate::protocols::tls::TlsStream; +use crate::protocols::IO; +use crate::tls::ssl::ConnectConfiguration; + +/// Perform the TLS handshake for the given connection with the given configuration +pub async fn handshake( + conn_config: ConnectConfiguration, + domain: &str, + io: S, +) -> Result> { + let ssl = conn_config + .into_ssl(domain) + .explain_err(TLSHandshakeFailure, |e| format!("tls config error: {e}"))?; + let mut stream = TlsStream::new(ssl, io) + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + + stream.connect().await.map_err(|e| { + let err_msg = format!("TLS connect() failed: {e}, SNI: {domain}"); + if let Some(context) = e.context { + Error::explain(e.etype, format!("{}, {}", err_msg, context.as_str())) + } else { + Error::explain(e.etype, err_msg) + } + })?; + Ok(stream) +} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs new file mode 100644 index 00000000..a4c52fb5 --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs @@ -0,0 +1,18 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS specific implementation +pub mod client; +pub mod server; +pub(super) mod stream; diff --git a/pingora-core/src/protocols/ssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs similarity index 53% rename from pingora-core/src/protocols/ssl/server.rs rename to pingora-core/src/protocols/tls/boringssl_openssl/server.rs index c85846b1..6270614d 100644 --- a/pingora-core/src/protocols/ssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -12,31 +12,60 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! TLS server specific implementation +//! BoringSSL & OpenSSL TLS server specific implementation -use super::SslStream; -use crate::protocols::{Shutdown, IO}; +use crate::protocols::Ssl; +use std::pin::Pin; + +use async_trait::async_trait; +use pingora_error::ErrorType::{TLSHandshakeFailure, TLSWantX509Lookup}; +use pingora_error::{OrErr, Result}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; +use crate::protocols::tls::TlsStream; +use crate::protocols::IO; use crate::tls::ext; use crate::tls::ext::ssl_from_acceptor; -use crate::tls::ssl; -use crate::tls::ssl::{SslAcceptor, SslRef}; +use crate::tls::ssl::SslAcceptor; -use async_trait::async_trait; -use log::warn; -use pingora_error::{ErrorType::*, OrErr, Result}; -use std::pin::Pin; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +#[async_trait] +impl ResumableAccept for TlsStream { + async fn start_accept(mut self: Pin<&mut Self>) -> Result { + // safety: &mut self + let ssl_mut = unsafe { ext::ssl_mut(self.get_ssl().unwrap()) }; + ext::suspend_when_need_ssl_cert(ssl_mut); + let res = self.accept().await; + + match res { + Ok(()) => Ok(true), + Err(e) => { + if e.etype == TLSWantX509Lookup { + Ok(false) + } else { + Err(e) + } + } + } + } -/// Prepare a TLS stream for handshake -pub fn prepare_tls_stream(ssl_acceptor: &SslAcceptor, io: S) -> Result> { - let ssl = ssl_from_acceptor(ssl_acceptor) + async fn resume_accept(mut self: Pin<&mut Self>) -> Result<()> { + // safety: &mut ssl + let ssl_mut = unsafe { ext::ssl_mut(self.get_ssl().unwrap()) }; + ext::unblock_ssl_cert(ssl_mut); + self.accept().await + } +} + +fn prepare_tls_stream(acceptor: &SslAcceptor, io: S) -> Result> { + let ssl = ssl_from_acceptor(acceptor) .explain_err(TLSHandshakeFailure, |e| format!("ssl_acceptor error: {e}"))?; - SslStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}")) + TlsStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(ssl_acceptor: &SslAcceptor, io: S) -> Result> { - let mut stream = prepare_tls_stream(ssl_acceptor, io)?; +pub(crate) async fn handshake(acceptor: &SslAcceptor, io: S) -> Result> { + let mut stream = prepare_tls_stream(acceptor, io)?; stream .accept() .await @@ -45,19 +74,16 @@ pub async fn handshake(ssl_acceptor: &SslAcceptor, io: S) -> Result( - ssl_acceptor: &SslAcceptor, +pub(crate) async fn handshake_with_callback( + acceptor: &SslAcceptor, io: S, callbacks: &TlsAcceptCallbacks, -) -> Result> { - let mut tls_stream = prepare_tls_stream(ssl_acceptor, io)?; - let done = Pin::new(&mut tls_stream) - .start_accept() - .await - .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; +) -> pingora_error::Result> { + let mut tls_stream = prepare_tls_stream(acceptor, io)?; + let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { // safety: we do hold a mut ref of tls_stream - let ssl_mut = unsafe { ext::ssl_mut(tls_stream.ssl()) }; + let ssl_mut = unsafe { ext::ssl_mut(tls_stream.stream.ssl()) }; callbacks.certificate_callback(ssl_mut).await; Pin::new(&mut tls_stream) .resume_accept() @@ -69,85 +95,14 @@ pub async fn handshake_with_callback( } } -/// The APIs to customize things like certificate during TLS server side handshake -#[async_trait] -pub trait TlsAccept { - // TODO: return error? - /// This function is called in the middle of a TLS handshake. Structs who implement this function - /// should provide tls certificate and key to the [SslRef] via [ext::ssl_use_certificate] and [ext::ssl_use_private_key]. - async fn certificate_callback(&self, _ssl: &mut SslRef) -> () { - // does nothing by default - } -} - -pub type TlsAcceptCallbacks = Box; - -#[async_trait] -impl Shutdown for SslStream -where - S: AsyncRead + AsyncWrite + Sync + Unpin + Send, -{ - async fn shutdown(&mut self) { - match ::shutdown(self).await { - Ok(()) => {} - Err(e) => { - warn!("TLS shutdown failed, {e}"); - } - } - } -} - -/// Resumable TLS server side handshake. -#[async_trait] -pub trait ResumableAccept { - /// Start a resumable TLS accept handshake. - /// - /// * `Ok(true)` when the handshake is finished - /// * `Ok(false)`` when the handshake is paused midway - /// - /// For now, the accept will only pause when a certificate is needed. - async fn start_accept(self: Pin<&mut Self>) -> Result; - - /// Continue the TLS handshake - /// - /// This function should be called after the certificate is provided. - async fn resume_accept(self: Pin<&mut Self>) -> Result<(), ssl::Error>; -} - -#[async_trait] -impl ResumableAccept for SslStream { - async fn start_accept(mut self: Pin<&mut Self>) -> Result { - // safety: &mut self - let ssl_mut = unsafe { ext::ssl_mut(self.ssl()) }; - ext::suspend_when_need_ssl_cert(ssl_mut); - let res = self.accept().await; - - match res { - Ok(()) => Ok(true), - Err(e) => { - if ext::is_suspended_for_cert(&e) { - Ok(false) - } else { - Err(e) - } - } - } - } - - async fn resume_accept(mut self: Pin<&mut Self>) -> Result<(), ssl::Error> { - // safety: &mut ssl - let ssl_mut = unsafe { ext::ssl_mut(self.ssl()) }; - ext::unblock_ssl_cert(ssl_mut); - self.accept().await - } -} - #[tokio::test] async fn test_async_cert() { + use crate::listeners::tls::TlsSettings; + use crate::listeners::TlsAccept; + use crate::protocols::tls::server::TlsAcceptCallbacks; + use crate::tls::ssl; + use crate::tls::ssl::SslRef; use tokio::io::AsyncReadExt; - let acceptor = ssl::SslAcceptor::mozilla_intermediate_v5(ssl::SslMethod::tls()) - .unwrap() - .build(); struct Callback; #[async_trait] @@ -180,14 +135,13 @@ async fn test_async_cert() { .build(); let mut ssl = ssl::Ssl::new(&ssl_context).unwrap(); ssl.set_hostname("pingora.org").unwrap(); - ssl.set_verify(ssl::SslVerifyMode::NONE); // we don have a valid cert - let mut stream = SslStream::new(ssl, client).unwrap(); + ssl.set_verify(ssl::SslVerifyMode::NONE); // we don't have a valid cert + let mut stream = TlsStream::new(ssl, client).unwrap(); Pin::new(&mut stream).connect().await.unwrap(); let mut buf = [0; 1]; let _ = stream.read(&mut buf).await; }); - handshake_with_callback(&acceptor, server, &cb) - .await - .unwrap(); + let acceptor = TlsSettings::with_callbacks(cb).unwrap().build(); + acceptor.handshake(server).await.unwrap(); } diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs new file mode 100644 index 00000000..700a4b22 --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -0,0 +1,273 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS stream specific implementation + +use crate::tls::hash::MessageDigest; +use log::warn; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::SystemTime; +use tokio::io; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use crate::listeners::ALPN; +use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; +use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::SslDigest; +use crate::protocols::{GetProxyDigest, GetTimingDigest, Ssl, UniqueID}; +use crate::tls::error::ErrorStack; +use crate::tls::ext; +use crate::tls::tokio_ssl::SslStream; +use crate::tls::{ssl, ssl::SslRef, ssl_sys::X509_V_ERR_INVALID_CALL}; +use pingora_error::{Error, ErrorType::*, OrErr, Result}; + +#[derive(Debug)] +pub struct TlsStream { + pub(crate) stream: SslStream, + pub(super) digest: Option>, + pub(super) timing: TimingDigest, +} + +impl TlsStream { + /// Create a new TLS connection from the given `stream` + /// + /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS + /// handshake after. + pub(crate) fn new(ssl: ssl::Ssl, stream: T) -> Result { + let stream = SslStream::new(ssl, stream) + .explain_err(TLSHandshakeFailure, |e| format!("tls.rs stream error: {e}"))?; + + Ok(TlsStream { + stream, + timing: Default::default(), + digest: None, + }) + } + + #[inline] + pub(crate) fn clear_error() { + let errs = ErrorStack::get(); + if !errs.errors().is_empty() { + warn!("Clearing dirty TLS error stack: {}", errs); + } + } +} + +impl TlsStream { + /// Connect to the remote TLS server as a client + pub(crate) async fn connect(&mut self) -> Result<()> { + Self::clear_error(); + match Pin::new(&mut self.stream).connect().await { + Ok(_) => { + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } + Err(err) => self.transform_ssl_error(err), + } + } + + /// Finish the TLS handshake from client as a server + pub(crate) async fn accept(&mut self) -> Result<()> { + Self::clear_error(); + match Pin::new(&mut self.stream).accept().await { + Ok(_) => { + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } + Err(err) => self.transform_ssl_error(err), + } + } + + pub(crate) fn digest(&mut self) -> Option> { + Some(Arc::new(SslDigest::from_ssl(self.stream.ssl()))) + } + + fn transform_ssl_error(&self, e: ssl::Error) -> Result<()> { + let context = format!("ssl::ErrorCode: {:?}", e.code()); + if ext::is_suspended_for_cert(&e) { + Error::e_explain(TLSWantX509Lookup, format!("ssl::ErrorCode: {:?}", e.code())) + } else { + match e.code() { + ssl::ErrorCode::SSL => { + // Unify the return type of `verify_result` for openssl + #[cfg(not(feature = "boringssl"))] + fn verify_result(ssl: &SslRef) -> Result<(), i32> { + match ssl.verify_result().as_raw() { + crate::tls::ssl_sys::X509_V_OK => Ok(()), + e => Err(e), + } + } + + // Unify the return type of `verify_result` for boringssl + #[cfg(feature = "boringssl")] + fn verify_result(ssl: &SslRef) -> Result<(), i32> { + ssl.verify_result().map_err(|e| e.as_raw()) + } + + match verify_result(self.stream.ssl()) { + Ok(()) => Error::e_explain(TLSHandshakeFailure, context), + // X509_V_ERR_INVALID_CALL in case verify result was never set + Err(X509_V_ERR_INVALID_CALL) => { + Error::e_explain(TLSHandshakeFailure, context) + } + _ => Error::e_explain(InvalidCert, context), + } + } + /* likely network error, but still mark as TLS error */ + _ => Error::e_explain(TLSHandshakeFailure, context), + } + } + } +} + +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl Ssl for TlsStream { + fn get_ssl(&self) -> Option<&ssl::SslRef> { + Some(self.stream.ssl()) + } + + fn get_ssl_digest(&self) -> Option> { + self.digest.clone() + } + + fn selected_alpn_proto(&self) -> Option { + let ssl = self.stream.ssl(); + ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + } +} + +impl SslDigest { + pub fn from_ssl(ssl: &SslRef) -> Self { + let cipher = match ssl.current_cipher() { + Some(c) => c.name(), + None => "", + }; + + let (cert_digest, org, sn) = match ssl.peer_certificate() { + Some(cert) => { + let cert_digest = match cert.digest(MessageDigest::sha256()) { + Ok(c) => c.as_ref().to_vec(), + Err(_) => Vec::new(), + }; + ( + cert_digest, + crate::utils::tls::boringssl_openssl::get_x509_organization(&cert), + crate::utils::tls::boringssl_openssl::get_x509_serial(&cert).ok(), + ) + } + None => (Vec::new(), None, None), + }; + + SslDigest { + cipher, + version: ssl.version_str(), + organization: org, + serial_number: sn, + cert_digest, + } + } +} + +impl GetSocketDigest for TlsStream +where + S: GetSocketDigest, +{ + fn get_socket_digest(&self) -> Option> { + self.stream.get_ref().get_socket_digest() + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + self.stream.get_mut().set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for TlsStream +where + S: GetTimingDigest, +{ + fn get_timing_digest(&self) -> Vec> { + self.stream.get_ref().get_timing_digest() + } +} + +impl GetProxyDigest for TlsStream +where + S: GetProxyDigest, +{ + fn get_proxy_digest(&self) -> Option> { + self.stream.get_ref().get_proxy_digest() + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.stream.get_ref().id() + } +} diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs new file mode 100644 index 00000000..53c3ac87 --- /dev/null +++ b/pingora-core/src/protocols/tls/mod.rs @@ -0,0 +1,119 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The TLS layer implementations + +#[cfg(not(feature = "rustls"))] +pub(crate) mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; +pub mod server; + +#[cfg(not(feature = "rustls"))] +pub(crate) use boringssl_openssl::stream::TlsStream; +#[cfg(feature = "rustls")] +pub(crate) use rustls::stream::TlsStream; + +/// The protocol for Application-Layer Protocol Negotiation +#[derive(Hash, Clone, Debug)] +pub enum ALPN { + /// Prefer HTTP/1.1 only + H1, + /// Prefer HTTP/2 only + H2, + /// Prefer HTTP/2 over HTTP/1.1 + H2H1, +} + +impl std::fmt::Display for ALPN { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ALPN::H1 => write!(f, "H1"), + ALPN::H2 => write!(f, "H2"), + ALPN::H2H1 => write!(f, "H2H1"), + } + } +} + +impl ALPN { + /// Create a new ALPN according to the `max` and `min` version constraints + pub fn new(max: u8, min: u8) -> Self { + if max == 1 { + ALPN::H1 + } else if min == 2 { + ALPN::H2 + } else { + ALPN::H2H1 + } + } + + /// Return the max http version this [`ALPN`] allows + pub fn get_max_http_version(&self) -> u8 { + match self { + ALPN::H1 => 1, + _ => 2, + } + } + + /// Return the min http version this [`ALPN`] allows + pub fn get_min_http_version(&self) -> u8 { + match self { + ALPN::H2 => 2, + _ => 1, + } + } + + #[cfg(not(feature = "rustls"))] + pub(crate) fn to_wire_preference(&self) -> &[u8] { + // https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_alpn_select_cb.html + // "vector of nonempty, 8-bit length-prefixed, byte strings" + match self { + Self::H1 => b"\x08http/1.1", + Self::H2 => b"\x02h2", + Self::H2H1 => b"\x02h2\x08http/1.1", + } + } + + #[cfg(feature = "rustls")] + pub(crate) fn to_wire_protocols(&self) -> Vec> { + match self { + ALPN::H1 => vec![b"http/1.1".to_vec()], + ALPN::H2 => vec![b"h2".to_vec()], + ALPN::H2H1 => vec![b"h2".to_vec(), b"http/1.1".to_vec()], + } + } + + pub(crate) fn from_wire_selected(raw: &[u8]) -> Option { + match raw { + b"http/1.1" => Some(Self::H1), + b"h2" => Some(Self::H2), + _ => None, + } + } +} + +/// The TLS connection information +#[derive(Clone, Debug)] +pub struct SslDigest { + /// The cipher used + pub cipher: &'static str, + /// The TLS version of this connection + pub version: &'static str, + /// The organization of the peer's certificate + pub organization: Option, + /// The serial number of the peer's certificate + pub serial_number: Option, + /// The digest of the peer's certificate + pub cert_digest: Vec, +} diff --git a/pingora-core/src/protocols/tls/rustls/client.rs b/pingora-core/src/protocols/tls/rustls/client.rs new file mode 100644 index 00000000..3b66154f --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/client.rs @@ -0,0 +1,41 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS client specific implementation + +use crate::protocols::tls::rustls::stream::TlsStream; +use crate::protocols::IO; +use pingora_error::ErrorType::TLSHandshakeFailure; +use pingora_error::{Error, OrErr, Result}; +use pingora_rustls::TlsConnector; + +// Perform the TLS handshake for the given connection with the given configuration +pub async fn handshake( + connector: &TlsConnector, + domain: &str, + io: S, +) -> Result> { + let mut stream = TlsStream::from_connector(connector, domain, io) + .await + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + + let handshake_result = stream.connect().await; + match handshake_result { + Ok(()) => Ok(stream), + Err(e) => { + let context = format!("TLS connect() failed: {e}, SNI: {domain}"); + Error::e_explain(TLSHandshakeFailure, context) + } + } +} diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs new file mode 100644 index 00000000..66e97d3d --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -0,0 +1,18 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS specific implementation +pub mod client; +pub mod server; +pub(super) mod stream; diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs new file mode 100644 index 00000000..98597394 --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -0,0 +1,94 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS server specific implementation + +use crate::protocols::tls::rustls::stream::TlsStream; +use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; +use crate::protocols::IO; +use async_trait::async_trait; +use log::warn; +use pingora_error::{ErrorType::*, OrErr, Result}; +use pingora_rustls::TlsAcceptor; +use std::pin::Pin; +use tokio::io::{AsyncRead, AsyncWrite}; + +#[async_trait] +impl ResumableAccept for TlsStream { + async fn start_accept(mut self: Pin<&mut Self>) -> Result { + // TODO: suspend cert callback + let res = self.accept().await; + + match res { + Ok(()) => Ok(true), + Err(e) => { + if e.etype == TLSWantX509Lookup { + Ok(false) + } else { + Err(e) + } + } + } + } + + async fn resume_accept(mut self: Pin<&mut Self>) -> Result<()> { + // TODO: unblock cert callback + self.accept().await + } +} + +async fn prepare_tls_stream(acceptor: &TlsAcceptor, io: S) -> Result> { + TlsStream::from_acceptor(acceptor, io) + .await + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) +} + +/// Perform TLS handshake for the given connection with the given configuration +pub(crate) async fn handshake(acceptor: &TlsAcceptor, io: S) -> Result> { + let mut stream = prepare_tls_stream(acceptor, io).await?; + stream + .accept() + .await + .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; + Ok(stream) +} + +/// Perform TLS handshake for the given connection with the given configuration and callbacks +/// callbacks are currently not supported within pingora Rustls and are ignored +pub(crate) async fn handshake_with_callback( + acceptor: &TlsAcceptor, + io: S, + _callbacks: &TlsAcceptCallbacks, +) -> Result> { + let mut tls_stream = prepare_tls_stream(acceptor, io).await?; + let done = Pin::new(&mut tls_stream).start_accept().await?; + if !done { + // TODO: verify if/how callback in handshake can be done using Rustls + warn!("Callacks are not supported with feature \"rustls\"."); + + Pin::new(&mut tls_stream) + .resume_accept() + .await + .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; + Ok(tls_stream) + } else { + Ok(tls_stream) + } +} + +#[ignore] +#[tokio::test] +async fn test_async_cert() { + todo!("callback support and test for Rustls") +} diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs new file mode 100644 index 00000000..f6c9f3be --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -0,0 +1,332 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::listeners::ALPN; +use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; +use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::SslDigest; +use crate::protocols::{GetProxyDigest, GetTimingDigest, Ssl, UniqueID}; +use crate::utils::tls::rustls::get_organization_serial; +use core::fmt; +use core::fmt::Formatter; +use pingora_error::ErrorType::{AcceptError, ConnectError, InternalError, TLSHandshakeFailure}; +use pingora_error::{Error, ImmutStr, OrErr, Result}; +use pingora_rustls::{hash_certificate, Accept, Connect, ServerName, TlsConnector}; +use pingora_rustls::{TlsAcceptor, TlsStream as RusTlsStream}; +use std::fmt::Debug; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::SystemTime; +use tokio::io; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use x509_parser::nom::AsBytes; + +/// The TLS connection +pub struct TlsStream { + pub(crate) stream: Option>, + connect: Option>, + accept: Option>, + pub(super) digest: Option>, + pub(super) timing: TimingDigest, +} + +impl Debug for TlsStream { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + fmt.debug_struct("InnerStream") + .field("stream", &self.stream) + .field("connect", { + if self.connect.is_some() { + &"Some(Connect)" + } else { + &"None" + } + }) + .field("accept", { + if self.accept.is_some() { + &"Some(Accept)" + } else { + &"None" + } + }) + .finish() + } +} + +impl TlsStream { + /// Create a new TLS connection from the given `stream` + /// + /// Using RustTLS the stream is only returned after the handshake. + /// + /// The caller needs to perform [`Self::accept()`] to perform the TLS handshake. + pub(crate) async fn from_acceptor(acceptor: &TlsAcceptor, stream: T) -> Result { + let accept = acceptor.accept(stream); + + Ok(TlsStream { + accept: Some(accept), + digest: None, + connect: None, + stream: None, + timing: Default::default(), + }) + } + + /// Create a new TLS connection from the given `stream` + /// + /// Using RustTLS the stream is only returned after the handshake. + /// + /// The caller needs to perform [`Self::connect()`] to perform the TLS handshake. + pub async fn from_connector(connector: &TlsConnector, domain: &str, stream: T) -> Result { + let server = ServerName::try_from(domain) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) + .explain_err(InternalError, |e| { + format!("failed to parse domain: {}, error: {}", domain, e) + })? + .to_owned(); + + let connect = connector.connect(server.to_owned(), stream); + + Ok(TlsStream { + accept: None, + digest: None, + connect: Some(connect), + stream: None, + timing: Default::default(), + }) + } +} + +impl TlsStream { + /// Connect to the remote TLS server as a client + pub(crate) async fn connect(&mut self) -> Result<()> { + let connect = &mut self.connect; + + if let Some(ref mut connect) = connect { + let stream = connect + .await + .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; + self.stream = Some(RusTlsStream::Client(stream)); + self.connect = None; + + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } else { + Err(Error::explain( + ConnectError, + ImmutStr::from("TLS connect not available to perform handshake."), + )) + } + } + + /// Finish the TLS handshake from client as a server + /// no-op implementation within Rustls, handshake is performed during creation of stream. + pub(crate) async fn accept(&mut self) -> Result<()> { + if let Some(ref mut accept) = &mut self.accept { + let stream = accept + .await + .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; + self.stream = Some(RusTlsStream::Server(stream)); + self.connect = None; + + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } else { + Err(Error::explain( + AcceptError, + ImmutStr::from("TLS accept not available to perform handshake."), + )) + } + } + + pub(crate) fn digest(&mut self) -> Option> { + Some(Arc::new(SslDigest::from_stream(&self.stream))) + } +} + +impl GetSocketDigest for TlsStream +where + S: GetSocketDigest, +{ + fn get_socket_digest(&self) -> Option> { + if let Some(stream) = self.stream.as_ref() { + stream.get_ref().0.get_socket_digest() + } else { + None + } + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + self.stream + .as_mut() + .unwrap() + .get_mut() + .0 + .set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for TlsStream +where + S: GetTimingDigest, +{ + fn get_timing_digest(&self) -> Vec> { + self.stream + .as_ref() + .unwrap() + .get_ref() + .0 + .get_timing_digest() + } +} + +impl GetProxyDigest for TlsStream +where + S: GetProxyDigest, +{ + fn get_proxy_digest(&self) -> Option> { + if let Some(stream) = self.stream.as_ref() { + stream.get_ref().0.get_proxy_digest() + } else { + None + } + } +} + +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl Ssl for TlsStream { + fn get_ssl_digest(&self) -> Option> { + self.digest.clone() + } + + fn selected_alpn_proto(&self) -> Option { + let st = self.stream.as_ref(); + if let Some(stream) = st { + let proto = stream.get_ref().1.alpn_protocol(); + match proto { + None => None, + Some(raw) => ALPN::from_wire_selected(raw), + } + } else { + None + } + } +} + +impl SslDigest { + fn from_stream(stream: &Option>) -> Self { + let stream = stream.as_ref().unwrap(); + let (_io, session) = stream.get_ref(); + let protocol = session.protocol_version(); + let cipher_suite = session.negotiated_cipher_suite(); + let peer_certificates = session.peer_certificates(); + + let cipher = match cipher_suite { + Some(suite) => match suite.suite().as_str() { + Some(suite_str) => suite_str, + None => "", + }, + None => "", + }; + + let version = match protocol { + Some(proto) => match proto.as_str() { + Some(ver) => ver, + None => "", + }, + None => "", + }; + + let cert_digest = match peer_certificates { + Some(certs) => match certs.first() { + Some(cert) => hash_certificate(cert.clone()), + None => vec![], + }, + None => vec![], + }; + + let (organization, serial_number) = match peer_certificates { + Some(certs) => match certs.first() { + Some(cert) => { + let (organization, serial) = get_organization_serial(cert.as_bytes()); + (organization, Some(serial)) + } + None => (None, None), + }, + None => (None, None), + }; + + SslDigest { + cipher: &cipher, + version: &version, + organization, + serial_number, + cert_digest, + } + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.stream.as_ref().unwrap().get_ref().0.id() + } +} diff --git a/pingora-core/src/protocols/tls/server.rs b/pingora-core/src/protocols/tls/server.rs new file mode 100644 index 00000000..a4bd2e49 --- /dev/null +++ b/pingora-core/src/protocols/tls/server.rs @@ -0,0 +1,75 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! TLS server specific implementation + +use std::pin::Pin; + +use async_trait::async_trait; +use log::warn; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; + +use pingora_error::Result; + +use crate::protocols::tls::TlsStream; +use crate::protocols::Shutdown; + +#[cfg(not(feature = "rustls"))] +use crate::tls::ssl::SslRef; + +/// The APIs to customize things like certificate during TLS server side handshake +#[async_trait] +pub trait TlsAccept { + // TODO: return error? + /// This function is called in the middle of a TLS handshake. Structs who implement this function + /// should provide tls certificate and key to the [SslRef] via [ext::ssl_use_certificate] and [ext::ssl_use_private_key]. + #[cfg(not(feature = "rustls"))] + async fn certificate_callback(&self, _ssl: &mut SslRef) -> () { + // does nothing by default + } +} + +pub type TlsAcceptCallbacks = Box; + +#[async_trait] +impl Shutdown for TlsStream +where + S: AsyncRead + AsyncWrite + Sync + Unpin + Send, +{ + async fn shutdown(&mut self) { + match ::shutdown(self).await { + Ok(()) => {} + Err(e) => { + warn!("TLS shutdown failed, {e}"); + } + } + } +} + +/// Resumable TLS server side handshake. +#[async_trait] +pub trait ResumableAccept { + /// Start a resumable TLS accept handshake. + /// + /// * `Ok(true)` when the handshake is finished + /// * `Ok(false)`` when the handshake is paused midway + /// + /// For now, accept will only pause when a certificate is needed. + async fn start_accept(self: Pin<&mut Self>) -> Result; + + /// Continue the TLS handshake + /// + /// This function should be called after the certificate is provided. + async fn resume_accept(self: Pin<&mut Self>) -> Result<()>; +} diff --git a/pingora-core/src/upstreams/peer.rs b/pingora-core/src/upstreams/peer.rs index d0c8125a..0c60974c 100644 --- a/pingora-core/src/upstreams/peer.rs +++ b/pingora-core/src/upstreams/peer.rs @@ -32,10 +32,13 @@ use std::time::Duration; use crate::protocols::l4::socket::SocketAddr; use crate::protocols::ConnFdReusable; use crate::protocols::TcpKeepalive; -use crate::tls::x509::X509; -use crate::utils::{get_organization_unit, CertKey}; +#[cfg(not(feature = "rustls"))] +use crate::utils::tls::boringssl_openssl::{get_not_after, get_organizational_unit}; +#[cfg(feature = "rustls")] +use crate::utils::tls::rustls::{get_not_after, get_organizational_unit}; +use crate::utils::tls::CertKey; -pub use crate::protocols::ssl::ALPN; +pub use crate::protocols::tls::ALPN; /// The interface to trace the connection pub trait Tracing: Send + Sync + std::fmt::Debug { @@ -66,7 +69,7 @@ pub trait Peer: Display + Clone { fn tls(&self) -> bool; /// The SNI to send, if TLS is used fn sni(&self) -> &str; - /// To decide whether a [`Peer`] can use the connection established by another [`Peer`]. + /// To decide whether a [`Peer`] can use the connection established by another [`Peer`]. /// /// The connections to two peers are considered reusable to each other if their reuse hashes are /// the same @@ -144,7 +147,7 @@ pub trait Peer: Display + Clone { /// Get the CA cert to use to validate the server cert. /// /// If not set, the default CAs will be used. - fn get_ca(&self) -> Option<&Arc>> { + fn get_ca(&self) -> Option<&Arc<[Box<[u8]>]>> { match self.get_peer_options() { Some(opt) => opt.ca.as_ref(), None => None, @@ -304,7 +307,7 @@ pub struct PeerOptions { /* accept the cert if it's CN matches the SNI or this name */ pub alternative_cn: Option, pub alpn: ALPN, - pub ca: Option>>, + pub ca: Option]>>, pub tcp_keepalive: Option, pub tcp_recv_buf: Option, pub dscp: Option, @@ -385,8 +388,8 @@ impl Display for PeerOptions { write!( f, "CA: {}, expire: {},", - get_organization_unit(ca).unwrap_or_default(), - ca.not_after() + get_organizational_unit(ca).unwrap_or_default(), + get_not_after(ca), )?; } } diff --git a/pingora-core/src/utils/mod.rs b/pingora-core/src/utils/mod.rs index c36f7c82..8057d58c 100644 --- a/pingora-core/src/utils/mod.rs +++ b/pingora-core/src/utils/mod.rs @@ -15,12 +15,9 @@ //! This module contains various types that make it easier to work with bytes and X509 //! certificates. -// TODO: move below to its own mod -use crate::tls::{nid::Nid, pkey::PKey, pkey::Private, x509::X509}; -use crate::Result; +pub mod tls; + use bytes::Bytes; -use pingora_error::{ErrorType::*, OrErr}; -use std::hash::{Hash, Hasher}; /// A `BufRef` is a reference to a buffer of bytes. It removes the need for self-referential data /// structures. It is safe to use as long as the underlying buffer does not get mutated. @@ -108,125 +105,3 @@ pub const EMPTY_KV_REF: KVRef = KVRef { name: BufRef(0, 0), value: BufRef(0, 0), }; - -fn get_subject_name(cert: &X509, name_type: Nid) -> Option { - cert.subject_name() - .entries_by_nid(name_type) - .next() - .map(|name| { - name.data() - .as_utf8() - .map(|s| s.to_string()) - .unwrap_or_default() - }) -} - -/// Return the organization associated with the X509 certificate. -pub fn get_organization(cert: &X509) -> Option { - get_subject_name(cert, Nid::ORGANIZATIONNAME) -} - -/// Return the common name associated with the X509 certificate. -pub fn get_common_name(cert: &X509) -> Option { - get_subject_name(cert, Nid::COMMONNAME) -} - -/// Return the common name associated with the X509 certificate. -pub fn get_organization_unit(cert: &X509) -> Option { - get_subject_name(cert, Nid::ORGANIZATIONALUNITNAME) -} - -/// Return the serial number associated with the X509 certificate as a hexadecimal value. -pub fn get_serial(cert: &X509) -> Result { - let bn = cert - .serial_number() - .to_bn() - .or_err(InvalidCert, "Invalid serial")?; - let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; - - let hex_str: &str = hex.as_ref(); - Ok(hex_str.to_owned()) -} - -/// This type contains a list of one or more certificates and an associated private key. The leaf -/// certificate should always be first. -#[derive(Clone)] -pub struct CertKey { - certificates: Vec, - key: PKey, -} - -impl CertKey { - /// Create a new `CertKey` given a list of certificates and a private key. - pub fn new(certificates: Vec, key: PKey) -> CertKey { - assert!( - !certificates.is_empty(), - "expected a non-empty vector of certificates in CertKey::new" - ); - - CertKey { certificates, key } - } - - /// Peek at the leaf certificate. - pub fn leaf(&self) -> &X509 { - // This is safe due to the assertion above. - &self.certificates[0] - } - - /// Return the key. - pub fn key(&self) -> &PKey { - &self.key - } - - /// Return a slice of intermediate certificates. An empty slice means there are none. - pub fn intermediates(&self) -> &[X509] { - if self.certificates.len() <= 1 { - return &[]; - } - &self.certificates[1..] - } - - /// Return the organization from the leaf certificate. - pub fn organization(&self) -> Option { - get_organization(self.leaf()) - } - - /// Return the serial from the leaf certificate. - pub fn serial(&self) -> Result { - get_serial(self.leaf()) - } -} - -impl Hash for CertKey { - fn hash(&self, state: &mut H) { - for certificate in &self.certificates { - if let Ok(serial) = get_serial(certificate) { - serial.hash(state) - } - } - } -} - -// hide private key -impl std::fmt::Debug for CertKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CertKey") - .field("X509", &self.leaf()) - .finish() - } -} - -impl std::fmt::Display for CertKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let leaf = self.leaf(); - if let Some(cn) = get_common_name(leaf) { - // Write CN if it exists - write!(f, "CN: {cn},")?; - } else if let Some(org_unit) = get_organization_unit(leaf) { - // CA cert might not have CN, so print its unit name instead - write!(f, "Org Unit: {org_unit},")?; - } - write!(f, ", expire: {}", leaf.not_after()) - // ignore the details of the private key - } -} diff --git a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs new file mode 100644 index 00000000..0882cd1c --- /dev/null +++ b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs @@ -0,0 +1,107 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various helpers that make it easier to work with X509 certificates. + +use pingora_error::ErrorType::{InternalError, InvalidCert}; +use pingora_error::OrErr; + +use crate::tls::nid::Nid; +use crate::tls::pkey::{PKey, Private}; +use crate::tls::x509::X509; + +fn get_subject_name(cert: &X509, name_type: Nid) -> Option { + cert.subject_name() + .entries_by_nid(name_type) + .next() + .map(|name| { + name.data() + .as_utf8() + .map(|s| s.to_string()) + .unwrap_or_default() + }) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_x509_organization(cert: &X509) -> Option { + get_subject_name(cert, Nid::ORGANIZATIONNAME) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_organization(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::ORGANIZATIONNAME) +} + +/// Return the common name associated with the X509 certificate. +pub fn get_common_name(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::COMMONNAME) +} + +/// Return the organizational unit associated with the X509 certificate. +pub fn get_organizational_unit(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::ORGANIZATIONALUNITNAME) +} + +/// Return the common name associated with the X509 certificate. +pub fn get_not_after(cert: &[u8]) -> String { + let cert = der_to_x509(cert).unwrap(); + cert.not_after().to_string() +} + +/// Return the serial number associated with the X509 certificate as a hexadecimal value. +pub fn get_serial(cert: &[u8]) -> pingora_error::Result { + let cert = der_to_x509(cert).unwrap(); + let bn = cert + .serial_number() + .to_bn() + .or_err(InvalidCert, "Invalid serial")?; + let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; + + let hex_str: &str = hex.as_ref(); + Ok(hex_str.to_owned()) +} + +pub fn get_x509_serial(cert: &X509) -> pingora_error::Result { + let bn = cert + .serial_number() + .to_bn() + .or_err(InvalidCert, "Invalid serial")?; + let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; + + let hex_str: &str = hex.as_ref(); + Ok(hex_str.to_owned()) +} + +pub fn der_to_x509(ca: &[u8]) -> pingora_error::Result { + let cert = X509::from_der(ca).explain_err(InvalidCert, |e| { + format!( + "Failed to convert ca certificate in DER form to X509 cert. Error: {:?}", + e + ) + })?; + Ok(cert) +} + +pub fn der_to_private_key(key: &[u8]) -> pingora_error::Result> { + let key = PKey::private_key_from_der(key).explain_err(InternalError, |e| { + format!( + "Failed to convert private key in DER form to Pkey. Error: {:?}", + e + ) + })?; + Ok(key) +} diff --git a/pingora-core/src/utils/tls/mod.rs b/pingora-core/src/utils/tls/mod.rs new file mode 100644 index 00000000..fc4447ad --- /dev/null +++ b/pingora-core/src/utils/tls/mod.rs @@ -0,0 +1,112 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various types that make it easier to work with bytes and X509 +//! certificates. + +#[cfg(not(feature = "rustls"))] +pub mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub mod rustls; + +#[cfg(not(feature = "rustls"))] +use boringssl_openssl::{ + get_common_name, get_not_after, get_organization, get_organizational_unit, get_serial, +}; +#[cfg(feature = "rustls")] +use rustls::{ + get_common_name, get_not_after, get_organization, get_organizational_unit, get_serial, +}; +use std::hash::{Hash, Hasher}; + +/// This type contains a list of one or more certificates and an associated private key. The leaf +/// certificate should always be first. The certificates and keys are stored in Vec DER encoded +/// form for usage within OpenSSL/BoringSSL & RusTLS. +#[derive(Clone)] +pub struct CertKey { + certificates: Vec>, + key: Vec, +} + +impl CertKey { + /// Create a new `CertKey` given a list of certificates and a private key. + pub fn new(certificates: Vec>, key: Vec) -> CertKey { + assert!( + !certificates.is_empty() && !certificates.first().unwrap().is_empty(), + "expected a non-empty vector of certificates in CertKey::new" + ); + + CertKey { certificates, key } + } + + /// Peek at the leaf certificate. + pub fn leaf(&self) -> &Vec { + // This is safe due to the assertion above. + &self.certificates[0] + } + + /// Return the key. + pub fn key(&self) -> &Vec { + &self.key + } + + /// Return a slice of intermediate certificates. An empty slice means there are none. + pub fn intermediates(&self) -> Vec<&Vec> { + self.certificates.iter().skip(1).collect() + } + + /// Return the organization from the leaf certificate. + pub fn organization(&self) -> Option { + get_organization(self.leaf()) + } + + /// Return the serial from the leaf certificate. + pub fn serial(&self) -> String { + get_serial(self.leaf()).unwrap() + } +} + +// hide private key +impl std::fmt::Debug for CertKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CertKey") + .field("X509", &self.leaf()) + .finish() + } +} + +impl std::fmt::Display for CertKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let leaf = self.leaf(); + if let Some(cn) = get_common_name(leaf) { + // Write CN if it exists + write!(f, "CN: {cn},")?; + } else if let Some(org_unit) = get_organizational_unit(leaf) { + // CA cert might not have CN, so print its unit name instead + write!(f, "Org Unit: {org_unit},")?; + } + write!(f, ", expire: {}", get_not_after(leaf)) + // ignore the details of the private key + } +} + +impl Hash for CertKey { + fn hash(&self, state: &mut H) { + for certificate in &self.certificates { + if let Ok(serial) = get_serial(certificate) { + serial.hash(state) + } + } + } +} diff --git a/pingora-core/src/utils/tls/rustls/mod.rs b/pingora-core/src/utils/tls/rustls/mod.rs new file mode 100644 index 00000000..41520d79 --- /dev/null +++ b/pingora-core/src/utils/tls/rustls/mod.rs @@ -0,0 +1,71 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various helpers that make it easier to work with X509 certificates. + +use pingora_error::Result; +use x509_parser::prelude::FromDer; + +pub fn get_organization_serial(cert: &[u8]) -> (Option, String) { + let serial = get_serial(cert).expect("Failed to get serial for certificate."); + (get_organization(cert), serial) +} + +pub fn get_serial(cert: &[u8]) -> Result { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + Ok(x509cert.raw_serial_as_string()) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_organization(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert + .subject + .iter_organization() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} + +/// Return the organization unit associated with the X509 certificate. +pub fn get_organizational_unit(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert + .subject + .iter_organizational_unit() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} + +/// Return the organization unit associated with the X509 certificate. +pub fn get_not_after(cert: &[u8]) -> String { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert.validity.not_after.to_string() +} + +pub fn get_common_name(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert + .subject + .iter_common_name() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} diff --git a/pingora-error/src/lib.rs b/pingora-error/src/lib.rs index d8567633..935b2520 100644 --- a/pingora-error/src/lib.rs +++ b/pingora-error/src/lib.rs @@ -105,6 +105,7 @@ pub enum ErrorType { ConnectTimedout, ConnectRefused, ConnectNoRoute, + TLSWantX509Lookup, TLSHandshakeFailure, TLSHandshakeTimedout, InvalidCert, @@ -164,6 +165,7 @@ impl ErrorType { ErrorType::ConnectRefused => "ConnectRefused", ErrorType::ConnectNoRoute => "ConnectNoRoute", ErrorType::ConnectProxyFailure => "ConnectProxyFailure", + ErrorType::TLSWantX509Lookup => "TLSWantX509Lookup", ErrorType::TLSHandshakeFailure => "TLSHandshakeFailure", ErrorType::TLSHandshakeTimedout => "TLSHandshakeTimedout", ErrorType::InvalidCert => "InvalidCert", diff --git a/pingora-load-balancing/Cargo.toml b/pingora-load-balancing/Cargo.toml index 904e9f69..6cba1a8a 100644 --- a/pingora-load-balancing/Cargo.toml +++ b/pingora-load-balancing/Cargo.toml @@ -36,3 +36,4 @@ log = { workspace = true } default = ["openssl"] openssl = ["pingora-core/openssl"] boringssl = ["pingora-core/boringssl"] +rustls = ["pingora-core/rustls"] \ No newline at end of file diff --git a/pingora-proxy/Cargo.toml b/pingora-proxy/Cargo.toml index 65f7f677..eab96320 100644 --- a/pingora-proxy/Cargo.toml +++ b/pingora-proxy/Cargo.toml @@ -44,7 +44,7 @@ env_logger = "0.9" hyperlocal = "0.8" hyper = "0.14" tokio-tungstenite = "0.20.1" -pingora-load-balancing = { version = "0.3.0", path = "../pingora-load-balancing" } +pingora-load-balancing = { version = "0.3.0", path = "../pingora-load-balancing", default-features = false } prometheus = "0" futures-util = "0.3" serde = { version = "1.0", features = ["derive"] } @@ -55,3 +55,4 @@ serde_yaml = "0.8" default = ["openssl"] openssl = ["pingora-core/openssl", "pingora-cache/openssl"] boringssl = ["pingora-core/boringssl", "pingora-cache/boringssl"] +rustls = ["pingora-core/rustls", "pingora-cache/rustls"] \ No newline at end of file diff --git a/pingora-proxy/tests/test_basic.rs b/pingora-proxy/tests/test_basic.rs index 744f9b70..34e5933a 100644 --- a/pingora-proxy/tests/test_basic.rs +++ b/pingora-proxy/tests/test_basic.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod utils; - use hyper::{body::HttpBody, header::HeaderValue, Body, Client}; use hyperlocal::{UnixClientExt, Uri}; use reqwest::{header, StatusCode}; use utils::server_utils::init; +mod utils; + fn is_specified_port(port: u16) -> bool { (1..65535).contains(&port) } @@ -73,7 +73,12 @@ async fn test_h2_to_h1() { .build() .unwrap(); - let res = client.get("https://127.0.0.1:6150").send().await.unwrap(); + let res = client + .get("https://127.0.0.1:6150") + .header("sni", "openrusty.org") + .send() + .await + .unwrap(); assert_eq!(res.status(), reqwest::StatusCode::OK); assert_eq!(res.version(), reqwest::Version::HTTP_2); @@ -112,6 +117,7 @@ async fn test_h2_to_h2() { let res = client .get("https://127.0.0.1:6150") + .header("sni", "openrusty.org") .header("x-h2", "true") .send() .await @@ -165,6 +171,7 @@ async fn test_h2c_to_h2c() { assert_eq!(body.as_ref(), b"Hello World!\n"); } +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_h2_to_h2_host_override() { init(); @@ -200,6 +207,7 @@ async fn test_h2_to_h2_upload() { let res = client .get("https://127.0.0.1:6150/echo") + .header("sni", "openrusty.org") .header("x-h2", "true") .body(payload) .send() @@ -223,6 +231,7 @@ async fn test_h2_to_h1_upload() { let res = client .get("https://127.0.0.1:6150/echo") + .header("sni", "openrusty.org") .body(payload) .send() .await @@ -288,7 +297,10 @@ async fn test_simple_proxy_uds_peer() { assert!(is_specified_port(sockaddr.port())); assert_eq!(headers["x-upstream-client-addr"], "unset"); // unnamed UDS - assert_eq!(headers["x-upstream-server-addr"], "/tmp/nginx-test.sock"); + assert_eq!( + headers["x-upstream-server-addr"], + "/tmp/pingora_nginx_test.sock" + ); let body = res.text().await.unwrap(); assert_eq!(body, "Hello World!\n"); @@ -419,6 +431,8 @@ async fn test_dropped_conn() { test_dropped_conn_post_body_over().await; } +// currently not supported with Rustls implementaiton +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_tls_no_verify() { init(); @@ -448,6 +462,8 @@ async fn test_tls_verify_sni_not_host() { assert_eq!(res.status(), StatusCode::OK); } +// currently not supported with Rustls implementaiton +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_tls_none_verify_host() { init(); diff --git a/pingora-proxy/tests/utils/cert.rs b/pingora-proxy/tests/utils/cert.rs index 674a3ac7..0ff149c3 100644 --- a/pingora-proxy/tests/utils/cert.rs +++ b/pingora-proxy/tests/utils/cert.rs @@ -13,35 +13,57 @@ // limitations under the License. use once_cell::sync::Lazy; -use pingora_core::tls::pkey::{PKey, Private}; -use pingora_core::tls::x509::X509; use std::fs; -pub static ROOT_CERT: Lazy = Lazy::new(|| load_cert("keys/root.crt")); -pub static ROOT_KEY: Lazy> = Lazy::new(|| load_key("keys/root.key")); -pub static INTERMEDIATE_CERT: Lazy = Lazy::new(|| load_cert("keys/intermediate.crt")); -pub static INTERMEDIATE_KEY: Lazy> = Lazy::new(|| load_key("keys/intermediate.key")); -pub static LEAF_CERT: Lazy = Lazy::new(|| load_cert("keys/leaf.crt")); -pub static LEAF2_CERT: Lazy = Lazy::new(|| load_cert("keys/leaf2.crt")); -pub static LEAF_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf.key")); -pub static LEAF2_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf2.key")); -pub static SERVER_CERT: Lazy = Lazy::new(|| load_cert("keys/server.crt")); -pub static SERVER_KEY: Lazy> = Lazy::new(|| load_key("keys/key.pem")); -pub static CURVE_521_TEST_KEY: Lazy> = +#[cfg(feature = "rustls")] +use pingora_core::tls::{load_pem_file_ca, load_pem_file_private_key}; +#[cfg(not(feature = "rustls"))] +use pingora_core::tls::{ + pkey::{PKey, Private}, + x509::X509, +}; + +//pub static ROOT_CERT: Lazy = Lazy::new(|| load_cert("keys/root.crt")); +//pub static ROOT_KEY: Lazy> = Lazy::new(|| load_key("keys/root.key")); +pub static INTERMEDIATE_CERT: Lazy> = Lazy::new(|| load_cert("keys/intermediate.crt")); +//pub static INTERMEDIATE_KEY: Lazy> = Lazy::new(|| load_key("keys/intermediate.key")); +pub static LEAF_CERT: Lazy> = Lazy::new(|| load_cert("keys/leaf.crt")); +pub static LEAF2_CERT: Lazy> = Lazy::new(|| load_cert("keys/leaf2.crt")); +pub static LEAF_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf.key")); +pub static LEAF2_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf2.key")); +//pub static SERVER_CERT: Lazy = Lazy::new(|| load_cert("keys/server.crt")); +//pub static SERVER_KEY: Lazy> = Lazy::new(|| load_key("keys/key.pem")); +pub static CURVE_521_TEST_KEY: Lazy> = Lazy::new(|| load_key("keys/curve_test.521.key.pem")); -pub static CURVE_521_TEST_CERT: Lazy = Lazy::new(|| load_cert("keys/curve_test.521.crt")); -pub static CURVE_384_TEST_KEY: Lazy> = +pub static CURVE_521_TEST_CERT: Lazy> = Lazy::new(|| load_cert("keys/curve_test.521.crt")); +pub static CURVE_384_TEST_KEY: Lazy> = Lazy::new(|| load_key("keys/curve_test.384.key.pem")); -pub static CURVE_384_TEST_CERT: Lazy = Lazy::new(|| load_cert("keys/curve_test.384.crt")); +pub static CURVE_384_TEST_CERT: Lazy> = Lazy::new(|| load_cert("keys/curve_test.384.crt")); -fn load_cert(path: &str) -> X509 { +#[cfg(not(feature = "rustls"))] +fn load_cert(path: &str) -> Vec { let path = format!("{}/{path}", super::conf_dir()); let cert_bytes = fs::read(path).unwrap(); - X509::from_pem(&cert_bytes).unwrap() + X509::from_pem(&cert_bytes).unwrap().to_der().unwrap() } - -fn load_key(path: &str) -> PKey { +#[cfg(not(feature = "rustls"))] +fn load_key(path: &str) -> Vec { let path = format!("{}/{path}", super::conf_dir()); let key_bytes = fs::read(path).unwrap(); - PKey::private_key_from_pem(&key_bytes).unwrap() + PKey::private_key_from_pem(&key_bytes) + .unwrap() + .private_key_to_der() + .unwrap() +} + +#[cfg(feature = "rustls")] +fn load_cert(path: &str) -> Vec { + let path = format!("{}/{path}", super::conf_dir()); + load_pem_file_ca(&path) +} + +#[cfg(feature = "rustls")] +fn load_key(path: &str) -> Vec { + let path = format!("{}/{path}", super::conf_dir()); + load_pem_file_private_key(&path) } diff --git a/pingora-proxy/tests/utils/conf/keys/README.md b/pingora-proxy/tests/utils/conf/keys/README.md index 13965cd6..1242ea85 100644 --- a/pingora-proxy/tests/utils/conf/keys/README.md +++ b/pingora-proxy/tests/utils/conf/keys/README.md @@ -16,3 +16,13 @@ openssl ecparam -genkey -name secp256r1 -noout -out test_key.pem openssl req -new -key test_key.pem -out test.csr openssl x509 -req -in test.csr -CA server.crt -CAkey key.pem -CAcreateserial -CAserial test.srl -out test.crt -days 3650 -sha256 ``` + +``` +openssl version +# OpenSSL 3.1.1 + +echo '[v3_req]' > openssl.cnf +openssl req -config openssl.cnf -new -x509 -key key.pem -out server_rustls.crt -days 3650 -sha256 \ + -subj '/C=US/ST=CA/L=San Francisco/O=Cloudflare, Inc/CN=openrusty.org' \ + -addext "subjectAltName=DNS:*.openrusty.org,DNS:openrusty.org,DNS:cat.com,DNS:dog.com" +``` \ No newline at end of file diff --git a/pingora-proxy/tests/utils/conf/keys/server.crt b/pingora-proxy/tests/utils/conf/keys/server.crt deleted file mode 100644 index afb2d1e0..00000000 --- a/pingora-proxy/tests/utils/conf/keys/server.crt +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB9zCCAZ2gAwIBAgIUMI7aLvTxyRFCHhw57hGt4U6yupcwCgYIKoZIzj0EAwIw -ZDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMRgwFgYDVQQKDA9DbG91ZGZsYXJlLCBJbmMxFjAUBgNVBAMMDW9wZW5ydXN0 -eS5vcmcwHhcNMjIwNDExMjExMzEzWhcNMzIwNDA4MjExMzEzWjBkMQswCQYDVQQG -EwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xGDAWBgNV -BAoMD0Nsb3VkZmxhcmUsIEluYzEWMBQGA1UEAwwNb3BlbnJ1c3R5Lm9yZzBZMBMG -ByqGSM49AgEGCCqGSM49AwEHA0IABNn/9RZtR48knaJD6tk9BdccaJfZ0hGEPn6B -SDXmlmJPhcTBqa4iUwW/ABpGvO3FpJcNWasrX2k+qZLq3g205MKjLTArMCkGA1Ud -EQQiMCCCDyoub3BlbnJ1c3R5Lm9yZ4INb3BlbnJ1c3R5Lm9yZzAKBggqhkjOPQQD -AgNIADBFAiAjISZ9aEKmobKGlT76idO740J6jPaX/hOrm41MLeg69AIhAJqKrSyz -wD/AAF5fR6tXmBqlnpQOmtxfdy13wDr4MT3h ------END CERTIFICATE----- diff --git a/pingora-proxy/tests/keys/server.crt b/pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.crt similarity index 100% rename from pingora-proxy/tests/keys/server.crt rename to pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.crt diff --git a/pingora-proxy/tests/utils/conf/keys/server.csr b/pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.csr similarity index 100% rename from pingora-proxy/tests/utils/conf/keys/server.csr rename to pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.csr diff --git a/pingora-proxy/tests/utils/conf/keys/server_rustls.crt b/pingora-proxy/tests/utils/conf/keys/server_rustls.crt new file mode 100644 index 00000000..c02a214b --- /dev/null +++ b/pingora-proxy/tests/utils/conf/keys/server_rustls.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICJzCCAc6gAwIBAgIUU+G0acG/uiMu1ZDSjlcoY4gH53QwCgYIKoZIzj0EAwIw +ZDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMRgwFgYDVQQKDA9DbG91ZGZsYXJlLCBJbmMxFjAUBgNVBAMMDW9wZW5ydXN0 +eS5vcmcwHhcNMjQwNzI0MTMzOTQ4WhcNMzQwNzIyMTMzOTQ4WjBkMQswCQYDVQQG +EwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xGDAWBgNV +BAoMD0Nsb3VkZmxhcmUsIEluYzEWMBQGA1UEAwwNb3BlbnJ1c3R5Lm9yZzBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABNn/9RZtR48knaJD6tk9BdccaJfZ0hGEPn6B +SDXmlmJPhcTBqa4iUwW/ABpGvO3FpJcNWasrX2k+qZLq3g205MKjXjBcMDsGA1Ud +EQQ0MDKCDyoub3BlbnJ1c3R5Lm9yZ4INb3BlbnJ1c3R5Lm9yZ4IHY2F0LmNvbYIH +ZG9nLmNvbTAdBgNVHQ4EFgQUnfYAFWyQnSN57IGokj7jcz8ChJQwCgYIKoZIzj0E +AwIDRwAwRAIgQr+Ly2cH04CncbnbhUf4hBl5frTp1pXgGnn8dYjd+UcCICuunEtp +H/a42/sVGBFvjS6FOFe6ZDs4oWBNEqQSw0S2 +-----END CERTIFICATE----- diff --git a/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf b/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf index a41a743d..fdc88a88 100644 --- a/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf +++ b/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf @@ -6,7 +6,7 @@ error_log /dev/stdout; #error_log logs/error.log notice; #error_log logs/error.log info; -pid /tmp/mock_origin.pid; +pid /tmp/pingora_mock_origin.pid; master_process off; daemon off; @@ -85,7 +85,7 @@ http { listen 8001; listen [::]:8000; #listen 8443 ssl; - listen unix:/tmp/nginx-test.sock; + listen unix:/tmp/pingora_nginx_test.sock; listen 8443 ssl http2; server_name localhost; @@ -97,6 +97,9 @@ http { # for benchmark http2_max_requests 999999; + # increase max body size for /upload/ test + client_max_body_size 128m; + #charset koi8-r; #access_log logs/host.access.log main; @@ -285,7 +288,6 @@ http { } location /upload/ { - client_max_body_size 1G; content_by_lua_block { ngx.req.read_body() ngx.print(string.rep("A", 64)) diff --git a/pingora-proxy/tests/utils/mock_origin.rs b/pingora-proxy/tests/utils/mock_origin.rs index db84f8df..de13be69 100644 --- a/pingora-proxy/tests/utils/mock_origin.rs +++ b/pingora-proxy/tests/utils/mock_origin.rs @@ -13,15 +13,35 @@ // limitations under the License. use once_cell::sync::Lazy; +use std::path::Path; use std::process; use std::{thread, time}; pub static MOCK_ORIGIN: Lazy = Lazy::new(init); fn init() -> bool { + #[cfg(feature = "rustls")] + let src_cert_path = format!( + "{}/tests/utils/conf/keys/server_rustls.crt", + env!("CARGO_MANIFEST_DIR") + ); + #[cfg(not(feature = "rustls"))] + let src_cert_path = format!( + "{}/tests/utils/conf/keys/server_boringssl_openssl.crt", + env!("CARGO_MANIFEST_DIR") + ); + + let mut dst_cert_path = format!("{}/tests/keys/server.crt", env!("CARGO_MANIFEST_DIR")); + std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); + dst_cert_path = format!( + "{}/tests/utils/conf/keys/server.crt", + env!("CARGO_MANIFEST_DIR") + ); + std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); + // TODO: figure out a way to kill openresty when exiting process::Command::new("pkill") - .args(["-F", "/tmp/mock_origin.pid"]) + .args(["-F", "/tmp/pingora_mock_origin.pid"]) .spawn() .unwrap(); let _origin = thread::spawn(|| { diff --git a/pingora-proxy/tests/utils/server_utils.rs b/pingora-proxy/tests/utils/server_utils.rs index ec1a9627..4dfdb9e3 100644 --- a/pingora-proxy/tests/utils/server_utils.rs +++ b/pingora-proxy/tests/utils/server_utils.rs @@ -32,7 +32,7 @@ use pingora_core::protocols::{l4::socket::SocketAddr, Digest}; use pingora_core::server::configuration::Opt; use pingora_core::services::Service; use pingora_core::upstreams::peer::HttpPeer; -use pingora_core::utils::CertKey; +use pingora_core::utils::tls::CertKey; use pingora_error::{Error, ErrorSource, Result}; use pingora_http::{RequestHeader, ResponseHeader}; use pingora_proxy::{ProxyHttp, Session}; @@ -281,7 +281,7 @@ impl ProxyHttp for ExampleProxyHttp { let req = session.req_header(); if req.headers.contains_key("x-uds-peer") { return Ok(Box::new(HttpPeer::new_uds( - "/tmp/nginx-test.sock", + "/tmp/pingora_nginx_test.sock", false, "".to_string(), )?)); diff --git a/pingora-rustls/Cargo.toml b/pingora-rustls/Cargo.toml new file mode 100644 index 00000000..c4f6134e --- /dev/null +++ b/pingora-rustls/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "pingora-rustls" +version = "0.3.0" +authors = ["Harald Gutmann "] +license = "Apache-2.0" +edition = "2021" +repository = "https://github.com/cloudflare/pingora" +categories = ["asynchronous", "network-programming"] +keywords = ["async", "tls", "ssl", "pingora"] +description = """ +RusTLS async APIs for Pingora. +""" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "pingora_rustls" +path = "src/lib.rs" + +[dependencies] +log = "0.4.21" +ring = "0.17.8" +rustls = "0.23.12" +rustls-native-certs = "0.7.1" +rustls-pemfile = "2.1.2" +rustls-pki-types = "1.7.0" +tokio-rustls = "0.26.0" + +[dev-dependencies] +tokio-test = "0.4.3" +tokio = { workspace = true, features = ["full"] } diff --git a/pingora-rustls/LICENSE b/pingora-rustls/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/pingora-rustls/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs new file mode 100644 index 00000000..be6a42c8 --- /dev/null +++ b/pingora-rustls/src/lib.rs @@ -0,0 +1,157 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![warn(clippy::all)] + +use std::fs::File; +use std::io::BufReader; + +use log::{error, warn}; +pub use rustls::{version, ClientConfig, RootCertStore, ServerConfig, Stream}; +pub use rustls_native_certs::load_native_certs; +use rustls_pemfile::Item; +pub use rustls_pki_types::{CertificateDer, PrivateKeyDer, ServerName}; +pub use tokio_rustls::client::TlsStream as ClientTlsStream; +pub use tokio_rustls::server::TlsStream as ServerTlsStream; +pub use tokio_rustls::{Accept, Connect, TlsAcceptor, TlsConnector, TlsStream}; + +fn load_file(path: &String) -> BufReader { + let file = File::open(path).expect("io error"); + BufReader::new(file) +} +fn load_pem_file(path: &String) -> Result, std::io::Error> { + let iter: Vec = rustls_pemfile::read_all(&mut load_file(path)) + .filter_map(|f| { + if let Ok(f) = f { + Some(f) + } else { + let err = f.err().unwrap(); + warn!( + "Skipping PEM element in file \"{}\" due to error \"{}\"", + path, err + ); + None + } + }) + .collect(); + Ok(iter) +} + +pub fn load_ca_file_into_store(path: &String, cert_store: &mut RootCertStore) { + let ca_file = load_pem_file(path); + match ca_file { + Ok(cas) => { + cas.into_iter().for_each(|pem_item| { + // only loading certificates, handling a CA file + match pem_item { + Item::X509Certificate(content) => match cert_store.add(content) { + Ok(_) => {} + Err(err) => { + error!("{}", err) + } + }, + Item::Pkcs1Key(_) => {} + Item::Pkcs8Key(_) => {} + Item::Sec1Key(_) => {} + Item::Crl(_) => {} + Item::Csr(_) => {} + _ => {} + } + }); + } + Err(err) => { + error!( + "Failed to load configured ca file located at \"{}\", error: \"{}\"", + path, err + ); + } + } +} + +pub fn load_platform_certs_incl_env_into_store(ca_certs: &mut RootCertStore) { + // this includes handling of ENV vars SSL_CERT_FILE & SSL_CERT_DIR + let native_platform_certs = load_native_certs(); + match native_platform_certs { + Ok(certs) => { + for cert in certs { + ca_certs.add(cert).unwrap(); + } + } + Err(err) => { + error!( + "Failed to load native platform ca-certificates: \"{:?}\". Continuing without ...", + err + ); + } + } +} + +pub fn load_certs_key_file<'a>( + cert: &String, + key: &String, +) -> Option<(Vec>, PrivateKeyDer<'a>)> { + let certs_file = load_pem_file(cert) + .unwrap_or_else(|_| panic!("Failed to load configured cert file located at {}.", cert)); + let key_file = load_pem_file(key) + .unwrap_or_else(|_| panic!("Failed to load configured key file located at {}.", cert)); + + let mut certs: Vec> = vec![]; + certs_file.into_iter().for_each(|i| { + if let Item::X509Certificate(cert) = i { + certs.push(cert) + } + }); + + let private_key = match key_file.into_iter().next()? { + Item::Pkcs1Key(key) => Some(PrivateKeyDer::from(key)), + Item::Pkcs8Key(key) => Some(PrivateKeyDer::from(key)), + Item::Sec1Key(key) => Some(PrivateKeyDer::from(key)), + _ => None, + }; + + if certs.is_empty() || private_key.is_none() { + None + } else { + Some((certs, private_key?)) + } +} + +pub fn load_pem_file_ca(path: &String) -> Vec { + let mut reader = load_file(path); + let cas_file = rustls_pemfile::certs(&mut reader); + let ca = cas_file.into_iter().find_map(|pem_item| { + if let Ok(item) = pem_item { + Some(item) + } else { + None + } + }); + match ca { + None => Vec::new(), + Some(ca) => ca.to_vec(), + } +} + +pub fn load_pem_file_private_key(path: &String) -> Vec { + let key = rustls_pemfile::private_key(&mut load_file(path)); + if let Ok(Some(key)) = key { + return key.secret_der().to_vec(); + } + Vec::new() +} + +pub fn hash_certificate(cert: CertificateDer) -> Vec { + let hash = ring::digest::digest(&ring::digest::SHA256, cert.as_ref()); + hash.as_ref().to_vec() +} diff --git a/pingora/Cargo.toml b/pingora/Cargo.toml index 75baf1b1..f5b7d455 100644 --- a/pingora/Cargo.toml +++ b/pingora/Cargo.toml @@ -60,6 +60,12 @@ boringssl = [ "pingora-cache?/boringssl", "pingora-load-balancing?/boringssl", ] +rustls = [ + "pingora-core/rustls", + "pingora-proxy?/rustls", + "pingora-cache?/rustls", + "pingora-load-balancing?/rustls", +] proxy = ["pingora-proxy"] lb = ["pingora-load-balancing", "proxy"] cache = ["pingora-cache"] diff --git a/pingora/examples/server.rs b/pingora/examples/server.rs index a2a092e9..54d781d9 100644 --- a/pingora/examples/server.rs +++ b/pingora/examples/server.rs @@ -50,30 +50,35 @@ impl BackgroundService for ExampleBackgroundService { } } -use pingora::tls::pkey::{PKey, Private}; -use pingora::tls::x509::X509; -struct DynamicCert { - cert: X509, - key: PKey, -} +#[cfg(not(feature = "rustls"))] +mod boringssl_openssl { + use super::*; + use pingora::tls::pkey::{PKey, Private}; + use pingora::tls::x509::X509; + + pub(super) struct DynamicCert { + cert: X509, + key: PKey, + } -impl DynamicCert { - fn new(cert: &str, key: &str) -> Box { - let cert_bytes = std::fs::read(cert).unwrap(); - let cert = X509::from_pem(&cert_bytes).unwrap(); + impl DynamicCert { + pub(super) fn new(cert: &str, key: &str) -> Box { + let cert_bytes = std::fs::read(cert).unwrap(); + let cert = X509::from_pem(&cert_bytes).unwrap(); - let key_bytes = std::fs::read(key).unwrap(); - let key = PKey::private_key_from_pem(&key_bytes).unwrap(); - Box::new(DynamicCert { cert, key }) + let key_bytes = std::fs::read(key).unwrap(); + let key = PKey::private_key_from_pem(&key_bytes).unwrap(); + Box::new(DynamicCert { cert, key }) + } } -} -#[async_trait] -impl pingora::listeners::TlsAccept for DynamicCert { - async fn certificate_callback(&self, ssl: &mut pingora::tls::ssl::SslRef) { - use pingora::tls::ext; - ext::ssl_use_certificate(ssl, &self.cert).unwrap(); - ext::ssl_use_private_key(ssl, &self.key).unwrap(); + #[async_trait] + impl pingora::listeners::TlsAccept for DynamicCert { + async fn certificate_callback(&self, ssl: &mut pingora::tls::ssl::SslRef) { + use pingora::tls::ext; + ext::ssl_use_certificate(ssl, &self.cert).unwrap(); + ext::ssl_use_private_key(ssl, &self.key).unwrap(); + } } } @@ -132,12 +137,27 @@ pub fn main() { echo_service_http.add_tcp_with_settings("0.0.0.0:6145", options); echo_service_http.add_uds("/tmp/echo.sock", None); - let dynamic_cert = DynamicCert::new(&cert_path, &key_path); - let mut tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); - // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo - tls_settings - .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) - .unwrap(); + let mut tls_settings; + // NOTE: dynamic certificate callback is only supported with BoringSSL/OpenSSL + #[cfg(not(feature = "rustls"))] + { + use std::ops::DerefMut; + + let dynamic_cert = boringssl_openssl::DynamicCert::new(&cert_path, &key_path); + tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); + // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo + + tls_settings + .deref_mut() + .deref_mut() + .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) + .unwrap(); + } + #[cfg(feature = "rustls")] + { + tls_settings = + pingora::listeners::TlsSettings::intermediate(&cert_path, &key_path).unwrap(); + } tls_settings.enable_h2(); echo_service_http.add_tls_with_settings("0.0.0.0:6148", None, tls_settings);