From c2711034077963ec3c2345d40c9ff4c895211157 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Wed, 24 Jul 2024 18:11:54 +0200 Subject: [PATCH 01/10] Add Rustls compile time implementation --- .gitignore | 4 +- Cargo.toml | 1 + pingora-cache/Cargo.toml | 1 + pingora-core/Cargo.toml | 7 +- pingora-core/src/connectors/mod.rs | 95 ++----- .../{tls.rs => tls/boringssl_openssl/mod.rs} | 139 ++++----- pingora-core/src/connectors/tls/mod.rs | 154 ++++++++++ pingora-core/src/connectors/tls/rustls/mod.rs | 215 ++++++++++++++ pingora-core/src/lib.rs | 7 +- pingora-core/src/listeners/mod.rs | 30 +- pingora-core/src/listeners/tls.rs | 163 ----------- .../listeners/tls/boringssl_openssl/mod.rs | 123 ++++++++ pingora-core/src/listeners/tls/mod.rs | 127 +++++++++ pingora-core/src/listeners/tls/rustls/mod.rs | 93 ++++++ pingora-core/src/protocols/digest.rs | 2 +- pingora-core/src/protocols/l4/stream.rs | 6 +- pingora-core/src/protocols/mod.rs | 10 +- pingora-core/src/protocols/ssl/client.rs | 114 -------- pingora-core/src/protocols/ssl/digest.rs | 65 ----- pingora-core/src/protocols/ssl/mod.rs | 246 ---------------- .../protocols/tls/boringssl_openssl/client.rs | 47 ++++ .../protocols/tls/boringssl_openssl/mod.rs | 171 +++++++++++ .../{ssl => tls/boringssl_openssl}/server.rs | 168 +++++------ .../protocols/tls/boringssl_openssl/stream.rs | 166 +++++++++++ pingora-core/src/protocols/tls/mod.rs | 265 ++++++++++++++++++ .../src/protocols/tls/rustls/client.rs | 41 +++ pingora-core/src/protocols/tls/rustls/mod.rs | 216 ++++++++++++++ .../src/protocols/tls/rustls/server.rs | 95 +++++++ .../src/protocols/tls/rustls/stream.rs | 158 +++++++++++ pingora-core/src/protocols/tls/server.rs | 88 ++++++ pingora-core/src/upstreams/mod.rs | 2 +- pingora-core/src/upstreams/peer.rs | 19 +- pingora-core/src/utils/mod.rs | 129 +-------- .../src/utils/tls/boringssl_openssl/mod.rs | 99 +++++++ pingora-core/src/utils/tls/mod.rs | 108 +++++++ pingora-core/src/utils/tls/rustls/mod.rs | 66 +++++ pingora-error/src/lib.rs | 2 + pingora-load-balancing/Cargo.toml | 1 + pingora-proxy/Cargo.toml | 3 +- pingora-proxy/tests/test_basic.rs | 20 +- pingora-proxy/tests/utils/cert.rs | 58 ++-- pingora-proxy/tests/utils/conf/keys/README.md | 10 + .../tests/utils/conf/keys/server.crt | 13 - .../conf/keys/server_boringssl_openssl.crt} | 0 ...erver.csr => server_boringssl_openssl.csr} | 0 .../tests/utils/conf/keys/server_rustls.crt | 14 + .../tests/utils/conf/origin/conf/nginx.conf | 8 +- pingora-proxy/tests/utils/mock_origin.rs | 13 +- pingora-proxy/tests/utils/server_utils.rs | 4 +- pingora-rustls/Cargo.toml | 31 ++ pingora-rustls/LICENSE | 202 +++++++++++++ pingora-rustls/src/lib.rs | 158 +++++++++++ pingora/Cargo.toml | 6 + pingora/examples/server.rs | 68 +++-- 54 files changed, 2965 insertions(+), 1086 deletions(-) rename pingora-core/src/connectors/{tls.rs => tls/boringssl_openssl/mod.rs} (75%) create mode 100644 pingora-core/src/connectors/tls/mod.rs create mode 100644 pingora-core/src/connectors/tls/rustls/mod.rs delete mode 100644 pingora-core/src/listeners/tls.rs create mode 100644 pingora-core/src/listeners/tls/boringssl_openssl/mod.rs create mode 100644 pingora-core/src/listeners/tls/mod.rs create mode 100644 pingora-core/src/listeners/tls/rustls/mod.rs delete mode 100644 pingora-core/src/protocols/ssl/client.rs delete mode 100644 pingora-core/src/protocols/ssl/digest.rs delete mode 100644 pingora-core/src/protocols/ssl/mod.rs create mode 100644 pingora-core/src/protocols/tls/boringssl_openssl/client.rs create mode 100644 pingora-core/src/protocols/tls/boringssl_openssl/mod.rs rename pingora-core/src/protocols/{ssl => tls/boringssl_openssl}/server.rs (54%) create mode 100644 pingora-core/src/protocols/tls/boringssl_openssl/stream.rs create mode 100644 pingora-core/src/protocols/tls/mod.rs create mode 100644 pingora-core/src/protocols/tls/rustls/client.rs create mode 100644 pingora-core/src/protocols/tls/rustls/mod.rs create mode 100644 pingora-core/src/protocols/tls/rustls/server.rs create mode 100644 pingora-core/src/protocols/tls/rustls/stream.rs create mode 100644 pingora-core/src/protocols/tls/server.rs create mode 100644 pingora-core/src/utils/tls/boringssl_openssl/mod.rs create mode 100644 pingora-core/src/utils/tls/mod.rs create mode 100644 pingora-core/src/utils/tls/rustls/mod.rs delete mode 100644 pingora-proxy/tests/utils/conf/keys/server.crt rename pingora-proxy/tests/{keys/server.crt => utils/conf/keys/server_boringssl_openssl.crt} (100%) rename pingora-proxy/tests/utils/conf/keys/{server.csr => server_boringssl_openssl.csr} (100%) create mode 100644 pingora-proxy/tests/utils/conf/keys/server_rustls.crt create mode 100644 pingora-rustls/Cargo.toml create mode 100644 pingora-rustls/LICENSE create mode 100644 pingora-rustls/src/lib.rs diff --git a/.gitignore b/.gitignore index abc8cf51..f5636a63 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ dhat-heap.json .vscode .idea .cover -bleeper.user.toml \ No newline at end of file +bleeper.user.toml +pingora-proxy/tests/keys/server.crt +pingora-proxy/tests/utils/conf/keys/server.crt \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 2fdc570e..37b6228e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "pingora-lru", "pingora-openssl", "pingora-boringssl", + "pingora-rustls", "pingora-runtime", "pingora-ketama", "pingora-load-balancing", diff --git a/pingora-cache/Cargo.toml b/pingora-cache/Cargo.toml index 2dc3f2f7..253b311f 100644 --- a/pingora-cache/Cargo.toml +++ b/pingora-cache/Cargo.toml @@ -67,3 +67,4 @@ harness = false default = ["openssl"] openssl = ["pingora-core/openssl"] boringssl = ["pingora-core/boringssl"] +rustls = ["pingora-core/rustls"] \ No newline at end of file diff --git a/pingora-core/Cargo.toml b/pingora-core/Cargo.toml index c13823c3..75d14bdb 100644 --- a/pingora-core/Cargo.toml +++ b/pingora-core/Cargo.toml @@ -20,8 +20,9 @@ path = "src/lib.rs" [dependencies] pingora-runtime = { version = "0.3.0", path = "../pingora-runtime" } -pingora-openssl = { version = "0.3.0", path = "../pingora-openssl", optional = true } pingora-boringssl = { version = "0.3.0", path = "../pingora-boringssl", optional = true } +pingora-openssl = { version = "0.3.0", path = "../pingora-openssl", optional = true } +pingora-rustls = { version = "0.3.0", path = "../pingora-rustls", optional = true } pingora-pool = { version = "0.3.0", path = "../pingora-pool" } pingora-error = { version = "0.3.0", path = "../pingora-error" } pingora-timeout = { version = "0.3.0", path = "../pingora-timeout" } @@ -68,6 +69,7 @@ openssl-probe = "0.1" tokio-test = "0.4" zstd = "0" httpdate = "1" +x509-parser = { version = "0.16.0", optional = true } [dev-dependencies] matches = "0.1" @@ -81,4 +83,5 @@ jemallocator = "0.5" default = ["openssl"] openssl = ["pingora-openssl"] boringssl = ["pingora-boringssl"] -patched_http1 = [] +rustls = ["pingora-rustls", "dep:x509-parser"] +patched_http1 = [] \ No newline at end of file diff --git a/pingora-core/src/connectors/mod.rs b/pingora-core/src/connectors/mod.rs index 9bfd91b8..a0f0a980 100644 --- a/pingora-core/src/connectors/mod.rs +++ b/pingora-core/src/connectors/mod.rs @@ -14,27 +14,30 @@ //! Connecting to servers -pub mod http; -mod l4; -mod offload; -mod tls; - -use crate::protocols::Stream; -use crate::server::configuration::ServerConf; -use crate::tls::ssl::SslConnector; -use crate::upstreams::peer::{Peer, ALPN}; - -use l4::connect as l4_connect; -use log::{debug, error, warn}; -use offload::OffloadRuntime; -use parking_lot::RwLock; -use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use pingora_pool::{ConnectionMeta, ConnectionPool}; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; + +use futures::future::FutureExt; +use log::{debug, error, warn}; +use parking_lot::RwLock; +use tokio::io::AsyncReadExt; use tokio::sync::Mutex; +use offload::OffloadRuntime; +use pingora_error::{ErrorType::*, OrErr, Result}; +use pingora_pool::{ConnectionMeta, ConnectionPool}; + +use crate::connectors::tls::{Connector, do_connect}; +use crate::protocols::Stream; +use crate::server::configuration::ServerConf; +use crate::upstreams::peer::{ALPN, Peer}; + +pub mod http; +mod l4; +mod offload; +mod tls; + /// The options to configure a [TransportConnector] #[derive(Clone)] pub struct ConnectorOptions { @@ -123,7 +126,7 @@ impl ConnectorOptions { /// [TransportConnector] provides APIs to connect to servers via TCP or TLS with connection reuse pub struct TransportConnector { - tls_ctx: tls::Connector, + tls_ctx: Connector, connection_pool: Arc>>>, offload: Option, bind_to_v4: Vec, @@ -149,7 +152,7 @@ impl TransportConnector { .as_ref() .map_or_else(Vec::new, |o| o.bind_to_v6.clone()); TransportConnector { - tls_ctx: tls::Connector::new(options), + tls_ctx: Connector::new(options), connection_pool: Arc::new(ConnectionPool::new(pool_size)), offload: offload.map(|v| OffloadRuntime::new(v.0, v.1)), bind_to_v4, @@ -268,45 +271,8 @@ impl TransportConnector { } } -// Perform the actual L4 and tls connection steps while respecting the peer's -// connection timeout if there is one -async fn do_connect( - peer: &P, - bind_to: Option, - alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result { - // Create the future that does the connections, but don't evaluate it until - // we decide if we need a timeout or not - let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx); - - match peer.total_connection_timeout() { - Some(t) => match pingora_timeout::timeout(t, connect_future).await { - Ok(res) => res, - Err(_) => Error::e_explain( - ConnectTimedout, - format!("connecting to server {peer}, total-connection timeout {t:?}"), - ), - }, - None => connect_future.await, - } -} -// Perform the actual L4 and tls connection steps with no timeout -async fn do_connect_inner( - peer: &P, - bind_to: Option, - alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result { - let stream = l4_connect(peer, bind_to).await?; - if peer.tls() { - let tls_stream = tls::connect(stream, peer, alpn_override, tls_ctx).await?; - Ok(Box::new(tls_stream)) - } else { - Ok(Box::new(stream)) - } -} + struct PreferredHttpVersion { // TODO: shard to avoid the global lock @@ -337,9 +303,6 @@ impl PreferredHttpVersion { } } -use futures::future::FutureExt; -use tokio::io::AsyncReadExt; - /// Test whether a stream is already closed or not reusable (server sent unexpected data) fn test_reusable_stream(stream: &mut Stream) -> bool { let mut buf = [0; 1]; @@ -365,13 +328,14 @@ fn test_reusable_stream(stream: &mut Stream) -> bool { #[cfg(test)] mod tests { + use tokio::io::AsyncWriteExt; + use tokio::net::UnixListener; + use pingora_error::ErrorType; - use super::*; - use crate::tls::ssl::SslMethod; use crate::upstreams::peer::BasicPeer; - use tokio::io::AsyncWriteExt; - use tokio::net::UnixListener; + + use super::*; // 192.0.2.1 is effectively a black hole const BLACK_HOLE: &str = "192.0.2.1:79"; @@ -475,8 +439,8 @@ mod tests { /// This assumes that the connection will fail to on the peer and returns /// the decomposed error type and message async fn get_do_connect_failure_with_peer(peer: &BasicPeer) -> (ErrorType, String) { - let ssl_connector = SslConnector::builder(SslMethod::tls()).unwrap().build(); - let stream = do_connect(peer, None, None, &ssl_connector).await; + let connector = Connector::new(None); + let stream = do_connect(peer, None, None, &connector.ctx).await; match stream { Ok(_) => panic!("should throw an error"), Err(e) => ( @@ -509,6 +473,7 @@ mod tests { } #[tokio::test] + #[ignore] async fn test_do_connect_without_total_timeout() { let peer = BasicPeer::new(BLACK_HOLE); let (etype, context) = get_do_connect_failure_with_peer(&peer).await; diff --git a/pingora-core/src/connectors/tls.rs b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs similarity index 75% rename from pingora-core/src/connectors/tls.rs rename to pingora-core/src/connectors/tls/boringssl_openssl/mod.rs index 8a6bd635..48408a16 100644 --- a/pingora-core/src/connectors/tls.rs +++ b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs @@ -12,24 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -use log::debug; -use pingora_error::{Error, ErrorType::*, OrErr, Result}; +//! BoringSSL & OpenSSL TLS connector specific implementation + +use std::any::Any; use std::sync::{Arc, Once}; -use super::ConnectorOptions; -use crate::protocols::ssl::client::handshake; -use crate::protocols::ssl::SslStream; +use log::debug; + +use pingora_error::{Error, OrErr, Result}; +use pingora_error::ErrorType::{ConnectTimedout, InternalError}; + +use crate::connectors::ConnectorOptions; +use crate::listeners::ALPN; use crate::protocols::IO; +use crate::protocols::tls::boringssl_openssl::client::handshake; +use crate::protocols::tls::TlsStream; use crate::tls::ext::{ add_host, clear_error_stack, ssl_add_chain_cert, ssl_set_groups_list, ssl_set_renegotiate_mode_freely, ssl_set_verify_cert_store, ssl_use_certificate, ssl_use_private_key, ssl_use_second_key_share, }; +use crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion}; #[cfg(feature = "boringssl")] use crate::tls::ssl::SslCurve; -use crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion}; use crate::tls::x509::store::X509StoreBuilder; -use crate::upstreams::peer::{Peer, ALPN}; +use crate::upstreams::peer::Peer; +use crate::utils::tls::boringssl_openssl::{der_to_private_key, der_to_x509}; + +use super::{Connector, replace_leftmost_underscore, TlsConnectorContext}; const CIPHER_LIST: &str = "AES-128-GCM-SHA256\ :AES-256-GCM-SHA384\ @@ -60,6 +70,7 @@ const SIGALG_LIST: &str = "ECDSA_SECP256R1_SHA256\ :RSA_PKCS1_SHA512\ :RSA_PKCS1_SHA1\ :ECDSA_SECP521R1_SHA512"; + /** * Enabled curves for ECDHE (signature key exchange). * As of 4/10/2023, the only addition to boringssl's defaults is SECP521R1. @@ -83,13 +94,17 @@ fn init_ssl_cert_env_vars() { INIT_CA_ENV.call_once(openssl_probe::init_ssl_cert_env_vars); } -#[derive(Clone)] -pub struct Connector { - pub(crate) ctx: Arc, // Arc to support clone -} +pub(crate) struct TlsConnectorCtx(pub(crate) SslConnector); -impl Connector { - pub fn new(options: Option) -> Self { +impl TlsConnectorContext for TlsConnectorCtx { + fn as_any(&self) -> &dyn Any { + self + } + + fn build_connector(options: Option) -> Connector + where + Self: Sized + { let mut builder = SslConnector::builder(SslMethod::tls()).unwrap(); // TODO: make these conf // Set supported ciphers. @@ -142,49 +157,23 @@ impl Connector { } Connector { - ctx: Arc::new(builder.build()), - } - } -} - -/* - OpenSSL considers underscores in hostnames non-compliant. - We replace the underscore in the leftmost label as we must support these - hostnames for wildcard matches and we have not patched OpenSSL. - - https://github.com/openssl/openssl/issues/12566 - - > The labels must follow the rules for ARPANET host names. They must - > start with a letter, end with a letter or digit, and have as interior - > characters only letters, digits, and hyphen. There are also some - > restrictions on the length. Labels must be 63 characters or less. - - https://datatracker.ietf.org/doc/html/rfc1034#section-3.5 -*/ -fn replace_leftmost_underscore(sni: &str) -> Option { - // wildcard is only leftmost label - if let Some((leftmost, rest)) = sni.split_once('.') { - // if not a subdomain or leftmost does not contain underscore return - if !rest.contains('.') || !leftmost.contains('_') { - return None; + ctx: Arc::new(TlsConnectorCtx(builder.build())), } - // we have a subdomain, replace underscores - let leftmost = leftmost.replace('_', "-"); - return Some(format!("{leftmost}.{rest}")); } - None } -pub(crate) async fn connect( +pub(super) async fn connect( stream: T, peer: &P, alpn_override: Option, - tls_ctx: &SslConnector, -) -> Result> -where - T: IO, - P: Peer + Send + Sync, + tls_ctx: &Arc, +) -> Result> + where + T: IO, + P: Peer + Send + Sync { - let mut ssl_conf = tls_ctx.configure().unwrap(); + let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); + let mut ssl_conf = ctx.0.configure().unwrap(); ssl_set_renegotiate_mode_freely(&mut ssl_conf); @@ -192,8 +181,9 @@ where // TODO: store X509Store in the peer directly if let Some(ca_list) = peer.get_ca() { let mut store_builder = X509StoreBuilder::new().unwrap(); - for ca in &***ca_list { - store_builder.add_cert(ca.clone()).unwrap(); + for ca in &**ca_list { + let cert = der_to_x509(&**ca)?; + store_builder.add_cert(cert).unwrap(); } ssl_set_verify_cert_store(&mut ssl_conf, &store_builder.build()) .or_err(InternalError, "failed to load cert store")?; @@ -202,16 +192,19 @@ where // Set up client cert/key if let Some(key_pair) = peer.get_client_cert_key() { debug!("setting client cert and key"); - ssl_use_certificate(&mut ssl_conf, key_pair.leaf()) + let leaf = der_to_x509(&*key_pair.leaf())?; + ssl_use_certificate(&mut ssl_conf, &leaf) .or_err(InternalError, "invalid client cert")?; - ssl_use_private_key(&mut ssl_conf, key_pair.key()) + let key = der_to_private_key(&*key_pair.key())?; + ssl_use_private_key(&mut ssl_conf, &key) .or_err(InternalError, "invalid client key")?; let intermediates = key_pair.intermediates(); if !intermediates.is_empty() { debug!("adding intermediate certificates for mTLS chain"); for int in intermediates { - ssl_add_chain_cert(&mut ssl_conf, int) + let cert = der_to_x509(int)?; + ssl_add_chain_cert(&mut ssl_conf, &cert) .or_err(InternalError, "invalid intermediate client cert")?; } } @@ -282,42 +275,4 @@ where }, None => connect_future.await, } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_replace_leftmost_underscore() { - let none_cases = [ - "", - "some", - "some.com", - "1.1.1.1:5050", - "dog.dot.com", - "dog.d_t.com", - "dog.dot.c_m", - "d_g.com", - "_", - "dog.c_m", - ]; - - for case in none_cases { - assert!(replace_leftmost_underscore(case).is_none(), "{}", case); - } - - assert_eq!( - Some("bb-b.some.com".to_string()), - replace_leftmost_underscore("bb_b.some.com") - ); - assert_eq!( - Some("a-a-a.some.com".to_string()), - replace_leftmost_underscore("a_a_a.some.com") - ); - assert_eq!( - Some("-.some.com".to_string()), - replace_leftmost_underscore("_.some.com") - ); - } -} +} \ No newline at end of file diff --git a/pingora-core/src/connectors/tls/mod.rs b/pingora-core/src/connectors/tls/mod.rs new file mode 100644 index 00000000..8bce8a95 --- /dev/null +++ b/pingora-core/src/connectors/tls/mod.rs @@ -0,0 +1,154 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; +use std::net::SocketAddr; +use std::sync::Arc; + +use pingora_error::{Error, Result}; +use pingora_error::ErrorType::ConnectTimedout; + +use crate::connectors::l4::connect as l4_connect; +#[cfg(not(feature = "rustls"))] +use crate::connectors::tls::boringssl_openssl::connect as tls_connect; +#[cfg(not(feature = "rustls"))] +use crate::connectors::tls::boringssl_openssl::TlsConnectorCtx; +#[cfg(feature = "rustls")] +use crate::connectors::tls::rustls::connect as tls_connect; +#[cfg(feature = "rustls")] +use crate::connectors::tls::rustls::TlsConnectorCtx; +use crate::protocols::Stream; +use crate::upstreams::peer::{ALPN, Peer}; + +use super::ConnectorOptions; + +#[cfg(not(feature = "rustls"))] +pub(crate) mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; + +#[derive(Clone)] +pub struct Connector { + pub(crate) ctx: Arc, // Arc to support clone +} + +impl Connector { + pub fn new(options: Option) -> Self { + TlsConnectorCtx::build_connector(options) + } +} + +pub(crate) trait TlsConnectorContext { + fn as_any(&self) -> &dyn Any; + + fn build_connector(options: Option) -> Connector + where Self: Sized; +} + +pub(super) async fn do_connect( + peer: &P, + bind_to: Option, + alpn_override: Option, + tls_ctx: &Arc +) -> Result { + // Create the future that does the connections, but don't evaluate it until + // we decide if we need a timeout or not + let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx); + + match peer.total_connection_timeout() { + Some(t) => match pingora_timeout::timeout(t, connect_future).await { + Ok(res) => res, + Err(_) => Error::e_explain( + ConnectTimedout, + format!("connecting to server {peer}, total-connection timeout {t:?}"), + ), + }, + None => connect_future.await, + } +} + +async fn do_connect_inner( + peer: &P, + bind_to: Option, + alpn_override: Option, + tls_ctx: &Arc +) -> Result { + let stream = l4_connect(peer, bind_to).await?; + if peer.tls() { + let tls_stream = tls_connect(stream, peer, alpn_override, tls_ctx).await?; + Ok(Box::new(tls_stream)) + } else { + Ok(Box::new(stream)) + } +} + +/* + OpenSSL considers underscores in hostnames non-compliant. + We replace the underscore in the leftmost label as we must support these + hostnames for wildcard matches and we have not patched OpenSSL. + + https://github.com/openssl/openssl/issues/12566 + + > The labels must follow the rules for ARPANET host names. They must + > start with a letter, end with a letter or digit, and have as interior + > characters only letters, digits, and hyphen. There are also some + > restrictions on the length. Labels must be 63 characters or less. + - https://datatracker.ietf.org/doc/html/rfc1034#section-3.5 +*/ +pub(crate) fn replace_leftmost_underscore(sni: &str) -> Option { + // wildcard is only leftmost label + if let Some((leftmost, rest)) = sni.split_once('.') { + // if not a subdomain or leftmost does not contain underscore return + if !rest.contains('.') || !leftmost.contains('_') { + return None; + } + // we have a subdomain, replace underscores + let leftmost = leftmost.replace('_', "-"); + return Some(format!("{leftmost}.{rest}")); + } + None +} + +#[cfg(test)] +mod tests { + #[test] + fn test_replace_leftmost_underscore() { + let none_cases = [ + "", + "some", + "some.com", + "1.1.1.1:5050", + "dog.dot.com", + "dog.d_t.com", + "dog.dot.c_m", + "d_g.com", + "_", + "dog.c_m", + ]; + + for case in none_cases { + assert!(super::replace_leftmost_underscore(case).is_none(), "{}", case); + } + + assert_eq!( + Some("bb-b.some.com".to_string()), super::replace_leftmost_underscore("bb_b.some.com") + ); + assert_eq!( + Some("a-a-a.some.com".to_string()), super::replace_leftmost_underscore("a_a_a.some.com") + ); + assert_eq!( + Some("-.some.com".to_string()), super::replace_leftmost_underscore("_.some.com") + ); + } +} \ No newline at end of file diff --git a/pingora-core/src/connectors/tls/rustls/mod.rs b/pingora-core/src/connectors/tls/rustls/mod.rs new file mode 100644 index 00000000..88e04793 --- /dev/null +++ b/pingora-core/src/connectors/tls/rustls/mod.rs @@ -0,0 +1,215 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS connector specific implementation + +use std::any::Any; +use std::sync::Arc; + +use log::debug; + +use pingora_error::{Error, OrErr, Result}; +use pingora_error::ErrorType::{ConnectTimedout, InvalidCert}; +use pingora_rustls::{load_ca_file_into_store, load_certs_key_file, load_platform_certs_incl_env_into_store}; +use pingora_rustls::CertificateDer; +use pingora_rustls::ClientConfig; +use pingora_rustls::ClientConfig as RusTlsClientConfig; +use pingora_rustls::PrivateKeyDer; +use pingora_rustls::RootCertStore; +use pingora_rustls::TlsConnector as RusTlsConnector; +use pingora_rustls::version; + +use crate::connectors::ConnectorOptions; +use crate::listeners::ALPN; +use crate::protocols::IO; +use crate::protocols::tls::rustls::client::handshake; +use crate::protocols::tls::TlsStream; +use crate::upstreams::peer::Peer; + +use super::{Connector, replace_leftmost_underscore, TlsConnectorContext}; + +pub(crate) struct TlsConnectorCtx { + config: RusTlsClientConfig, + ca_certs: RootCertStore +} +impl TlsConnectorContext for TlsConnectorCtx { + fn as_any(&self) -> &dyn Any { + self + } + + fn build_connector(options: Option) -> Connector + where + Self: Sized + { + // NOTE: Rustls only supports TLS 1.2 & 1.3 + + // TODO: currently using Rustls defaults + // - support SSLKEYLOGFILE + // - set supported ciphers/algorithms/curves + // - add options for CRL/OCSP validation + + let (ca_certs, certs_key) = { + let mut ca_certs = RootCertStore::empty(); + let mut certs_key = None; + + if let Some(conf) = options.as_ref() { + if let Some(ca_file_path) = conf.ca_file.as_ref() { + load_ca_file_into_store(ca_file_path, &mut ca_certs); + } else { + load_platform_certs_incl_env_into_store(&mut ca_certs); + } + if let Some((cert, key)) = conf.cert_key_file.as_ref() { + certs_key = load_certs_key_file(&cert, &key); + } + // TODO: support SSLKEYLOGFILE + } else { + load_platform_certs_incl_env_into_store(&mut ca_certs); + } + + (ca_certs, certs_key) + }; + + // TODO: WebPkiServerVerifier for CRL/OCSP validation + let builder = + ClientConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_root_certificates(ca_certs.clone()); + + let config = match certs_key { + Some((certs, key)) => { + match builder.with_client_auth_cert(certs.clone(), key.clone_key()) { + Ok(config) => { config } + Err(err) => { + // TODO: is there a viable alternative to the panic? + // falling back to no client auth... does not seem to be reasonable. + panic!("{}", format!("Failed to configure client auth cert/key. Error: {}", err)); + } + } + } + None => { + builder.with_no_client_auth() + } + }; + + Connector { + ctx: Arc::new(TlsConnectorCtx { + config, + ca_certs + }), + } + } +} + +pub(super) async fn connect( + stream: T, + peer: &P, + alpn_override: Option, + tls_ctx: &Arc +) -> Result> +where + T: IO, + P: Peer + Send + Sync +{ + let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); + let mut config = ctx.config.clone(); + + // TODO: setup CA/verify cert store from peer + // looks like the fields are always None + // peer.get_ca() + + let key_pair = peer.get_client_cert_key(); + let updated_config: Option = match key_pair { + None => { None } + Some(key_arc) => { + debug!("setting client cert and key"); + + let mut cert_chain = vec![]; + debug!("adding leaf certificate to mTLS cert chain"); + cert_chain.push(key_arc.leaf().to_owned()); + + debug!("adding intermediate certificates to mTLS cert chain"); + key_arc.intermediates().to_owned().iter() + .map(|i| i.to_vec()) + .for_each(|i| cert_chain.push(i)); + + let certs: Vec = cert_chain.into_iter() + .map(|c| c.as_slice().to_owned().into()).collect(); + let private_key: PrivateKeyDer = key_arc.key().as_slice().to_owned().try_into().unwrap(); + + let builder = + ClientConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_root_certificates(ctx.ca_certs.clone()); + + let updated_config = builder.with_client_auth_cert(certs, private_key) + .explain_err(InvalidCert, |e| format!("Failed to use peer cert/key to update Rustls config: {:?}",e))?; + Some(updated_config) + } + }; + + if let Some(alpn) = alpn_override.as_ref().or(peer.get_alpn()) { + config.alpn_protocols = alpn.to_wire_protocols(); + } + + // TODO: curve setup from peer + // - second key share from peer, currently only used in boringssl with PQ features + + let tls_conn = if let Some(cfg) = updated_config { + RusTlsConnector::from(Arc::new(cfg)) + } else { + RusTlsConnector::from(Arc::new(config)) + }; + + // TODO: for consistent behaviour between TLS providers some additions are required + // - allowing to disable verification + // - the validation/replace logic would need adjustments to match the boringssl/openssl behaviour + // implementing a custom certificate_verifier could be used to achieve matching behaviour + //let d_conf = config.dangerous(); + //d_conf.set_certificate_verifier(...); + + let mut domain = peer.sni().to_string(); + if peer.sni().is_empty() { + // use ip in case SNI is not present + // TODO: disable validation + domain = peer.address().as_inet().unwrap().ip().to_string() + } + if peer.verify_cert() { + if peer.verify_hostname() { + // TODO: streamline logic with replacing first underscore within TLS implementations + if let Some(sni_s) = replace_leftmost_underscore(peer.sni()) { + domain = sni_s; + } + if let Some(alt_cn) = peer.alternative_cn() { + if !alt_cn.is_empty() { + domain = alt_cn.to_string(); + // TODO: streamline logic with replacing first underscore within TLS implementations + if let Some(alt_cn_s) = replace_leftmost_underscore(alt_cn) { + domain = alt_cn_s; + } + } + } + } + } + + let connect_future = handshake(&tls_conn, &domain, stream); + + match peer.connection_timeout() { + Some(t) => match pingora_timeout::timeout(t, connect_future).await { + Ok(res) => res, + Err(_) => Error::e_explain( + ConnectTimedout, + format!("connecting to server {}, timeout {:?}", peer, t), + ), + }, + None => connect_future.await, + } +} \ No newline at end of file diff --git a/pingora-core/src/lib.rs b/pingora-core/src/lib.rs index 3435fe58..35c81393 100644 --- a/pingora-core/src/lib.rs +++ b/pingora-core/src/lib.rs @@ -54,12 +54,15 @@ pub use pingora_error::{ErrorType::*, *}; // If both openssl and boringssl are enabled, prefer boringssl. // This is to make sure that boringssl can override the default openssl feature // when this crate is used indirectly by other crates. -#[cfg(feature = "boringssl")] +#[cfg(all(not(feature = "rustls"), feature = "boringssl"))] pub use pingora_boringssl as tls; -#[cfg(all(not(feature = "boringssl"), feature = "openssl"))] +#[cfg(all(not(feature = "rustls"), not(feature = "boringssl"), feature = "openssl"))] pub use pingora_openssl as tls; +#[cfg(all(not(feature = "boringssl"), not(feature = "openssl"), feature = "rustls"))] +pub use pingora_rustls as tls; + pub mod prelude { pub use crate::server::configuration::Opt; pub use crate::server::Server; diff --git a/pingora-core/src/listeners/mod.rs b/pingora-core/src/listeners/mod.rs index cc8edb87..4637229a 100644 --- a/pingora-core/src/listeners/mod.rs +++ b/pingora-core/src/listeners/mod.rs @@ -14,21 +14,24 @@ //! The listening endpoints (TCP and TLS) and their configurations. -mod l4; -mod tls; - -use crate::protocols::Stream; -use crate::server::ListenFds; - -use pingora_error::Result; use std::{fs::Permissions, sync::Arc}; use l4::{ListenerEndpoint, Stream as L4Stream}; +pub use l4::{ServerAddress, TcpSocketOptions}; +use pingora_error::Result; +pub use tls::{ALPN, TlsSettings}; use tls::Acceptor; -pub use crate::protocols::ssl::server::TlsAccept; -pub use l4::{ServerAddress, TcpSocketOptions}; -pub use tls::{TlsSettings, ALPN}; +use crate::protocols::{IO, Stream}; +#[cfg(not(feature = "rustls"))] +use crate::protocols::tls::TlsStream as TlsStreamProvider; +#[cfg(feature = "rustls")] +use crate::protocols::tls::TlsStream as TlsStreamProvider; +pub use crate::protocols::tls::server::TlsAccept; +use crate::server::ListenFds; + +mod l4; +pub mod tls; struct TransportStackBuilder { l4: ServerAddress, @@ -82,7 +85,7 @@ pub(crate) struct UninitializedStream { impl UninitializedStream { pub async fn handshake(self) -> Result { if let Some(tls) = self.tls { - let tls_stream = tls.tls_handshake(self.l4).await?; + let tls_stream : TlsStreamProvider> = tls.handshake(Box::new(self.l4)).await?; Ok(Box::new(tls_stream)) } else { Ok(Box::new(self.l4)) @@ -182,10 +185,11 @@ impl Listeners { #[cfg(test)] mod test { - use super::*; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; - use tokio::time::{sleep, Duration}; + use tokio::time::{Duration, sleep}; + + use super::*; #[tokio::test] async fn test_listen_tcp() { diff --git a/pingora-core/src/listeners/tls.rs b/pingora-core/src/listeners/tls.rs deleted file mode 100644 index 1dd63d46..00000000 --- a/pingora-core/src/listeners/tls.rs +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use log::debug; -use pingora_error::{ErrorType, OrErr, Result}; -use std::ops::{Deref, DerefMut}; - -use crate::protocols::ssl::{ - server::{handshake, handshake_with_callback, TlsAcceptCallbacks}, - SslStream, -}; -use crate::protocols::IO; -use crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; - -pub use crate::protocols::ssl::ALPN; - -pub const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); - -pub(crate) struct Acceptor { - ssl_acceptor: SslAcceptor, - callbacks: Option, -} - -/// The TLS settings of a listening endpoint -pub struct TlsSettings { - accept_builder: SslAcceptorBuilder, - callbacks: Option, -} - -impl From for TlsSettings { - fn from(settings: SslAcceptorBuilder) -> Self { - TlsSettings { - accept_builder: settings, - callbacks: None, - } - } -} - -impl Deref for TlsSettings { - type Target = SslAcceptorBuilder; - - fn deref(&self) -> &Self::Target { - &self.accept_builder - } -} - -impl DerefMut for TlsSettings { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.accept_builder - } -} - -impl TlsSettings { - /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29) - /// server side TLS settings. Users can adjust the TLS settings after this object is created. - /// Return error if the provided certificate and private key are invalid or not found. - pub fn intermediate(cert_path: &str, key_path: &str) -> Result { - let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( - TLS_CONF_ERR, - "fail to create mozilla_intermediate_v5 Acceptor", - )?; - accept_builder - .set_private_key_file(key_path, SslFiletype::PEM) - .or_err_with(TLS_CONF_ERR, || format!("fail to read key file {key_path}"))?; - accept_builder - .set_certificate_chain_file(cert_path) - .or_err_with(TLS_CONF_ERR, || { - format!("fail to read cert file {cert_path}") - })?; - Ok(TlsSettings { - accept_builder, - callbacks: None, - }) - } - - /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks] - /// is needed to provide the certificate during the TLS handshake. - pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result { - let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( - TLS_CONF_ERR, - "fail to create mozilla_intermediate_v5 Acceptor", - )?; - Ok(TlsSettings { - accept_builder, - callbacks: Some(callbacks), - }) - } - - /// Enable HTTP/2 support for this endpoint, which is default off. - /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed - pub fn enable_h2(&mut self) { - self.set_alpn(ALPN::H2H1); - } - - /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details - pub fn set_alpn(&mut self, alpn: ALPN) { - match alpn { - ALPN::H2H1 => self - .accept_builder - .set_alpn_select_callback(alpn::prefer_h2), - ALPN::H1 => self.accept_builder.set_alpn_select_callback(alpn::h1_only), - ALPN::H2 => self.accept_builder.set_alpn_select_callback(alpn::h2_only), - } - } - - pub(crate) fn build(self) -> Acceptor { - Acceptor { - ssl_acceptor: self.accept_builder.build(), - callbacks: self.callbacks, - } - } -} - -impl Acceptor { - pub async fn tls_handshake(&self, stream: S) -> Result> { - debug!("new ssl session"); - // TODO: be able to offload this handshake in a thread pool - if let Some(cb) = self.callbacks.as_ref() { - handshake_with_callback(&self.ssl_acceptor, stream, cb).await - } else { - handshake(&self.ssl_acceptor, stream).await - } - } -} - -mod alpn { - use super::*; - use crate::tls::ssl::{select_next_proto, AlpnError, SslRef}; - - // A standard implementation provided by the SSL lib is used below - - pub fn prefer_h2<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { - match select_next_proto(ALPN::H2H1.to_wire_preference(), alpn_in) { - Some(p) => Ok(p), - _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1 - } - } - - pub fn h1_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { - match select_next_proto(ALPN::H1.to_wire_preference(), alpn_in) { - Some(p) => Ok(p), - _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1 - } - } - - pub fn h2_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { - match select_next_proto(ALPN::H2.to_wire_preference(), alpn_in) { - Some(p) => Ok(p), - _ => Err(AlpnError::ALERT_FATAL), // cannot agree - } - } -} diff --git a/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs new file mode 100644 index 00000000..054b7505 --- /dev/null +++ b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs @@ -0,0 +1,123 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL listener specific implementation + +use core::any::Any; +use async_trait::async_trait; +use pingora_error::{ErrorType, OrErr, Result}; +use crate::listeners::{ALPN, TlsSettings}; +use crate::listeners::tls::{NativeBuilder, TlsAcceptor, TlsAcceptorBuilder}; +use crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; +const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); + +struct TlsAcc(SslAcceptor); +pub(super) struct TlsAcceptorBuil(SslAcceptorBuilder); + +#[async_trait] +impl TlsAcceptor for TlsAcc { + fn get_acceptor(&self) -> &dyn Any { + &self.0 + } +} + +impl TlsAcceptorBuilder for TlsAcceptorBuil { + fn build(self: Box) -> Box { + let builder = (*self).0; + Box::new(TlsAcc(SslAcceptorBuilder::build(builder))) + } + + fn set_alpn(&mut self, alpn: ALPN) { + match alpn { + ALPN::H2H1 => self.0.set_alpn_select_callback(alpn::prefer_h2), + ALPN::H1 => self.0.set_alpn_select_callback(alpn::h1_only), + ALPN::H2 => self.0.set_alpn_select_callback(alpn::h2_only), + } + } + + fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where Self: Sized { + let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( + TLS_CONF_ERR, + "fail to create mozilla_intermediate_v5 Acceptor", + )?; + accept_builder + .set_private_key_file(key_path, SslFiletype::PEM) + .or_err_with(TLS_CONF_ERR, || format!("fail to read key file {key_path}"))?; + accept_builder + .set_certificate_chain_file(cert_path) + .or_err_with(TLS_CONF_ERR, || { + format!("fail to read cert file {cert_path}") + })?; + Ok(TlsAcceptorBuil(accept_builder)) + } + + fn acceptor_with_callbacks() -> Result + where Self: Sized { + let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( + TLS_CONF_ERR, + "fail to create mozilla_intermediate_v5 Acceptor", + )?; + Ok(TlsAcceptorBuil(accept_builder)) + } + + fn as_any(&mut self) -> &mut dyn Any { + self as &mut dyn Any + } +} + +impl NativeBuilder for Box { + type Builder = SslAcceptorBuilder; + + fn native(&mut self) -> &mut Self::Builder { + self.as_any().downcast_mut::().unwrap() + } +} + +impl From for TlsSettings { + fn from(settings: SslAcceptorBuilder) -> Self { + TlsSettings { + accept_builder: Box::new(TlsAcceptorBuil(settings)), + callbacks: None, + } + } +} + +mod alpn { + use crate::protocols::ALPN; + use crate::tls::ssl::{AlpnError, select_next_proto, SslRef}; + + // A standard implementation provided by the SSL lib is used below + + pub fn prefer_h2<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { + match select_next_proto(ALPN::H2H1.to_wire_preference(), alpn_in) { + Some(p) => Ok(p), + _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1 + } + } + + pub fn h1_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { + match select_next_proto(ALPN::H1.to_wire_preference(), alpn_in) { + Some(p) => Ok(p), + _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1 + } + } + + pub fn h2_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> { + match select_next_proto(ALPN::H2.to_wire_preference(), alpn_in) { + Some(p) => Ok(p), + _ => Err(AlpnError::ALERT_FATAL), // cannot agree + } + } +} \ No newline at end of file diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs new file mode 100644 index 00000000..08961caa --- /dev/null +++ b/pingora-core/src/listeners/tls/mod.rs @@ -0,0 +1,127 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; + +use async_trait::async_trait; +use log::debug; + +use pingora_error::Result; +#[cfg(not(feature = "rustls"))] +use crate::listeners::tls::boringssl_openssl::TlsAcceptorBuil; +#[cfg(feature = "rustls")] +use crate::listeners::tls::rustls::TlsAcceptorBuil; +use crate::protocols::IO; +pub use crate::protocols::tls::ALPN; +#[cfg(not(feature = "rustls"))] +use crate::protocols::tls::boringssl_openssl::server::{handshake, handshake_with_callback}; +#[cfg(feature = "rustls")] +use crate::protocols::tls::rustls::server::{handshake, handshake_with_callback}; +use crate::protocols::tls::server::TlsAcceptCallbacks; +use crate::protocols::tls::TlsStream; + +#[cfg(not(feature = "rustls"))] +pub mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; + +pub(crate) struct Acceptor { + ssl_acceptor: Box, + callbacks: Option, +} + +#[async_trait] +pub trait TlsAcceptor { + fn get_acceptor(&self) -> &dyn Any; +} + +/// The TLS settings of a listening endpoint +pub struct TlsSettings { + accept_builder: Box, + callbacks: Option, +} + +pub trait TlsAcceptorBuilder: Any { + fn build(self: Box) -> Box; + fn set_alpn(&mut self, alpn: ALPN); + fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where Self: Sized; + fn acceptor_with_callbacks() -> Result + where Self: Sized; + fn as_any(&mut self) -> &mut dyn Any; +} + +pub trait NativeBuilder { + type Builder; + fn native(&mut self) -> &mut Self::Builder; +} + +impl TlsSettings { + /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29) + /// server side TLS settings. Users can adjust the TLS settings after this object is created. + /// Return error if the provided certificate and private key are invalid or not found. + pub fn intermediate(cert_path: &str, key_path: &str) -> Result { + Ok(TlsSettings { + accept_builder: Box::new(TlsAcceptorBuil::acceptor_intermediate(cert_path, key_path)?), + callbacks: None, + }) + } + + /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks] + /// is needed to provide the certificate during the TLS handshake. + pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result { + Ok(TlsSettings { + accept_builder: Box::new(TlsAcceptorBuil::acceptor_with_callbacks()?), + callbacks: Some(callbacks), + }) + } + + /// Enable HTTP/2 support for this endpoint, which is default off. + /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed + pub fn enable_h2(&mut self) { + self.set_alpn(ALPN::H2H1); + } + + /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details + pub fn set_alpn(&mut self, alpn: ALPN) { + self.accept_builder.set_alpn(alpn); + } + + pub(crate) fn build(self) -> Acceptor { + Acceptor { + ssl_acceptor: self.accept_builder.build(), + callbacks: self.callbacks, + } + } + + pub fn get_builder(&mut self) -> &mut Box { + &mut (self.accept_builder) + } +} + +impl Acceptor { + pub async fn handshake(&self, stream: Box) -> Result>> { + debug!("new tls session"); + // TODO: be able to offload this handshake in a thread pool + if let Some(cb) = self.callbacks.as_ref() { + handshake_with_callback(self, stream, cb).await + } else { + handshake(self, stream).await + } + } + + pub(crate) fn inner(&self) -> &dyn Any { + self.ssl_acceptor.get_acceptor() + } +} diff --git a/pingora-core/src/listeners/tls/rustls/mod.rs b/pingora-core/src/listeners/tls/rustls/mod.rs new file mode 100644 index 00000000..87103df8 --- /dev/null +++ b/pingora-core/src/listeners/tls/rustls/mod.rs @@ -0,0 +1,93 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS listener specific implementation + +use std::any::Any; +use std::sync::Arc; + +use async_trait::async_trait; + +use pingora_error::{Error, ErrorSource, ImmutStr, OrErr, Result}; +use pingora_error::ErrorType::InternalError; +use pingora_rustls::{TlsAcceptor as RusTlsAcceptor, version}; +use pingora_rustls::load_certs_key_file; +use pingora_rustls::ServerConfig; + +use crate::listeners::ALPN; +use crate::listeners::tls::{TlsAcceptor, TlsAcceptorBuilder}; + +pub(super) struct TlsAcceptorBuil { + alpn_protocols: Option>>, + cert_path: String, + key_path: String +} + +struct TlsAcc { + acceptor: RusTlsAcceptor +} + +#[async_trait] +impl TlsAcceptor for TlsAcc { + fn get_acceptor(&self) -> &dyn Any { + &self.acceptor + } +} + +impl TlsAcceptorBuilder for TlsAcceptorBuil { + fn build(self: Box) -> Box { + let (certs, key) = load_certs_key_file(&self.cert_path, &self.key_path) + .expect(format!("Failed to load provided certificates \"{}\" or key \"{}\".", self.cert_path, self.key_path).as_str()); + + let mut config = ServerConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_no_client_auth() + .with_single_cert(certs, key) + .explain_err(InternalError, |e| format!("Failed to create server listener config: {}", e)) + .unwrap(); + + if let Some(alpn_protocols) = self.alpn_protocols { + config.alpn_protocols = alpn_protocols; + } + + Box::new(TlsAcc { + acceptor: RusTlsAcceptor::from(Arc::new(config)) + }) + } + fn set_alpn(&mut self, alpn: ALPN) { + self.alpn_protocols = Some(alpn.to_wire_protocols()); + } + + fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + where Self: Sized { + Ok(TlsAcceptorBuil { + alpn_protocols: None, + cert_path: cert_path.to_string(), + key_path: key_path.to_string() + }) + } + + fn acceptor_with_callbacks() -> Result + where Self: Sized { + // TODO: verify if/how callback in handshake can be done using Rustls + Err(Error::create(InternalError, + ErrorSource::Internal, + Some(ImmutStr::from("Certificate callbacks are not supported with feature \"rustls\".")), + None + )) + } + + fn as_any(&mut self) -> &mut dyn Any { + self as &mut dyn Any + } +} \ No newline at end of file diff --git a/pingora-core/src/protocols/digest.rs b/pingora-core/src/protocols/digest.rs index e080a89a..56ec3725 100644 --- a/pingora-core/src/protocols/digest.rs +++ b/pingora-core/src/protocols/digest.rs @@ -22,7 +22,7 @@ use once_cell::sync::OnceCell; use super::l4::ext::{get_recv_buf, get_tcp_info, TCP_INFO}; use super::l4::socket::SocketAddr; use super::raw_connect::ProxyDigest; -use super::ssl::digest::SslDigest; +use super::tls::SslDigest; /// The information can be extracted from a connection #[derive(Clone, Debug, Default)] diff --git a/pingora-core/src/protocols/l4/stream.rs b/pingora-core/src/protocols/l4/stream.rs index ee91a8aa..fa0a1685 100644 --- a/pingora-core/src/protocols/l4/stream.rs +++ b/pingora-core/src/protocols/l4/stream.rs @@ -28,10 +28,7 @@ use tokio::net::{TcpStream, UnixStream}; use crate::protocols::l4::ext::{set_tcp_keepalive, TcpKeepalive}; use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{ - GetProxyDigest, GetSocketDigest, GetTimingDigest, Shutdown, SocketDigest, Ssl, TimingDigest, - UniqueID, -}; +use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, IO, Shutdown, SocketDigest, Ssl, TimingDigest, UniqueID}; use crate::upstreams::peer::Tracer; #[derive(Debug)] @@ -208,6 +205,7 @@ impl UniqueID for Stream { } impl Ssl for Stream {} +impl Ssl for Box {} #[async_trait] impl Shutdown for Stream { diff --git a/pingora-core/src/protocols/mod.rs b/pingora-core/src/protocols/mod.rs index 32695e1c..d212bd0f 100644 --- a/pingora-core/src/protocols/mod.rs +++ b/pingora-core/src/protocols/mod.rs @@ -18,14 +18,14 @@ mod digest; pub mod http; pub mod l4; pub mod raw_connect; -pub mod ssl; +pub mod tls; pub use digest::{ Digest, GetProxyDigest, GetSocketDigest, GetTimingDigest, ProtoDigest, SocketDigest, TimingDigest, }; pub use l4::ext::TcpKeepalive; -pub use ssl::ALPN; +pub use tls::ALPN; use async_trait::async_trait; use std::fmt::Debug; @@ -48,19 +48,19 @@ pub trait UniqueID { /// Interface to get TLS info pub trait Ssl { /// Return the TLS info if the connection is over TLS + #[cfg(not(feature = "rustls"))] fn get_ssl(&self) -> Option<&crate::tls::ssl::SslRef> { None } /// Return the [`ssl::SslDigest`] for logging - fn get_ssl_digest(&self) -> Option> { + fn get_ssl_digest(&self) -> Option> { None } /// Return selected ALPN if any fn selected_alpn_proto(&self) -> Option { - let ssl = self.get_ssl()?; - ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + None } } diff --git a/pingora-core/src/protocols/ssl/client.rs b/pingora-core/src/protocols/ssl/client.rs deleted file mode 100644 index 9edf29b2..00000000 --- a/pingora-core/src/protocols/ssl/client.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! TLS client specific implementation - -use super::SslStream; -use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{ - GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest, TimingDigest, IO, -}; -use crate::tls::{ssl, ssl::ConnectConfiguration, ssl_sys::X509_V_ERR_INVALID_CALL}; - -use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use std::sync::Arc; -use std::time::Duration; - -/// Perform the TLS handshake for the given connection with the given configuration -pub async fn handshake( - conn_config: ConnectConfiguration, - domain: &str, - io: S, -) -> Result> { - let ssl = conn_config - .into_ssl(domain) - .explain_err(TLSHandshakeFailure, |e| format!("ssl config error: {e}"))?; - let mut stream = SslStream::new(ssl, io) - .explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}"))?; - let handshake_result = stream.connect().await; - match handshake_result { - Ok(()) => Ok(stream), - Err(e) => { - let context = format!("TLS connect() failed: {e}, SNI: {domain}"); - match e.code() { - ssl::ErrorCode::SSL => { - // Unify the return type of `verify_result` for openssl - #[cfg(not(feature = "boringssl"))] - fn verify_result(stream: SslStream) -> Result<(), i32> { - match stream.ssl().verify_result().as_raw() { - crate::tls::ssl_sys::X509_V_OK => Ok(()), - e => Err(e), - } - } - - // Unify the return type of `verify_result` for boringssl - #[cfg(feature = "boringssl")] - fn verify_result(stream: SslStream) -> Result<(), i32> { - stream.ssl().verify_result().map_err(|e| e.as_raw()) - } - - match verify_result(stream) { - Ok(()) => Error::e_explain(TLSHandshakeFailure, context), - // X509_V_ERR_INVALID_CALL in case verify result was never set - Err(X509_V_ERR_INVALID_CALL) => { - Error::e_explain(TLSHandshakeFailure, context) - } - _ => Error::e_explain(InvalidCert, context), - } - } - /* likely network error, but still mark as TLS error */ - _ => Error::e_explain(TLSHandshakeFailure, context), - } - } - } -} - -impl GetTimingDigest for SslStream -where - S: GetTimingDigest, -{ - fn get_timing_digest(&self) -> Vec> { - let mut ts_vec = self.get_ref().get_timing_digest(); - ts_vec.push(Some(self.timing.clone())); - ts_vec - } - fn get_read_pending_time(&self) -> Duration { - self.get_ref().get_read_pending_time() - } - - fn get_write_pending_time(&self) -> Duration { - self.get_ref().get_write_pending_time() - } -} - -impl GetProxyDigest for SslStream -where - S: GetProxyDigest, -{ - fn get_proxy_digest(&self) -> Option> { - self.get_ref().get_proxy_digest() - } -} - -impl GetSocketDigest for SslStream -where - S: GetSocketDigest, -{ - fn get_socket_digest(&self) -> Option> { - self.get_ref().get_socket_digest() - } - fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - self.get_mut().set_socket_digest(socket_digest) - } -} diff --git a/pingora-core/src/protocols/ssl/digest.rs b/pingora-core/src/protocols/ssl/digest.rs deleted file mode 100644 index 3cdb7aa0..00000000 --- a/pingora-core/src/protocols/ssl/digest.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! TLS information from the TLS connection - -use crate::tls::{hash::MessageDigest, ssl::SslRef}; -use crate::utils; - -/// The TLS connection information -#[derive(Clone, Debug)] -pub struct SslDigest { - /// The cipher used - pub cipher: &'static str, - /// The TLS version of this connection - pub version: &'static str, - /// The organization of the peer's certificate - pub organization: Option, - /// The serial number of the peer's certificate - pub serial_number: Option, - /// The digest of the peer's certificate - pub cert_digest: Vec, -} - -impl SslDigest { - pub fn from_ssl(ssl: &SslRef) -> Self { - let cipher = match ssl.current_cipher() { - Some(c) => c.name(), - None => "", - }; - - let (cert_digest, org, sn) = match ssl.peer_certificate() { - Some(cert) => { - let cert_digest = match cert.digest(MessageDigest::sha256()) { - Ok(c) => c.as_ref().to_vec(), - Err(_) => Vec::new(), - }; - ( - cert_digest, - utils::get_organization(&cert), - utils::get_serial(&cert).ok(), - ) - } - None => (Vec::new(), None, None), - }; - - SslDigest { - cipher, - version: ssl.version_str(), - organization: org, - serial_number: sn, - cert_digest, - } - } -} diff --git a/pingora-core/src/protocols/ssl/mod.rs b/pingora-core/src/protocols/ssl/mod.rs deleted file mode 100644 index f1ce8b95..00000000 --- a/pingora-core/src/protocols/ssl/mod.rs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2024 Cloudflare, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The TLS layer implementations - -pub mod client; -pub mod digest; -pub mod server; - -use crate::protocols::digest::TimingDigest; -use crate::protocols::{Ssl, UniqueID}; -use crate::tls::{self, ssl, tokio_ssl::SslStream as InnerSsl}; -use log::warn; -use pingora_error::{ErrorType::*, OrErr, Result}; -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::SystemTime; -use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; - -pub use digest::SslDigest; - -/// The TLS connection -#[derive(Debug)] -pub struct SslStream { - ssl: InnerSsl, - digest: Option>, - timing: TimingDigest, -} - -impl SslStream -where - T: AsyncRead + AsyncWrite + std::marker::Unpin, -{ - /// Create a new TLS connection from the given `stream` - /// - /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS - /// handshake after. - pub fn new(ssl: ssl::Ssl, stream: T) -> Result { - let ssl = InnerSsl::new(ssl, stream) - .explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}"))?; - - Ok(SslStream { - ssl, - digest: None, - timing: Default::default(), - }) - } - - /// Connect to the remote TLS server as a client - pub async fn connect(&mut self) -> Result<(), ssl::Error> { - Self::clear_error(); - Pin::new(&mut self.ssl).connect().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = Some(Arc::new(SslDigest::from_ssl(self.ssl()))); - Ok(()) - } - - /// Finish the TLS handshake from client as a server - pub async fn accept(&mut self) -> Result<(), ssl::Error> { - Self::clear_error(); - Pin::new(&mut self.ssl).accept().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = Some(Arc::new(SslDigest::from_ssl(self.ssl()))); - Ok(()) - } - - #[inline] - fn clear_error() { - let errs = tls::error::ErrorStack::get(); - if !errs.errors().is_empty() { - warn!("Clearing dirty TLS error stack: {}", errs); - } - } -} - -impl SslStream { - pub fn ssl_digest(&self) -> Option> { - self.digest.clone() - } -} - -use std::ops::{Deref, DerefMut}; - -impl Deref for SslStream { - type Target = InnerSsl; - - fn deref(&self) -> &Self::Target { - &self.ssl - } -} - -impl DerefMut for SslStream { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.ssl - } -} - -impl AsyncRead for SslStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_read(cx, buf) - } -} - -impl AsyncWrite for SslStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context, - buf: &[u8], - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_write(cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_shutdown(cx) - } - - fn poll_write_vectored( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Self::clear_error(); - Pin::new(&mut self.ssl).poll_write_vectored(cx, bufs) - } - - fn is_write_vectored(&self) -> bool { - true - } -} - -impl UniqueID for SslStream -where - T: UniqueID, -{ - fn id(&self) -> i32 { - self.ssl.get_ref().id() - } -} - -impl Ssl for SslStream { - fn get_ssl(&self) -> Option<&ssl::SslRef> { - Some(self.ssl()) - } - - fn get_ssl_digest(&self) -> Option> { - self.ssl_digest() - } -} - -/// The protocol for Application-Layer Protocol Negotiation -#[derive(Hash, Clone, Debug)] -pub enum ALPN { - /// Prefer HTTP/1.1 only - H1, - /// Prefer HTTP/2 only - H2, - /// Prefer HTTP/2 over HTTP/1.1 - H2H1, -} - -impl std::fmt::Display for ALPN { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ALPN::H1 => write!(f, "H1"), - ALPN::H2 => write!(f, "H2"), - ALPN::H2H1 => write!(f, "H2H1"), - } - } -} - -impl ALPN { - /// Create a new ALPN according to the `max` and `min` version constraints - pub fn new(max: u8, min: u8) -> Self { - if max == 1 { - ALPN::H1 - } else if min == 2 { - ALPN::H2 - } else { - ALPN::H2H1 - } - } - - /// Return the max http version this [`ALPN`] allows - pub fn get_max_http_version(&self) -> u8 { - match self { - ALPN::H1 => 1, - _ => 2, - } - } - - /// Return the min http version this [`ALPN`] allows - pub fn get_min_http_version(&self) -> u8 { - match self { - ALPN::H2 => 2, - _ => 1, - } - } - - pub(crate) fn to_wire_preference(&self) -> &[u8] { - // https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_alpn_select_cb.html - // "vector of nonempty, 8-bit length-prefixed, byte strings" - match self { - Self::H1 => b"\x08http/1.1", - Self::H2 => b"\x02h2", - Self::H2H1 => b"\x02h2\x08http/1.1", - } - } - - pub(crate) fn from_wire_selected(raw: &[u8]) -> Option { - match raw { - b"http/1.1" => Some(Self::H1), - b"h2" => Some(Self::H2), - _ => None, - } - } -} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs new file mode 100644 index 00000000..49d35231 --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs @@ -0,0 +1,47 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS client specific implementation + +use pingora_error::{Error, ErrorType::*, OrErr, Result}; + +use crate::protocols::IO; +use crate::protocols::tls::boringssl_openssl::TlsStream; +use crate::tls::ssl::ConnectConfiguration; + +/// Perform the TLS handshake for the given connection with the given configuration +pub async fn handshake( + conn_config: ConnectConfiguration, + domain: &str, + io: S, +) -> Result> { + let ssl = conn_config + .into_ssl(domain) + .explain_err(TLSHandshakeFailure, |e| format!("tls config error: {e}"))?; + let mut stream = TlsStream::new(ssl, io) + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + + stream.connect().await + .map_err(|e| { + let err_msg = format!("TLS connect() failed: {e}, SNI: {domain}"); + if let Some(context) = e.context { + Error::explain(e.etype, + format!("{}, {}", err_msg, context.as_str())) + } else { + Error::explain(e.etype, err_msg) + } + })?; + Ok(stream) +} + diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs new file mode 100644 index 00000000..47b35a6f --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs @@ -0,0 +1,171 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS specific implementation + +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; + +use pingora_error::ErrorType::TLSHandshakeFailure; +use pingora_error::{Result, OrErr}; + +use crate::protocols::{ALPN, Ssl, UniqueID}; +use crate::protocols::tls::boringssl_openssl::stream::InnerStream; +use crate::protocols::tls::SslDigest; +use crate::tls::hash::MessageDigest; +use crate::tls::ssl; +use crate::tls::ssl::SslRef; +use crate::utils::tls::boringssl_openssl::{get_x509_organization, get_x509_serial}; + +use super::TlsStream; + +pub(super) mod stream; +pub mod client; +pub mod server; + +impl TlsStream +where + T: AsyncRead + AsyncWrite + Unpin + Send +{ + /// Create a new TLS connection from the given `stream` + /// + /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS + /// handshake after. + pub fn new(ssl: ssl::Ssl, stream: T) -> Result { + let tls = InnerStream::new(ssl, stream) + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + Ok(TlsStream { + tls, + digest: None, + timing: Default::default(), + }) + } +} + +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Self::clear_error(&self); + Pin::new(&mut self.tls.0).poll_read(cx, buf) + } +} + +impl TlsStream { + #[inline] + fn clear_error(&self) { + InnerStream::::clear_error() + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Self::clear_error(&self); + Pin::new(&mut self.tls.0).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(&self); + Pin::new(&mut self.tls.0).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(&self); + Pin::new(&mut self.tls.0).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Self::clear_error(&self); + Pin::new(&mut self.tls.0).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.tls.0.get_ref().id() + } +} + +impl Ssl for TlsStream { + fn get_ssl(&self) -> Option<&ssl::SslRef> { + Some(self.tls.0.ssl()) + } + + fn get_ssl_digest(&self) -> Option> { + self.ssl_digest() + } + + fn selected_alpn_proto(&self) -> Option { + let ssl = self.tls.0.ssl(); + ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + } +} + +impl SslDigest { + pub fn from_ssl(ssl: &SslRef) -> Self { + let cipher = match ssl.current_cipher() { + Some(c) => c.name(), + None => "", + }; + + let (cert_digest, org, sn) = match ssl.peer_certificate() { + Some(cert) => { + let cert_digest = match cert.digest(MessageDigest::sha256()) { + Ok(c) => c.as_ref().to_vec(), + Err(_) => Vec::new(), + }; + ( + cert_digest, + get_x509_organization(&cert), + get_x509_serial(&cert).ok(), + ) + } + None => (Vec::new(), None, None), + }; + + SslDigest { + cipher, + version: ssl.version_str(), + organization: org, + serial_number: sn, + cert_digest, + } + } +} \ No newline at end of file diff --git a/pingora-core/src/protocols/ssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs similarity index 54% rename from pingora-core/src/protocols/ssl/server.rs rename to pingora-core/src/protocols/tls/boringssl_openssl/server.rs index c85846b1..639bbcb9 100644 --- a/pingora-core/src/protocols/ssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -12,31 +12,62 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! TLS server specific implementation +//! BoringSSL & OpenSSL TLS server specific implementation -use super::SslStream; -use crate::protocols::{Shutdown, IO}; +use std::pin::Pin; + +use async_trait::async_trait; +use tokio::io::{AsyncRead, AsyncWrite}; + +use pingora_error::{OrErr, Result}; +use pingora_error::ErrorType::{TLSHandshakeFailure, TLSWantX509Lookup}; + +use crate::listeners::tls::Acceptor; +use crate::protocols::{IO, Ssl}; +use crate::protocols::tls::boringssl_openssl::TlsStream; +use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; use crate::tls::ext; use crate::tls::ext::ssl_from_acceptor; -use crate::tls::ssl; -use crate::tls::ssl::{SslAcceptor, SslRef}; +use crate::tls::ssl::SslAcceptor; -use async_trait::async_trait; -use log::warn; -use pingora_error::{ErrorType::*, OrErr, Result}; -use std::pin::Pin; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +#[async_trait] +impl ResumableAccept for TlsStream { + async fn start_accept(mut self: Pin<&mut Self>) -> Result { + // safety: &mut self + let ssl_mut = unsafe { ext::ssl_mut(self.get_ssl().unwrap()) }; + ext::suspend_when_need_ssl_cert(ssl_mut); + let res = self.accept().await; -/// Prepare a TLS stream for handshake -pub fn prepare_tls_stream(ssl_acceptor: &SslAcceptor, io: S) -> Result> { + match res { + Ok(()) => Ok(true), + Err(e) => { + if e.etype == TLSWantX509Lookup { + Ok(false) + } else { + Err(e) + } + } + } + } + + async fn resume_accept(mut self: Pin<&mut Self>) -> Result<()> { + // safety: &mut ssl + let ssl_mut = unsafe { ext::ssl_mut(self.get_ssl().unwrap()) }; + ext::unblock_ssl_cert(ssl_mut); + self.accept().await + } +} + +fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { + let ssl_acceptor = acceptor.inner().downcast_ref::().unwrap(); let ssl = ssl_from_acceptor(ssl_acceptor) .explain_err(TLSHandshakeFailure, |e| format!("ssl_acceptor error: {e}"))?; - SslStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("ssl stream error: {e}")) + TlsStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(ssl_acceptor: &SslAcceptor, io: S) -> Result> { - let mut stream = prepare_tls_stream(ssl_acceptor, io)?; +pub async fn handshake(acceptor: &Acceptor, io: Box) -> Result>> { + let mut stream = prepare_tls_stream(acceptor, io)?; stream .accept() .await @@ -45,19 +76,18 @@ pub async fn handshake(ssl_acceptor: &SslAcceptor, io: S) -> Result( - ssl_acceptor: &SslAcceptor, - io: S, +pub async fn handshake_with_callback( + acceptor: &Acceptor, + io: Box, callbacks: &TlsAcceptCallbacks, -) -> Result> { - let mut tls_stream = prepare_tls_stream(ssl_acceptor, io)?; +) -> pingora_error::Result>> { + let mut tls_stream = prepare_tls_stream(acceptor, io)?; let done = Pin::new(&mut tls_stream) .start_accept() - .await - .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; + .await?; if !done { // safety: we do hold a mut ref of tls_stream - let ssl_mut = unsafe { ext::ssl_mut(tls_stream.ssl()) }; + let ssl_mut = unsafe { ext::ssl_mut(tls_stream.0.ssl()) }; callbacks.certificate_callback(ssl_mut).await; Pin::new(&mut tls_stream) .resume_accept() @@ -69,85 +99,14 @@ pub async fn handshake_with_callback( } } -/// The APIs to customize things like certificate during TLS server side handshake -#[async_trait] -pub trait TlsAccept { - // TODO: return error? - /// This function is called in the middle of a TLS handshake. Structs who implement this function - /// should provide tls certificate and key to the [SslRef] via [ext::ssl_use_certificate] and [ext::ssl_use_private_key]. - async fn certificate_callback(&self, _ssl: &mut SslRef) -> () { - // does nothing by default - } -} - -pub type TlsAcceptCallbacks = Box; - -#[async_trait] -impl Shutdown for SslStream -where - S: AsyncRead + AsyncWrite + Sync + Unpin + Send, -{ - async fn shutdown(&mut self) { - match ::shutdown(self).await { - Ok(()) => {} - Err(e) => { - warn!("TLS shutdown failed, {e}"); - } - } - } -} - -/// Resumable TLS server side handshake. -#[async_trait] -pub trait ResumableAccept { - /// Start a resumable TLS accept handshake. - /// - /// * `Ok(true)` when the handshake is finished - /// * `Ok(false)`` when the handshake is paused midway - /// - /// For now, the accept will only pause when a certificate is needed. - async fn start_accept(self: Pin<&mut Self>) -> Result; - - /// Continue the TLS handshake - /// - /// This function should be called after the certificate is provided. - async fn resume_accept(self: Pin<&mut Self>) -> Result<(), ssl::Error>; -} - -#[async_trait] -impl ResumableAccept for SslStream { - async fn start_accept(mut self: Pin<&mut Self>) -> Result { - // safety: &mut self - let ssl_mut = unsafe { ext::ssl_mut(self.ssl()) }; - ext::suspend_when_need_ssl_cert(ssl_mut); - let res = self.accept().await; - - match res { - Ok(()) => Ok(true), - Err(e) => { - if ext::is_suspended_for_cert(&e) { - Ok(false) - } else { - Err(e) - } - } - } - } - - async fn resume_accept(mut self: Pin<&mut Self>) -> Result<(), ssl::Error> { - // safety: &mut ssl - let ssl_mut = unsafe { ext::ssl_mut(self.ssl()) }; - ext::unblock_ssl_cert(ssl_mut); - self.accept().await - } -} - #[tokio::test] async fn test_async_cert() { + use crate::tls::ssl; use tokio::io::AsyncReadExt; - let acceptor = ssl::SslAcceptor::mozilla_intermediate_v5(ssl::SslMethod::tls()) - .unwrap() - .build(); + use crate::tls::ssl::SslRef; + use crate::listeners::TlsAccept; + use crate::listeners::tls::TlsSettings; + use crate::protocols::tls::server::TlsAcceptCallbacks; struct Callback; #[async_trait] @@ -180,14 +139,13 @@ async fn test_async_cert() { .build(); let mut ssl = ssl::Ssl::new(&ssl_context).unwrap(); ssl.set_hostname("pingora.org").unwrap(); - ssl.set_verify(ssl::SslVerifyMode::NONE); // we don have a valid cert - let mut stream = SslStream::new(ssl, client).unwrap(); + ssl.set_verify(ssl::SslVerifyMode::NONE); // we don't have a valid cert + let mut stream = TlsStream::new(ssl, client).unwrap(); Pin::new(&mut stream).connect().await.unwrap(); let mut buf = [0; 1]; let _ = stream.read(&mut buf).await; }); - handshake_with_callback(&acceptor, server, &cb) - .await - .unwrap(); -} + let acceptor = TlsSettings::with_callbacks(cb).unwrap().build(); + acceptor.handshake(Box::new(server)).await.unwrap(); +} \ No newline at end of file diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs new file mode 100644 index 00000000..9a3a07c9 --- /dev/null +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -0,0 +1,166 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BoringSSL & OpenSSL TLS stream specific implementation + +use std::pin::Pin; +use std::sync::Arc; +use async_trait::async_trait; +use log::warn; +use tokio::io::{AsyncRead, AsyncWrite}; + +use pingora_error::{Error, ErrorType::*, OrErr, Result}; + +use crate::listeners::ALPN; +use crate::protocols::{GetProxyDigest, GetTimingDigest}; +use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; +use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::InnerTlsStream; +use crate::protocols::tls::SslDigest; +use crate::tls::tokio_ssl::SslStream; +use crate::tls::error::ErrorStack; +use crate::tls::ext; +use crate::tls::tokio_ssl; +use crate::tls::{ssl, ssl::SslRef, ssl_sys::X509_V_ERR_INVALID_CALL}; + +#[derive(Debug)] +pub struct InnerStream(pub(crate) tokio_ssl::SslStream); + +impl InnerStream { + /// Create a new TLS connection from the given `stream` + /// + /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS + /// handshake after. + pub(crate) fn new(ssl: ssl::Ssl, stream: T) -> Result { + let ssl = SslStream::new(ssl, stream) + .explain_err(TLSHandshakeFailure, |e| format!("tls.rs stream error: {e}"))?; + + Ok(InnerStream(ssl)) + } + + #[inline] + pub(crate) fn clear_error() { + let errs = ErrorStack::get(); + if !errs.errors().is_empty() { + warn!("Clearing dirty TLS error stack: {}", errs); + } + } +} + +#[async_trait] +impl InnerTlsStream for InnerStream { + /// Connect to the remote TLS server as a client + async fn connect(&mut self) -> Result<()> { + Self::clear_error(); + match Pin::new(&mut self.0).connect().await { + Ok(_) => { + Ok(()) + } + Err(err) => { + self.transform_ssl_error(err) + } + } + } + + /// Finish the TLS handshake from client as a server + async fn accept(&mut self) -> Result<()> { + Self::clear_error(); + match Pin::new(&mut self.0).accept().await { + Ok(_) => { + Ok(()) + } + Err(err) => { + self.transform_ssl_error(err) + } + } + } + + fn digest(&mut self) -> Option> { + Some(Arc::new(SslDigest::from_ssl(self.0.ssl()))) + } + + fn selected_alpn_proto(&mut self) -> Option { + let ssl = self.0.ssl(); + ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + } +} + +impl InnerStream { + fn transform_ssl_error(&self, e: ssl::Error) -> Result<()> { + let context = format!("ssl::ErrorCode: {:?}", e.code()); + if ext::is_suspended_for_cert(&e) { + Error::e_explain(TLSWantX509Lookup, format!("ssl::ErrorCode: {:?}", e.code())) + } else { + match e.code() { + ssl::ErrorCode::SSL => { + // Unify the return type of `verify_result` for openssl + #[cfg(not(feature = "boringssl"))] + fn verify_result(ssl: &SslRef) -> Result<(), i32> { + match ssl.verify_result().as_raw() { + crate::tls::ssl_sys::X509_V_OK => Ok(()), + e => Err(e), + } + } + + // Unify the return type of `verify_result` for boringssl + #[cfg(feature = "boringssl")] + fn verify_result(ssl: &SslRef) -> Result<(), i32> { + ssl.verify_result().map_err(|e| e.as_raw()) + } + + match verify_result::(self.0.ssl()) { + Ok(()) => Error::e_explain(TLSHandshakeFailure, context), + // X509_V_ERR_INVALID_CALL in case verify result was never set + Err(X509_V_ERR_INVALID_CALL) => { + Error::e_explain(TLSHandshakeFailure, context) + } + _ => Error::e_explain(InvalidCert, context), + } + } + /* likely network error, but still mark as TLS error */ + _ => Error::e_explain(TLSHandshakeFailure, context), + } + } + } +} + +impl GetSocketDigest for InnerStream + where + S: GetSocketDigest, +{ + fn get_socket_digest(&self) -> Option> { + self.0.get_ref().get_socket_digest() + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + self.0.get_mut().set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for InnerStream + where + S: GetTimingDigest, +{ + fn get_timing_digest(&self) -> Vec> { + self.0.get_ref().get_timing_digest() + } +} + +impl GetProxyDigest for InnerStream + where + S: GetProxyDigest, +{ + fn get_proxy_digest(&self) -> Option> { + self.0.get_ref().get_proxy_digest() + } +} \ No newline at end of file diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs new file mode 100644 index 00000000..77203f81 --- /dev/null +++ b/pingora-core/src/protocols/tls/mod.rs @@ -0,0 +1,265 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The TLS layer implementations + +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use async_trait::async_trait; +use tokio::io::{AsyncRead, AsyncWrite}; +use pingora_error::Result; + +use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, IO, SocketDigest, UniqueID}; +use crate::protocols::digest::TimingDigest; +use crate::protocols::raw_connect::ProxyDigest; + +pub mod server; +#[cfg(not(feature = "rustls"))] +pub(crate) mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub(crate) mod rustls; + +#[cfg(not(feature = "rustls"))] +use boringssl_openssl::stream::InnerStream; +#[cfg(feature = "rustls")] +use rustls::stream::InnerStream; + + +/// The TLS connection +#[derive(Debug)] +pub struct TlsStream { + tls: InnerStream, + digest: Option>, + timing: TimingDigest, +} + +#[async_trait] +pub trait InnerTlsStream { + async fn connect(&mut self) -> Result<()>; + async fn accept(&mut self) -> Result<()>; + + /// Return the [`ssl::SslDigest`] for logging + fn digest(&mut self) -> Option>; + + /// Return selected ALPN if any + fn selected_alpn_proto(&mut self) -> Option; +} + + +impl GetSocketDigest for Box +{ + fn get_socket_digest(&self) -> Option> { + (**self).get_socket_digest() + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + (**self).set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for Box +{ + fn get_timing_digest(&self) -> Vec> { + vec![] + } +} + +impl GetProxyDigest for Box +{ + fn get_proxy_digest(&self) -> Option> { + (**self).get_proxy_digest() + } +} + +impl UniqueID for Box +{ + fn id(&self) -> i32 { + (**self).id() + } +} + + +/// The protocol for Application-Layer Protocol Negotiation +#[derive(Hash, Clone, Debug)] +pub enum ALPN { + /// Prefer HTTP/1.1 only + H1, + /// Prefer HTTP/2 only + H2, + /// Prefer HTTP/2 over HTTP/1.1 + H2H1, +} + +impl std::fmt::Display for ALPN { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ALPN::H1 => write!(f, "H1"), + ALPN::H2 => write!(f, "H2"), + ALPN::H2H1 => write!(f, "H2H1"), + } + } +} + +impl ALPN { + /// Create a new ALPN according to the `max` and `min` version constraints + pub fn new(max: u8, min: u8) -> Self { + if max == 1 { + ALPN::H1 + } else if min == 2 { + ALPN::H2 + } else { + ALPN::H2H1 + } + } + + /// Return the max http version this [`ALPN`] allows + pub fn get_max_http_version(&self) -> u8 { + match self { + ALPN::H1 => 1, + _ => 2, + } + } + + /// Return the min http version this [`ALPN`] allows + pub fn get_min_http_version(&self) -> u8 { + match self { + ALPN::H2 => 2, + _ => 1, + } + } + + #[cfg(not(feature = "rustls"))] + pub(crate) fn to_wire_preference(&self) -> &[u8] { + // https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_alpn_select_cb.html + // "vector of nonempty, 8-bit length-prefixed, byte strings" + match self { + Self::H1 => b"\x08http/1.1", + Self::H2 => b"\x02h2", + Self::H2H1 => b"\x02h2\x08http/1.1", + } + } + + #[cfg(feature = "rustls")] + pub(crate) fn to_wire_protocols(&self) -> Vec> { + match self { + ALPN::H1 => vec![b"http/1.1".to_vec()], + ALPN::H2 => vec![b"h2".to_vec()], + ALPN::H2H1 => vec![b"h2".to_vec(), b"http/1.1".to_vec()] + } + } + + pub(crate) fn from_wire_selected(raw: &[u8]) -> Option { + match raw { + b"http/1.1" => Some(Self::H1), + b"h2" => Some(Self::H2), + _ => None, + } + } +} + +/// The TLS connection information +#[derive(Clone, Debug)] +pub struct SslDigest { + /// The cipher used + pub cipher: &'static str, + /// The TLS version of this connection + pub version: &'static str, + /// The organization of the peer's certificate + pub organization: Option, + /// The serial number of the peer's certificate + pub serial_number: Option, + /// The digest of the peer's certificate + pub cert_digest: Vec, +} + + +impl GetSocketDigest for TlsStream +where + S: GetSocketDigest, +{ + fn get_socket_digest(&self) -> Option> { + self.tls.get_socket_digest() + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + self.tls.set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for TlsStream +where + S: GetTimingDigest, +{ + fn get_timing_digest(&self) -> Vec> { + let mut ts_vec = self.tls.get_timing_digest(); + ts_vec.push(Some(self.timing.clone())); + ts_vec + } + fn get_read_pending_time(&self) -> Duration { + self.tls.get_read_pending_time() + } + + fn get_write_pending_time(&self) -> Duration { + self.tls.get_write_pending_time() + } +} + +impl GetProxyDigest for TlsStream +where + S: GetProxyDigest, +{ + fn get_proxy_digest(&self) -> Option> { + self.tls.get_proxy_digest() + } +} + +impl TlsStream { + pub fn ssl_digest(&self) -> Option> { + self.digest.clone() + } +} + +impl TlsStream +where + T: AsyncRead + AsyncWrite + Unpin + Send +{ + /// Connect to the remote TLS server as a client + pub(crate) async fn connect(&mut self) -> Result<()> { + self.tls.connect().await?; + self.timing.established_ts = SystemTime::now(); + self.digest = self.tls.digest(); + Ok(()) + } + + /// Finish the TLS handshake from client as a server + pub(crate) async fn accept(&mut self) -> Result<()> { + self.tls.accept().await?; + self.timing.established_ts = SystemTime::now(); + self.digest = self.tls.digest(); + Ok(()) + } +} + +impl Deref for TlsStream { + type Target = InnerStream; + + fn deref(&self) -> &Self::Target { + &self.tls + } +} + +impl DerefMut for TlsStream { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.tls + } +} diff --git a/pingora-core/src/protocols/tls/rustls/client.rs b/pingora-core/src/protocols/tls/rustls/client.rs new file mode 100644 index 00000000..9ed65ad3 --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/client.rs @@ -0,0 +1,41 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS client specific implementation + +use pingora_error::{Error, OrErr, Result}; +use pingora_error::ErrorType::TLSHandshakeFailure; +use pingora_rustls::TlsConnector; +use crate::protocols::IO; +use crate::protocols::tls::rustls::TlsStream; + +// Perform the TLS handshake for the given connection with the given configuration +pub async fn handshake( + connector: &TlsConnector, + domain: &str, + io: S, +) -> Result> { + let mut stream = TlsStream::from_connector(connector, domain, io).await + .explain_err(TLSHandshakeFailure, |e| format!("tip: tls stream error: {e}"))?; + + let handshake_result = stream.connect().await; + match handshake_result { + Ok(()) => Ok(stream), + Err(e) => { + let context = format!("TLS connect() failed: {e}, SNI: {domain}"); + Error::e_explain(TLSHandshakeFailure, context) + } + } +} + diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs new file mode 100644 index 00000000..5d59603b --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -0,0 +1,216 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS specific implementation + +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; +use x509_parser::nom::AsBytes; +use pingora_error::{OrErr, Result}; +use pingora_error::ErrorType::{InternalError, TLSHandshakeFailure}; +use pingora_rustls::{hash_certificate, ServerName, TlsConnector}; +use pingora_rustls::TlsStream as RusTlsStream; + +use crate::utils::tls::rustls::{get_organization_serial}; + +use crate::listeners::tls::{Acceptor}; +use crate::protocols::{ALPN, Ssl, UniqueID}; +use crate::protocols::tls::rustls::stream::InnerStream; +use crate::protocols::tls::SslDigest; + +use super::TlsStream; + +pub mod client; +pub mod server; +pub(super) mod stream; + +impl TlsStream +where + T: AsyncRead + AsyncWrite + Unpin + Send, +{ + /// Create a new TLS connection from the given `stream` + /// + /// Using RustTLS the stream is only returned after the handshake. + /// The caller does therefor not need to perform [`Self::connect()`]. + pub async fn from_connector(connector: &TlsConnector, domain: &str, stream: T) -> Result { + let server = ServerName::try_from(domain) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) + .explain_err(InternalError, |e| format!("failed to parse domain: {}, error: {}", domain, e))? + .to_owned(); + + let tls = InnerStream::from_connector(connector, server, stream).await + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + + Ok(TlsStream { + tls, + digest: None, + timing: Default::default(), + }) + } + + /// Create a new TLS connection from the given `stream` + /// + /// Using RustTLS the stream is only returned after the handshake. + /// The caller does therefor not need to perform [`Self::accept()`]. + pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { + let tls = InnerStream::from_acceptor(acceptor, stream).await + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; + + Ok(TlsStream { + tls, + digest: None, + timing: Default::default(), + }) + } +} + +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.tls.stream.as_ref().unwrap().get_ref().0.id() + } +} + +impl Ssl for TlsStream { + fn get_ssl_digest(&self) -> Option> { + self.ssl_digest() + } + + fn selected_alpn_proto(&self) -> Option { + let st = self.tls.stream.as_ref(); + if let Some(stream) = st { + let proto = stream.get_ref().1.alpn_protocol(); + match proto { + None => { None } + Some(raw) => { + ALPN::from_wire_selected(raw) + } + } + } else { + None + } + } +} + +impl SslDigest { + fn from_stream(stream: &Option>) -> Self { + let stream = stream.as_ref().unwrap(); + let (_io, session) = stream.get_ref(); + let protocol = session.protocol_version(); + let cipher_suite = session.negotiated_cipher_suite(); + let peer_certificates = session.peer_certificates(); + + let cipher = match cipher_suite { + Some(suite) => { + match suite.suite().as_str() { + Some(suite_str) => suite_str, + None => "", + } + }, + None => "", + }; + + let version = match protocol { + Some(proto) => { + match proto.as_str() { + Some(ver) => ver, + None => "", + } + }, + None => "", + }; + + let cert_digest = match peer_certificates { + Some(certs) => { + match certs.first() { + Some(cert) => hash_certificate(cert.clone()), + None => vec![], + } + }, + None => vec![], + }; + + let (organization, serial_number) = match peer_certificates { + Some(certs) => { + match certs.first() { + Some(cert) => { + let (organization, serial) = get_organization_serial(cert.as_bytes()); + (organization, Some(serial)) + }, + None => (None, None), + } + }, + None => (None, None), + }; + + SslDigest { + cipher: &cipher, + version: &version, + organization, + serial_number, + cert_digest, + } + } +} \ No newline at end of file diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs new file mode 100644 index 00000000..ad67fa58 --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -0,0 +1,95 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rustls TLS server specific implementation + +use std::pin::Pin; +use async_trait::async_trait; +use log::warn; +use tokio::io::{AsyncRead, AsyncWrite}; +use crate::protocols::tls::rustls::TlsStream; +use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; +use pingora_error::{ErrorType::*, OrErr, Result}; +use crate::protocols::IO; +use crate::listeners::tls::Acceptor; + +#[async_trait] +impl ResumableAccept for TlsStream { + async fn start_accept(mut self: Pin<&mut Self>) -> Result { + // TODO: suspend cert callback + let res = self.accept().await; + + match res { + Ok(()) => Ok(true), + Err(e) => { + if e.etype == TLSWantX509Lookup { + Ok(false) + } else { + Err(e) + } + } + } + } + + async fn resume_accept(mut self: Pin<&mut Self>) -> Result<()> { + // TODO: unblock cert callback + self.accept().await + } +} + +async fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { + TlsStream::from_acceptor(acceptor, io).await + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) +} + +/// Perform TLS handshake for the given connection with the given configuration +pub async fn handshake(acceptor: &Acceptor, io: Box) -> Result>> { + let mut stream = prepare_tls_stream(acceptor, io).await?; + stream + .accept() + .await + .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; + Ok(stream) +} + +/// Perform TLS handshake for the given connection with the given configuration and callbacks +/// callbacks are currently not supported within pingora Rustls and are ignored +pub async fn handshake_with_callback( + acceptor: &Acceptor, + io: Box, + _callbacks: &TlsAcceptCallbacks, +) -> Result>> { + let mut tls_stream = prepare_tls_stream(acceptor, io).await?; + let done = Pin::new(&mut tls_stream) + .start_accept() + .await?; + if !done { + // TODO: verify if/how callback in handshake can be done using Rustls + warn!("Callacks are not supported with feature \"rustls\"."); + + Pin::new(&mut tls_stream) + .resume_accept() + .await + .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; + Ok(tls_stream) + } else { + Ok(tls_stream) + } +} + +#[ignore] +#[tokio::test] +async fn test_async_cert() { + todo!("callback support and test for Rustls") +} \ No newline at end of file diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs new file mode 100644 index 00000000..4c3556d5 --- /dev/null +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -0,0 +1,158 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::sync::Arc; +use async_trait::async_trait; +use tokio::io::{AsyncRead, AsyncWrite}; +use pingora_error::{Error, ImmutStr, OrErr, Result}; +use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; +use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; +use pingora_rustls::TlsAcceptor as RusTlsAcceptor; +use pingora_rustls::TlsStream as RusTlsStream; +use pingora_rustls::NoDebug; + +use crate::listeners::ALPN; +use crate::listeners::tls::Acceptor; +use crate::protocols::{GetProxyDigest, GetTimingDigest}; +use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; +use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::InnerTlsStream; +use crate::protocols::tls::SslDigest; + +#[derive(Debug)] +pub struct InnerStream { + pub(crate) stream: Option>, + connect: NoDebug>>, + accept: NoDebug>>, +} + +impl InnerStream { + /// Create a new TLS connection from the given `stream` + /// + /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS + /// handshake after. + pub(crate) async fn from_connector(connector: &TlsConnector, server: ServerName<'_>, stream: T) -> Result { + let connect = connector.connect(server.to_owned(), stream); + Ok(InnerStream { + accept: None.into(), + connect: Some(connect).into(), + stream: None, + }) + } + + pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { + let tls_acceptor = acceptor.inner().downcast_ref::().unwrap(); + let accept = tls_acceptor.accept(stream); + + Ok(InnerStream { + accept: Some(accept).into(), + connect: None.into(), + stream: None, + }) + } +} + +#[async_trait] +impl InnerTlsStream for InnerStream { + /// Connect to the remote TLS server as a client + async fn connect(&mut self) -> Result<()> { + let connect = &mut (*self.connect); + + if let Some(ref mut connect) = connect { + let stream = connect.await + .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; + self.stream = Some(RusTlsStream::Client(stream)); + self.connect = None.into(); + + Ok(()) + } else { + Err(Error::explain(ConnectError, ImmutStr::from("TLS connect not available to perform handshake."))) + } + } + + /// Finish the TLS handshake from client as a server + /// no-op implementation within Rustls, handshake is performed during creation of stream. + async fn accept(&mut self) -> Result<()> { + let accept = &mut (*self.accept); + + if let Some(ref mut accept) = accept { + let stream = accept.await + .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; + self.stream = Some(RusTlsStream::Server(stream)); + self.connect = None.into(); + + Ok(()) + } else { + Err(Error::explain(AcceptError, ImmutStr::from("TLS accept not available to perform handshake."))) + } + } + + fn digest(&mut self) -> Option> { + Some(Arc::new(SslDigest::from_stream(&self.stream))) + } + + fn selected_alpn_proto(&mut self) -> Option { + if let Some(stream) = self.stream.as_ref() { + let proto = stream.get_ref().1.alpn_protocol(); + match proto { + None => { None } + Some(raw) => { + ALPN::from_wire_selected(raw) + } + } + } else { + None + } + } +} + + +impl GetSocketDigest for InnerStream + where + S: GetSocketDigest, +{ + fn get_socket_digest(&self) -> Option> { + if let Some(stream) = self.stream.as_ref() { + stream.get_ref().0.get_socket_digest() + } else { + None + } + } + fn set_socket_digest(&mut self, socket_digest: SocketDigest) { + self.stream.as_mut().unwrap().get_mut().0.set_socket_digest(socket_digest) + } +} + +impl GetTimingDigest for InnerStream + where + S: GetTimingDigest, +{ + fn get_timing_digest(&self) -> Vec> { + self.stream.as_ref().unwrap().get_ref().0.get_timing_digest() + } +} + +impl GetProxyDigest for InnerStream + where + S: GetProxyDigest, +{ + fn get_proxy_digest(&self) -> Option> { + if let Some(stream) = self.stream.as_ref() { + stream.get_ref().0.get_proxy_digest() + } else { + None + } + } +} \ No newline at end of file diff --git a/pingora-core/src/protocols/tls/server.rs b/pingora-core/src/protocols/tls/server.rs new file mode 100644 index 00000000..948d40fd --- /dev/null +++ b/pingora-core/src/protocols/tls/server.rs @@ -0,0 +1,88 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! TLS server specific implementation + +use std::pin::Pin; + +use async_trait::async_trait; +use log::warn; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; + +use pingora_error::Result; + +use crate::protocols::{IO, Shutdown}; +use crate::protocols::tls::TlsStream; + +#[cfg(not(feature = "rustls"))] +use crate::tls::ssl::SslRef; + +/// The APIs to customize things like certificate during TLS server side handshake +#[async_trait] +pub trait TlsAccept { + // TODO: return error? + /// This function is called in the middle of a TLS handshake. Structs who implement this function + /// should provide tls certificate and key to the [SslRef] via [ext::ssl_use_certificate] and [ext::ssl_use_private_key]. + #[cfg(not(feature = "rustls"))] + async fn certificate_callback(&self, _ssl: &mut SslRef) -> () { + // does nothing by default + } +} + +pub type TlsAcceptCallbacks = Box; + +#[async_trait] +impl Shutdown for TlsStream +where + S: AsyncRead + AsyncWrite + Sync + Unpin + Send, +{ + async fn shutdown(&mut self) { + match ::shutdown(self).await { + Ok(()) => {} + Err(e) => { + warn!("TLS shutdown failed, {e}"); + } + } + } +} + +#[async_trait] +impl Shutdown for Box +{ + async fn shutdown(&mut self) { + match ::shutdown(self).await { + Ok(()) => {} + Err(e) => { + warn!("TLS shutdown failed, {e}"); + } + } + } +} + +/// Resumable TLS server side handshake. +#[async_trait] +pub trait ResumableAccept { + /// Start a resumable TLS accept handshake. + /// + /// * `Ok(true)` when the handshake is finished + /// * `Ok(false)`` when the handshake is paused midway + /// + /// For now, accept will only pause when a certificate is needed. + async fn start_accept(self: Pin<&mut Self>) -> Result; + + /// Continue the TLS handshake + /// + /// This function should be called after the certificate is provided. + async fn resume_accept(self: Pin<&mut Self>) -> Result<()>; +} \ No newline at end of file diff --git a/pingora-core/src/upstreams/mod.rs b/pingora-core/src/upstreams/mod.rs index 7352b615..663a4973 100644 --- a/pingora-core/src/upstreams/mod.rs +++ b/pingora-core/src/upstreams/mod.rs @@ -14,4 +14,4 @@ //! The interface to connect to a remote server -pub mod peer; +pub mod peer; \ No newline at end of file diff --git a/pingora-core/src/upstreams/peer.rs b/pingora-core/src/upstreams/peer.rs index d0c8125a..340311a2 100644 --- a/pingora-core/src/upstreams/peer.rs +++ b/pingora-core/src/upstreams/peer.rs @@ -32,10 +32,13 @@ use std::time::Duration; use crate::protocols::l4::socket::SocketAddr; use crate::protocols::ConnFdReusable; use crate::protocols::TcpKeepalive; -use crate::tls::x509::X509; -use crate::utils::{get_organization_unit, CertKey}; +use crate::utils::tls::CertKey; +#[cfg(not(feature = "rustls"))] +use crate::utils::tls::boringssl_openssl::{get_organizational_unit, get_not_after}; +#[cfg(feature = "rustls")] +use crate::utils::tls::rustls::{get_organizational_unit, get_not_after}; -pub use crate::protocols::ssl::ALPN; +pub use crate::protocols::tls::ALPN; /// The interface to trace the connection pub trait Tracing: Send + Sync + std::fmt::Debug { @@ -66,7 +69,7 @@ pub trait Peer: Display + Clone { fn tls(&self) -> bool; /// The SNI to send, if TLS is used fn sni(&self) -> &str; - /// To decide whether a [`Peer`] can use the connection established by another [`Peer`]. + /// To decide whether a [`Peer`] can use the connection established by another [`Peer`]. /// /// The connections to two peers are considered reusable to each other if their reuse hashes are /// the same @@ -144,7 +147,7 @@ pub trait Peer: Display + Clone { /// Get the CA cert to use to validate the server cert. /// /// If not set, the default CAs will be used. - fn get_ca(&self) -> Option<&Arc>> { + fn get_ca(&self) -> Option<&Arc<[Box<[u8]>]>> { match self.get_peer_options() { Some(opt) => opt.ca.as_ref(), None => None, @@ -304,7 +307,7 @@ pub struct PeerOptions { /* accept the cert if it's CN matches the SNI or this name */ pub alternative_cn: Option, pub alpn: ALPN, - pub ca: Option>>, + pub ca: Option]>>, pub tcp_keepalive: Option, pub tcp_recv_buf: Option, pub dscp: Option, @@ -385,8 +388,8 @@ impl Display for PeerOptions { write!( f, "CA: {}, expire: {},", - get_organization_unit(ca).unwrap_or_default(), - ca.not_after() + get_organizational_unit(&**ca).unwrap_or_default(), + get_not_after(ca), )?; } } diff --git a/pingora-core/src/utils/mod.rs b/pingora-core/src/utils/mod.rs index c36f7c82..8057d58c 100644 --- a/pingora-core/src/utils/mod.rs +++ b/pingora-core/src/utils/mod.rs @@ -15,12 +15,9 @@ //! This module contains various types that make it easier to work with bytes and X509 //! certificates. -// TODO: move below to its own mod -use crate::tls::{nid::Nid, pkey::PKey, pkey::Private, x509::X509}; -use crate::Result; +pub mod tls; + use bytes::Bytes; -use pingora_error::{ErrorType::*, OrErr}; -use std::hash::{Hash, Hasher}; /// A `BufRef` is a reference to a buffer of bytes. It removes the need for self-referential data /// structures. It is safe to use as long as the underlying buffer does not get mutated. @@ -108,125 +105,3 @@ pub const EMPTY_KV_REF: KVRef = KVRef { name: BufRef(0, 0), value: BufRef(0, 0), }; - -fn get_subject_name(cert: &X509, name_type: Nid) -> Option { - cert.subject_name() - .entries_by_nid(name_type) - .next() - .map(|name| { - name.data() - .as_utf8() - .map(|s| s.to_string()) - .unwrap_or_default() - }) -} - -/// Return the organization associated with the X509 certificate. -pub fn get_organization(cert: &X509) -> Option { - get_subject_name(cert, Nid::ORGANIZATIONNAME) -} - -/// Return the common name associated with the X509 certificate. -pub fn get_common_name(cert: &X509) -> Option { - get_subject_name(cert, Nid::COMMONNAME) -} - -/// Return the common name associated with the X509 certificate. -pub fn get_organization_unit(cert: &X509) -> Option { - get_subject_name(cert, Nid::ORGANIZATIONALUNITNAME) -} - -/// Return the serial number associated with the X509 certificate as a hexadecimal value. -pub fn get_serial(cert: &X509) -> Result { - let bn = cert - .serial_number() - .to_bn() - .or_err(InvalidCert, "Invalid serial")?; - let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; - - let hex_str: &str = hex.as_ref(); - Ok(hex_str.to_owned()) -} - -/// This type contains a list of one or more certificates and an associated private key. The leaf -/// certificate should always be first. -#[derive(Clone)] -pub struct CertKey { - certificates: Vec, - key: PKey, -} - -impl CertKey { - /// Create a new `CertKey` given a list of certificates and a private key. - pub fn new(certificates: Vec, key: PKey) -> CertKey { - assert!( - !certificates.is_empty(), - "expected a non-empty vector of certificates in CertKey::new" - ); - - CertKey { certificates, key } - } - - /// Peek at the leaf certificate. - pub fn leaf(&self) -> &X509 { - // This is safe due to the assertion above. - &self.certificates[0] - } - - /// Return the key. - pub fn key(&self) -> &PKey { - &self.key - } - - /// Return a slice of intermediate certificates. An empty slice means there are none. - pub fn intermediates(&self) -> &[X509] { - if self.certificates.len() <= 1 { - return &[]; - } - &self.certificates[1..] - } - - /// Return the organization from the leaf certificate. - pub fn organization(&self) -> Option { - get_organization(self.leaf()) - } - - /// Return the serial from the leaf certificate. - pub fn serial(&self) -> Result { - get_serial(self.leaf()) - } -} - -impl Hash for CertKey { - fn hash(&self, state: &mut H) { - for certificate in &self.certificates { - if let Ok(serial) = get_serial(certificate) { - serial.hash(state) - } - } - } -} - -// hide private key -impl std::fmt::Debug for CertKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CertKey") - .field("X509", &self.leaf()) - .finish() - } -} - -impl std::fmt::Display for CertKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let leaf = self.leaf(); - if let Some(cn) = get_common_name(leaf) { - // Write CN if it exists - write!(f, "CN: {cn},")?; - } else if let Some(org_unit) = get_organization_unit(leaf) { - // CA cert might not have CN, so print its unit name instead - write!(f, "Org Unit: {org_unit},")?; - } - write!(f, ", expire: {}", leaf.not_after()) - // ignore the details of the private key - } -} diff --git a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs new file mode 100644 index 00000000..a6b68667 --- /dev/null +++ b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs @@ -0,0 +1,99 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various helpers that make it easier to work with X509 certificates. + +use pingora_error::ErrorType::{InternalError, InvalidCert}; +use pingora_error::OrErr; + +use crate::tls::nid::Nid; +use crate::tls::pkey::{PKey, Private}; +use crate::tls::x509::X509; + +fn get_subject_name(cert: &X509, name_type: Nid) -> Option { + cert.subject_name() + .entries_by_nid(name_type) + .next() + .map(|name| { + name.data() + .as_utf8() + .map(|s| s.to_string()) + .unwrap_or_default() + }) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_x509_organization(cert: &X509) -> Option { + get_subject_name(cert, Nid::ORGANIZATIONNAME) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_organization(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::ORGANIZATIONNAME) +} + +/// Return the common name associated with the X509 certificate. +pub fn get_common_name(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::COMMONNAME) +} + +/// Return the organizational unit associated with the X509 certificate. +pub fn get_organizational_unit(cert: &[u8]) -> Option { + let cert = der_to_x509(cert).unwrap(); + get_subject_name(&cert, Nid::ORGANIZATIONALUNITNAME) +} + +/// Return the common name associated with the X509 certificate. +pub fn get_not_after(cert: &[u8]) -> String { + let cert = der_to_x509(cert).unwrap(); + cert.not_after().to_string() +} + +/// Return the serial number associated with the X509 certificate as a hexadecimal value. +pub fn get_serial(cert: &[u8]) -> pingora_error::Result { + let cert = der_to_x509(cert).unwrap(); + let bn = cert + .serial_number() + .to_bn() + .or_err(InvalidCert, "Invalid serial")?; + let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; + + let hex_str: &str = hex.as_ref(); + Ok(hex_str.to_owned()) +} + +pub fn get_x509_serial(cert: &X509) -> pingora_error::Result { + let bn = cert + .serial_number() + .to_bn() + .or_err(InvalidCert, "Invalid serial")?; + let hex = bn.to_hex_str().or_err(InvalidCert, "Invalid serial")?; + + let hex_str: &str = hex.as_ref(); + Ok(hex_str.to_owned()) +} + +pub fn der_to_x509(ca: &[u8]) -> pingora_error::Result { + let cert = X509::from_der(&*ca) + .explain_err(InvalidCert, |e| format!("Failed to convert ca certificate in DER form to X509 cert. Error: {:?}", e))?; + Ok(cert) +} + +pub fn der_to_private_key(key: &[u8]) -> pingora_error::Result> { + let key = PKey::private_key_from_der(key) + .explain_err(InternalError, |e| format!("Failed to convert private key in DER form to Pkey. Error: {:?}", e))?; + Ok(key) +} \ No newline at end of file diff --git a/pingora-core/src/utils/tls/mod.rs b/pingora-core/src/utils/tls/mod.rs new file mode 100644 index 00000000..bab25a19 --- /dev/null +++ b/pingora-core/src/utils/tls/mod.rs @@ -0,0 +1,108 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various types that make it easier to work with bytes and X509 +//! certificates. + +#[cfg(not(feature = "rustls"))] +pub mod boringssl_openssl; +#[cfg(feature = "rustls")] +pub mod rustls; + +use std::hash::{Hash, Hasher}; +#[cfg(not(feature = "rustls"))] +use boringssl_openssl::{get_organization, get_serial, get_common_name, get_organizational_unit, get_not_after}; +#[cfg(feature = "rustls")] +use rustls::{get_organization, get_serial, get_common_name, get_organizational_unit, get_not_after}; + +/// This type contains a list of one or more certificates and an associated private key. The leaf +/// certificate should always be first. The certificates and keys are stored in Vec DER encoded +/// form for usage within OpenSSL/BoringSSL & RusTLS. +#[derive(Clone)] +pub struct CertKey { + certificates: Vec>, + key: Vec, +} + +impl CertKey { + /// Create a new `CertKey` given a list of certificates and a private key. + pub fn new(certificates: Vec>, key: Vec) -> CertKey { + assert!( + !certificates.is_empty() && !certificates.first().unwrap().is_empty(), + "expected a non-empty vector of certificates in CertKey::new" + ); + + CertKey { certificates, key } + } + + /// Peek at the leaf certificate. + pub fn leaf(&self) -> &Vec { + // This is safe due to the assertion above. + &self.certificates[0] + } + + /// Return the key. + pub fn key(&self) -> &Vec { + &self.key + } + + /// Return a slice of intermediate certificates. An empty slice means there are none. + pub fn intermediates(&self) -> Vec<&Vec> { + self.certificates.iter().skip(1).collect() + } + + /// Return the organization from the leaf certificate. + pub fn organization(&self) -> Option { + get_organization(self.leaf()) + } + + /// Return the serial from the leaf certificate. + pub fn serial(&self) -> String { + get_serial(self.leaf()).unwrap() + } +} + +// hide private key +impl std::fmt::Debug for CertKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CertKey") + .field("X509", &self.leaf()) + .finish() + } +} + +impl std::fmt::Display for CertKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let leaf = self.leaf(); + if let Some(cn) = get_common_name(leaf) { + // Write CN if it exists + write!(f, "CN: {cn},")?; + } else if let Some(org_unit) = get_organizational_unit(leaf) { + // CA cert might not have CN, so print its unit name instead + write!(f, "Org Unit: {org_unit},")?; + } + write!(f, ", expire: {}", get_not_after(leaf)) + // ignore the details of the private key + } +} + +impl Hash for CertKey { + fn hash(&self, state: &mut H) { + for certificate in &self.certificates { + if let Ok(serial) = get_serial(certificate) { + serial.hash(state) + } + } + } +} \ No newline at end of file diff --git a/pingora-core/src/utils/tls/rustls/mod.rs b/pingora-core/src/utils/tls/rustls/mod.rs new file mode 100644 index 00000000..7a857858 --- /dev/null +++ b/pingora-core/src/utils/tls/rustls/mod.rs @@ -0,0 +1,66 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains various helpers that make it easier to work with X509 certificates. + +use x509_parser::prelude::FromDer; +use pingora_error::Result; + +pub fn get_organization_serial(cert: &[u8]) -> (Option, String) { + let serial = get_serial(cert).expect("Failed to get serial for certificate."); + (get_organization(cert), serial) +} + +pub fn get_serial(cert: &[u8]) -> Result { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + Ok(x509cert.raw_serial_as_string()) +} + +/// Return the organization associated with the X509 certificate. +pub fn get_organization(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert.subject.iter_organization() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} + +/// Return the organization unit associated with the X509 certificate. +pub fn get_organizational_unit(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert.subject.iter_organizational_unit() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} + +/// Return the organization unit associated with the X509 certificate. +pub fn get_not_after(cert: &[u8]) -> String { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert.validity.not_after.to_string() +} + +pub fn get_common_name(cert: &[u8]) -> Option { + let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) + .expect("Failed to parse certificate from DER format."); + x509cert.subject.iter_common_name() + .filter_map(|a| a.as_str().ok()) + .map(|a| a.to_string()) + .reduce(|cur, next| cur + &next) +} + diff --git a/pingora-error/src/lib.rs b/pingora-error/src/lib.rs index d8567633..935b2520 100644 --- a/pingora-error/src/lib.rs +++ b/pingora-error/src/lib.rs @@ -105,6 +105,7 @@ pub enum ErrorType { ConnectTimedout, ConnectRefused, ConnectNoRoute, + TLSWantX509Lookup, TLSHandshakeFailure, TLSHandshakeTimedout, InvalidCert, @@ -164,6 +165,7 @@ impl ErrorType { ErrorType::ConnectRefused => "ConnectRefused", ErrorType::ConnectNoRoute => "ConnectNoRoute", ErrorType::ConnectProxyFailure => "ConnectProxyFailure", + ErrorType::TLSWantX509Lookup => "TLSWantX509Lookup", ErrorType::TLSHandshakeFailure => "TLSHandshakeFailure", ErrorType::TLSHandshakeTimedout => "TLSHandshakeTimedout", ErrorType::InvalidCert => "InvalidCert", diff --git a/pingora-load-balancing/Cargo.toml b/pingora-load-balancing/Cargo.toml index 904e9f69..6cba1a8a 100644 --- a/pingora-load-balancing/Cargo.toml +++ b/pingora-load-balancing/Cargo.toml @@ -36,3 +36,4 @@ log = { workspace = true } default = ["openssl"] openssl = ["pingora-core/openssl"] boringssl = ["pingora-core/boringssl"] +rustls = ["pingora-core/rustls"] \ No newline at end of file diff --git a/pingora-proxy/Cargo.toml b/pingora-proxy/Cargo.toml index 65f7f677..eab96320 100644 --- a/pingora-proxy/Cargo.toml +++ b/pingora-proxy/Cargo.toml @@ -44,7 +44,7 @@ env_logger = "0.9" hyperlocal = "0.8" hyper = "0.14" tokio-tungstenite = "0.20.1" -pingora-load-balancing = { version = "0.3.0", path = "../pingora-load-balancing" } +pingora-load-balancing = { version = "0.3.0", path = "../pingora-load-balancing", default-features = false } prometheus = "0" futures-util = "0.3" serde = { version = "1.0", features = ["derive"] } @@ -55,3 +55,4 @@ serde_yaml = "0.8" default = ["openssl"] openssl = ["pingora-core/openssl", "pingora-cache/openssl"] boringssl = ["pingora-core/boringssl", "pingora-cache/boringssl"] +rustls = ["pingora-core/rustls", "pingora-cache/rustls"] \ No newline at end of file diff --git a/pingora-proxy/tests/test_basic.rs b/pingora-proxy/tests/test_basic.rs index 744f9b70..6647d8a7 100644 --- a/pingora-proxy/tests/test_basic.rs +++ b/pingora-proxy/tests/test_basic.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod utils; - -use hyper::{body::HttpBody, header::HeaderValue, Body, Client}; +use hyper::{Body, body::HttpBody, Client, header::HeaderValue}; use hyperlocal::{UnixClientExt, Uri}; use reqwest::{header, StatusCode}; use utils::server_utils::init; +mod utils; + fn is_specified_port(port: u16) -> bool { (1..65535).contains(&port) } @@ -73,7 +73,9 @@ async fn test_h2_to_h1() { .build() .unwrap(); - let res = client.get("https://127.0.0.1:6150").send().await.unwrap(); + let res = client.get("https://127.0.0.1:6150") + .header("sni", "openrusty.org") + .send().await.unwrap(); assert_eq!(res.status(), reqwest::StatusCode::OK); assert_eq!(res.version(), reqwest::Version::HTTP_2); @@ -112,6 +114,7 @@ async fn test_h2_to_h2() { let res = client .get("https://127.0.0.1:6150") + .header("sni", "openrusty.org") .header("x-h2", "true") .send() .await @@ -165,6 +168,7 @@ async fn test_h2c_to_h2c() { assert_eq!(body.as_ref(), b"Hello World!\n"); } +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_h2_to_h2_host_override() { init(); @@ -200,6 +204,7 @@ async fn test_h2_to_h2_upload() { let res = client .get("https://127.0.0.1:6150/echo") + .header("sni", "openrusty.org") .header("x-h2", "true") .body(payload) .send() @@ -223,6 +228,7 @@ async fn test_h2_to_h1_upload() { let res = client .get("https://127.0.0.1:6150/echo") + .header("sni", "openrusty.org") .body(payload) .send() .await @@ -288,7 +294,7 @@ async fn test_simple_proxy_uds_peer() { assert!(is_specified_port(sockaddr.port())); assert_eq!(headers["x-upstream-client-addr"], "unset"); // unnamed UDS - assert_eq!(headers["x-upstream-server-addr"], "/tmp/nginx-test.sock"); + assert_eq!(headers["x-upstream-server-addr"], "/tmp/pingora_nginx_test.sock"); let body = res.text().await.unwrap(); assert_eq!(body, "Hello World!\n"); @@ -419,6 +425,8 @@ async fn test_dropped_conn() { test_dropped_conn_post_body_over().await; } +// currently not supported with Rustls implementaiton +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_tls_no_verify() { init(); @@ -448,6 +456,8 @@ async fn test_tls_verify_sni_not_host() { assert_eq!(res.status(), StatusCode::OK); } +// currently not supported with Rustls implementaiton +#[cfg(not(feature = "rustls"))] #[tokio::test] async fn test_tls_none_verify_host() { init(); diff --git a/pingora-proxy/tests/utils/cert.rs b/pingora-proxy/tests/utils/cert.rs index 674a3ac7..ce47a874 100644 --- a/pingora-proxy/tests/utils/cert.rs +++ b/pingora-proxy/tests/utils/cert.rs @@ -13,35 +13,51 @@ // limitations under the License. use once_cell::sync::Lazy; -use pingora_core::tls::pkey::{PKey, Private}; -use pingora_core::tls::x509::X509; use std::fs; -pub static ROOT_CERT: Lazy = Lazy::new(|| load_cert("keys/root.crt")); -pub static ROOT_KEY: Lazy> = Lazy::new(|| load_key("keys/root.key")); -pub static INTERMEDIATE_CERT: Lazy = Lazy::new(|| load_cert("keys/intermediate.crt")); -pub static INTERMEDIATE_KEY: Lazy> = Lazy::new(|| load_key("keys/intermediate.key")); -pub static LEAF_CERT: Lazy = Lazy::new(|| load_cert("keys/leaf.crt")); -pub static LEAF2_CERT: Lazy = Lazy::new(|| load_cert("keys/leaf2.crt")); -pub static LEAF_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf.key")); -pub static LEAF2_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf2.key")); -pub static SERVER_CERT: Lazy = Lazy::new(|| load_cert("keys/server.crt")); -pub static SERVER_KEY: Lazy> = Lazy::new(|| load_key("keys/key.pem")); -pub static CURVE_521_TEST_KEY: Lazy> = +#[cfg(not(feature = "rustls"))] +use pingora_core::tls::{pkey::{PKey, Private}, x509::X509}; +#[cfg(feature = "rustls")] +use pingora_core::tls::{load_pem_file_ca, load_pem_file_private_key}; + +//pub static ROOT_CERT: Lazy = Lazy::new(|| load_cert("keys/root.crt")); +//pub static ROOT_KEY: Lazy> = Lazy::new(|| load_key("keys/root.key")); +pub static INTERMEDIATE_CERT: Lazy> = Lazy::new(|| load_cert("keys/intermediate.crt")); +//pub static INTERMEDIATE_KEY: Lazy> = Lazy::new(|| load_key("keys/intermediate.key")); +pub static LEAF_CERT: Lazy> = Lazy::new(|| load_cert("keys/leaf.crt")); +pub static LEAF2_CERT: Lazy> = Lazy::new(|| load_cert("keys/leaf2.crt")); +pub static LEAF_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf.key")); +pub static LEAF2_KEY: Lazy> = Lazy::new(|| load_key("keys/leaf2.key")); +//pub static SERVER_CERT: Lazy = Lazy::new(|| load_cert("keys/server.crt")); +//pub static SERVER_KEY: Lazy> = Lazy::new(|| load_key("keys/key.pem")); +pub static CURVE_521_TEST_KEY: Lazy> = Lazy::new(|| load_key("keys/curve_test.521.key.pem")); -pub static CURVE_521_TEST_CERT: Lazy = Lazy::new(|| load_cert("keys/curve_test.521.crt")); -pub static CURVE_384_TEST_KEY: Lazy> = +pub static CURVE_521_TEST_CERT: Lazy> = Lazy::new(|| load_cert("keys/curve_test.521.crt")); +pub static CURVE_384_TEST_KEY: Lazy> = Lazy::new(|| load_key("keys/curve_test.384.key.pem")); -pub static CURVE_384_TEST_CERT: Lazy = Lazy::new(|| load_cert("keys/curve_test.384.crt")); +pub static CURVE_384_TEST_CERT: Lazy> = Lazy::new(|| load_cert("keys/curve_test.384.crt")); -fn load_cert(path: &str) -> X509 { +#[cfg(not(feature = "rustls"))] +fn load_cert(path: &str) -> Vec { let path = format!("{}/{path}", super::conf_dir()); let cert_bytes = fs::read(path).unwrap(); - X509::from_pem(&cert_bytes).unwrap() + X509::from_pem(&cert_bytes).unwrap().to_der().unwrap() } - -fn load_key(path: &str) -> PKey { +#[cfg(not(feature = "rustls"))] +fn load_key(path: &str) -> Vec { let path = format!("{}/{path}", super::conf_dir()); let key_bytes = fs::read(path).unwrap(); - PKey::private_key_from_pem(&key_bytes).unwrap() + PKey::private_key_from_pem(&key_bytes).unwrap().private_key_to_der().unwrap() +} + +#[cfg(feature = "rustls")] +fn load_cert(path: &str) -> Vec { + let path = format!("{}/{path}", super::conf_dir()); + load_pem_file_ca(&path) +} + +#[cfg(feature = "rustls")] +fn load_key(path: &str) -> Vec { + let path = format!("{}/{path}", super::conf_dir()); + load_pem_file_private_key(&path) } diff --git a/pingora-proxy/tests/utils/conf/keys/README.md b/pingora-proxy/tests/utils/conf/keys/README.md index 13965cd6..1242ea85 100644 --- a/pingora-proxy/tests/utils/conf/keys/README.md +++ b/pingora-proxy/tests/utils/conf/keys/README.md @@ -16,3 +16,13 @@ openssl ecparam -genkey -name secp256r1 -noout -out test_key.pem openssl req -new -key test_key.pem -out test.csr openssl x509 -req -in test.csr -CA server.crt -CAkey key.pem -CAcreateserial -CAserial test.srl -out test.crt -days 3650 -sha256 ``` + +``` +openssl version +# OpenSSL 3.1.1 + +echo '[v3_req]' > openssl.cnf +openssl req -config openssl.cnf -new -x509 -key key.pem -out server_rustls.crt -days 3650 -sha256 \ + -subj '/C=US/ST=CA/L=San Francisco/O=Cloudflare, Inc/CN=openrusty.org' \ + -addext "subjectAltName=DNS:*.openrusty.org,DNS:openrusty.org,DNS:cat.com,DNS:dog.com" +``` \ No newline at end of file diff --git a/pingora-proxy/tests/utils/conf/keys/server.crt b/pingora-proxy/tests/utils/conf/keys/server.crt deleted file mode 100644 index afb2d1e0..00000000 --- a/pingora-proxy/tests/utils/conf/keys/server.crt +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB9zCCAZ2gAwIBAgIUMI7aLvTxyRFCHhw57hGt4U6yupcwCgYIKoZIzj0EAwIw -ZDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMRgwFgYDVQQKDA9DbG91ZGZsYXJlLCBJbmMxFjAUBgNVBAMMDW9wZW5ydXN0 -eS5vcmcwHhcNMjIwNDExMjExMzEzWhcNMzIwNDA4MjExMzEzWjBkMQswCQYDVQQG -EwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xGDAWBgNV -BAoMD0Nsb3VkZmxhcmUsIEluYzEWMBQGA1UEAwwNb3BlbnJ1c3R5Lm9yZzBZMBMG -ByqGSM49AgEGCCqGSM49AwEHA0IABNn/9RZtR48knaJD6tk9BdccaJfZ0hGEPn6B -SDXmlmJPhcTBqa4iUwW/ABpGvO3FpJcNWasrX2k+qZLq3g205MKjLTArMCkGA1Ud -EQQiMCCCDyoub3BlbnJ1c3R5Lm9yZ4INb3BlbnJ1c3R5Lm9yZzAKBggqhkjOPQQD -AgNIADBFAiAjISZ9aEKmobKGlT76idO740J6jPaX/hOrm41MLeg69AIhAJqKrSyz -wD/AAF5fR6tXmBqlnpQOmtxfdy13wDr4MT3h ------END CERTIFICATE----- diff --git a/pingora-proxy/tests/keys/server.crt b/pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.crt similarity index 100% rename from pingora-proxy/tests/keys/server.crt rename to pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.crt diff --git a/pingora-proxy/tests/utils/conf/keys/server.csr b/pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.csr similarity index 100% rename from pingora-proxy/tests/utils/conf/keys/server.csr rename to pingora-proxy/tests/utils/conf/keys/server_boringssl_openssl.csr diff --git a/pingora-proxy/tests/utils/conf/keys/server_rustls.crt b/pingora-proxy/tests/utils/conf/keys/server_rustls.crt new file mode 100644 index 00000000..c02a214b --- /dev/null +++ b/pingora-proxy/tests/utils/conf/keys/server_rustls.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICJzCCAc6gAwIBAgIUU+G0acG/uiMu1ZDSjlcoY4gH53QwCgYIKoZIzj0EAwIw +ZDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMRgwFgYDVQQKDA9DbG91ZGZsYXJlLCBJbmMxFjAUBgNVBAMMDW9wZW5ydXN0 +eS5vcmcwHhcNMjQwNzI0MTMzOTQ4WhcNMzQwNzIyMTMzOTQ4WjBkMQswCQYDVQQG +EwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xGDAWBgNV +BAoMD0Nsb3VkZmxhcmUsIEluYzEWMBQGA1UEAwwNb3BlbnJ1c3R5Lm9yZzBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABNn/9RZtR48knaJD6tk9BdccaJfZ0hGEPn6B +SDXmlmJPhcTBqa4iUwW/ABpGvO3FpJcNWasrX2k+qZLq3g205MKjXjBcMDsGA1Ud +EQQ0MDKCDyoub3BlbnJ1c3R5Lm9yZ4INb3BlbnJ1c3R5Lm9yZ4IHY2F0LmNvbYIH +ZG9nLmNvbTAdBgNVHQ4EFgQUnfYAFWyQnSN57IGokj7jcz8ChJQwCgYIKoZIzj0E +AwIDRwAwRAIgQr+Ly2cH04CncbnbhUf4hBl5frTp1pXgGnn8dYjd+UcCICuunEtp +H/a42/sVGBFvjS6FOFe6ZDs4oWBNEqQSw0S2 +-----END CERTIFICATE----- diff --git a/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf b/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf index a41a743d..fdc88a88 100644 --- a/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf +++ b/pingora-proxy/tests/utils/conf/origin/conf/nginx.conf @@ -6,7 +6,7 @@ error_log /dev/stdout; #error_log logs/error.log notice; #error_log logs/error.log info; -pid /tmp/mock_origin.pid; +pid /tmp/pingora_mock_origin.pid; master_process off; daemon off; @@ -85,7 +85,7 @@ http { listen 8001; listen [::]:8000; #listen 8443 ssl; - listen unix:/tmp/nginx-test.sock; + listen unix:/tmp/pingora_nginx_test.sock; listen 8443 ssl http2; server_name localhost; @@ -97,6 +97,9 @@ http { # for benchmark http2_max_requests 999999; + # increase max body size for /upload/ test + client_max_body_size 128m; + #charset koi8-r; #access_log logs/host.access.log main; @@ -285,7 +288,6 @@ http { } location /upload/ { - client_max_body_size 1G; content_by_lua_block { ngx.req.read_body() ngx.print(string.rep("A", 64)) diff --git a/pingora-proxy/tests/utils/mock_origin.rs b/pingora-proxy/tests/utils/mock_origin.rs index db84f8df..5d266096 100644 --- a/pingora-proxy/tests/utils/mock_origin.rs +++ b/pingora-proxy/tests/utils/mock_origin.rs @@ -15,13 +15,24 @@ use once_cell::sync::Lazy; use std::process; use std::{thread, time}; +use std::path::Path; pub static MOCK_ORIGIN: Lazy = Lazy::new(init); fn init() -> bool { + #[cfg(feature = "rustls")] + let src_cert_path = format!("{}/tests/utils/conf/keys/server_rustls.crt", env!("CARGO_MANIFEST_DIR")); + #[cfg(not(feature = "rustls"))] + let src_cert_path = format!("{}/tests/utils/conf/keys/server_boringssl_openssl.crt", env!("CARGO_MANIFEST_DIR")); + + let mut dst_cert_path = format!("{}/tests/keys/server.crt", env!("CARGO_MANIFEST_DIR")); + std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); + dst_cert_path = format!("{}/tests/utils/conf/keys/server.crt", env!("CARGO_MANIFEST_DIR")); + std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); + // TODO: figure out a way to kill openresty when exiting process::Command::new("pkill") - .args(["-F", "/tmp/mock_origin.pid"]) + .args(["-F", "/tmp/pingora_mock_origin.pid"]) .spawn() .unwrap(); let _origin = thread::spawn(|| { diff --git a/pingora-proxy/tests/utils/server_utils.rs b/pingora-proxy/tests/utils/server_utils.rs index ec1a9627..4dfdb9e3 100644 --- a/pingora-proxy/tests/utils/server_utils.rs +++ b/pingora-proxy/tests/utils/server_utils.rs @@ -32,7 +32,7 @@ use pingora_core::protocols::{l4::socket::SocketAddr, Digest}; use pingora_core::server::configuration::Opt; use pingora_core::services::Service; use pingora_core::upstreams::peer::HttpPeer; -use pingora_core::utils::CertKey; +use pingora_core::utils::tls::CertKey; use pingora_error::{Error, ErrorSource, Result}; use pingora_http::{RequestHeader, ResponseHeader}; use pingora_proxy::{ProxyHttp, Session}; @@ -281,7 +281,7 @@ impl ProxyHttp for ExampleProxyHttp { let req = session.req_header(); if req.headers.contains_key("x-uds-peer") { return Ok(Box::new(HttpPeer::new_uds( - "/tmp/nginx-test.sock", + "/tmp/pingora_nginx_test.sock", false, "".to_string(), )?)); diff --git a/pingora-rustls/Cargo.toml b/pingora-rustls/Cargo.toml new file mode 100644 index 00000000..f68d8da0 --- /dev/null +++ b/pingora-rustls/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "pingora-rustls" +version = "0.3.0" +authors = ["Harald Gutmann "] +license = "Apache-2.0" +edition = "2021" +repository = "https://github.com/cloudflare/pingora" +categories = ["asynchronous", "network-programming"] +keywords = ["async", "tls", "ssl", "pingora"] +description = """ +RusTLS async APIs for Pingora. +""" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "pingora_rustls" +path = "src/lib.rs" + +[dependencies] +log = "0.4.21" +ring = "0.17.8" +rustls = "0.23.10" +rustls-native-certs = "0.7.1" +rustls-pemfile = "2.1.2" +rustls-pki-types = "1.7.0" +tokio-rustls = "0.26.0" +no_debug = "3.1.0" + +[dev-dependencies] +tokio-test = "0.4.3" +tokio = { workspace = true, features = ["full"] } diff --git a/pingora-rustls/LICENSE b/pingora-rustls/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/pingora-rustls/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs new file mode 100644 index 00000000..bb52474e --- /dev/null +++ b/pingora-rustls/src/lib.rs @@ -0,0 +1,158 @@ +// Copyright 2024 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![warn(clippy::all)] + +use std::fs::File; +use std::io::BufReader; + +use log::{error, warn}; +pub use rustls::{ClientConfig, RootCertStore, ServerConfig, Stream, version}; +pub use rustls_native_certs::load_native_certs; +use rustls_pemfile::Item; +pub use rustls_pki_types::{CertificateDer, PrivateKeyDer, ServerName}; +pub use tokio_rustls::{Accept, Connect, TlsAcceptor, TlsConnector, TlsStream}; +pub use tokio_rustls::client::TlsStream as ClientTlsStream; +pub use tokio_rustls::server::TlsStream as ServerTlsStream; +pub use no_debug::{NoDebug, WithTypeInfo, Ellipses}; + +fn load_file(path: &String) -> BufReader { + let file = File::open(path).expect("io error"); + BufReader::new(file) +} +fn load_pem_file(path: &String) -> Result, std::io::Error> { + let iter: Vec = rustls_pemfile::read_all(&mut load_file(path)).filter_map(|f| { + if f.is_ok() { + Some(f.unwrap()) + } else { + let err = f.err().unwrap(); + warn!("Skipping PEM element in file \"{}\" due to error \"{}\"", path, err); + None + } + }).collect(); + Ok(iter) +} + +pub fn load_ca_file_into_store(path: &String, cert_store: &mut RootCertStore) { + let ca_file = load_pem_file(path); + match ca_file { + Ok(cas) => { + cas.into_iter().for_each(|pem_item| { + // only loading certificates, handling a CA file + match pem_item { + Item::X509Certificate(content) => { + match cert_store.add(content) { + Ok(_) => {} + Err(err) => { error!("{}", err) } + } + } + Item::Pkcs1Key(_) => {} + Item::Pkcs8Key(_) => {} + Item::Sec1Key(_) => {} + Item::Crl(_) => {} + Item::Csr(_) => {} + _ => {} + } + }); + } + Err(err) => { + error!("Failed to load configured ca file located at \"{}\", error: \"{}\"", path, err); + } + } +} + +pub fn load_platform_certs_incl_env_into_store(ca_certs: &mut RootCertStore) { + // this includes handling of ENV vars SSL_CERT_FILE & SSL_CERT_DIR + let native_platform_certs = load_native_certs(); + match native_platform_certs { + Ok(certs) => { + for cert in certs { + ca_certs.add(cert).unwrap(); + } + } + Err(err) => { + error!("Failed to load native platform ca-certificates: \"{:?}\". Continuing without ...", err); + } + } +} + +pub fn load_certs_key_file<'a>(cert: &String, key: &String) -> Option<(Vec>, PrivateKeyDer<'a>)> { + let certs_file = load_pem_file(cert) + .expect(format!("Failed to load configured cert file located at {}.", cert).as_str()); + let key_file = load_pem_file(key) + .expect(format!("Failed to load configured key file located at {}.", cert).as_str()); + + let mut certs: Vec> = vec![]; + certs_file.into_iter().for_each(|i| { + match i { + Item::X509Certificate(cert) => { + certs.push(cert) + } + _ => {} + } + }); + + let private_key = match key_file.into_iter().next()? { + Item::Pkcs1Key(key) => { + Some(PrivateKeyDer::from(key)) + } + Item::Pkcs8Key(key) => { + Some(PrivateKeyDer::from(key)) + } + Item::Sec1Key(key) => { + Some(PrivateKeyDer::from(key)) + } + _ => { None } + }; + + if certs.is_empty() || private_key.is_none() { + None + } else { + Some((certs, private_key?)) + } +} + +pub fn load_pem_file_ca(path: &String) -> Vec { + let mut reader = load_file(path); + let cas_file = rustls_pemfile::certs(&mut reader); + let ca = cas_file.into_iter().find_map(|pem_item| { + if let Ok(item) = pem_item { + Some(item) + } else { + None + } + }); + match ca { + None => { Vec::new() } + Some(ca) => { + ca.to_vec() + } + } +} + +pub fn load_pem_file_private_key(path: &String) -> Vec { + let key = rustls_pemfile::private_key(&mut load_file(path)); + if let Ok(key) = key { + if let Some(key) = key { + return key.secret_der().to_vec(); + } + } + Vec::new() +} + + +pub fn hash_certificate(cert: CertificateDer) -> Vec { + let hash = ring::digest::digest(&ring::digest::SHA256, cert.as_ref()); + hash.as_ref().to_vec() +} diff --git a/pingora/Cargo.toml b/pingora/Cargo.toml index 75baf1b1..f5b7d455 100644 --- a/pingora/Cargo.toml +++ b/pingora/Cargo.toml @@ -60,6 +60,12 @@ boringssl = [ "pingora-cache?/boringssl", "pingora-load-balancing?/boringssl", ] +rustls = [ + "pingora-core/rustls", + "pingora-proxy?/rustls", + "pingora-cache?/rustls", + "pingora-load-balancing?/rustls", +] proxy = ["pingora-proxy"] lb = ["pingora-load-balancing", "proxy"] cache = ["pingora-cache"] diff --git a/pingora/examples/server.rs b/pingora/examples/server.rs index a2a092e9..083e6a73 100644 --- a/pingora/examples/server.rs +++ b/pingora/examples/server.rs @@ -50,30 +50,35 @@ impl BackgroundService for ExampleBackgroundService { } } -use pingora::tls::pkey::{PKey, Private}; -use pingora::tls::x509::X509; -struct DynamicCert { - cert: X509, - key: PKey, -} +#[cfg(not(feature = "rustls"))] +mod boringssl_openssl { + use pingora::tls::pkey::{PKey, Private}; + use pingora::tls::x509::X509; + use super::*; + + pub(super) struct DynamicCert { + cert: X509, + key: PKey, + } -impl DynamicCert { - fn new(cert: &str, key: &str) -> Box { - let cert_bytes = std::fs::read(cert).unwrap(); - let cert = X509::from_pem(&cert_bytes).unwrap(); + impl DynamicCert { + pub(super) fn new(cert: &str, key: &str) -> Box { + let cert_bytes = std::fs::read(cert).unwrap(); + let cert = X509::from_pem(&cert_bytes).unwrap(); - let key_bytes = std::fs::read(key).unwrap(); - let key = PKey::private_key_from_pem(&key_bytes).unwrap(); - Box::new(DynamicCert { cert, key }) + let key_bytes = std::fs::read(key).unwrap(); + let key = PKey::private_key_from_pem(&key_bytes).unwrap(); + Box::new(DynamicCert { cert, key }) + } } -} -#[async_trait] -impl pingora::listeners::TlsAccept for DynamicCert { - async fn certificate_callback(&self, ssl: &mut pingora::tls::ssl::SslRef) { - use pingora::tls::ext; - ext::ssl_use_certificate(ssl, &self.cert).unwrap(); - ext::ssl_use_private_key(ssl, &self.key).unwrap(); + #[async_trait] + impl pingora::listeners::TlsAccept for DynamicCert { + async fn certificate_callback(&self, ssl: &mut pingora::tls::ssl::SslRef) { + use pingora::tls::ext; + ext::ssl_use_certificate(ssl, &self.cert).unwrap(); + ext::ssl_use_private_key(ssl, &self.key).unwrap(); + } } } @@ -132,12 +137,23 @@ pub fn main() { echo_service_http.add_tcp_with_settings("0.0.0.0:6145", options); echo_service_http.add_uds("/tmp/echo.sock", None); - let dynamic_cert = DynamicCert::new(&cert_path, &key_path); - let mut tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); - // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo - tls_settings - .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) - .unwrap(); + let mut tls_settings; + // NOTE: dynamic certificate callback is only supported with BoringSSL/OpenSSL + #[cfg(not(feature = "rustls"))] + { + use pingora_core::listeners::tls::NativeBuilder; + + let dynamic_cert = boringssl_openssl::DynamicCert::new(&cert_path, &key_path); + tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); + // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo + tls_settings.get_builder().native() + .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) + .unwrap(); + } + #[cfg(feature = "rustls")] + { + tls_settings = pingora::listeners::TlsSettings::intermediate(&cert_path, &key_path).unwrap(); + } tls_settings.enable_h2(); echo_service_http.add_tls_with_settings("0.0.0.0:6148", None, tls_settings); From 76b6a680c3158ae9dd28eb72144ef0cd5d6b6499 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Wed, 24 Jul 2024 18:44:53 +0200 Subject: [PATCH 02/10] Run cargo fmt --all --- pingora-core/src/connectors/mod.rs | 7 +- .../connectors/tls/boringssl_openssl/mod.rs | 24 +++--- pingora-core/src/connectors/tls/mod.rs | 28 +++--- pingora-core/src/connectors/tls/rustls/mod.rs | 78 ++++++++++------- pingora-core/src/lib.rs | 12 ++- pingora-core/src/listeners/mod.rs | 11 +-- .../listeners/tls/boringssl_openssl/mod.rs | 20 +++-- pingora-core/src/listeners/tls/mod.rs | 17 ++-- pingora-core/src/listeners/tls/rustls/mod.rs | 57 ++++++++----- pingora-core/src/protocols/l4/stream.rs | 5 +- .../protocols/tls/boringssl_openssl/client.rs | 21 ++--- .../protocols/tls/boringssl_openssl/mod.rs | 10 +-- .../protocols/tls/boringssl_openssl/server.rs | 25 +++--- .../protocols/tls/boringssl_openssl/stream.rs | 38 ++++----- pingora-core/src/protocols/tls/mod.rs | 30 +++---- .../src/protocols/tls/rustls/client.rs | 14 +-- pingora-core/src/protocols/tls/rustls/mod.rs | 68 +++++++-------- .../src/protocols/tls/rustls/server.rs | 26 +++--- .../src/protocols/tls/rustls/stream.rs | 75 ++++++++++------ pingora-core/src/protocols/tls/server.rs | 7 +- pingora-core/src/upstreams/mod.rs | 2 +- pingora-core/src/upstreams/peer.rs | 6 +- .../src/utils/tls/boringssl_openssl/mod.rs | 18 ++-- pingora-core/src/utils/tls/mod.rs | 12 ++- pingora-core/src/utils/tls/rustls/mod.rs | 15 ++-- pingora-proxy/tests/test_basic.rs | 14 ++- pingora-proxy/tests/utils/cert.rs | 12 ++- pingora-proxy/tests/utils/mock_origin.rs | 17 +++- pingora-rustls/src/lib.rs | 85 ++++++++++--------- pingora/examples/server.rs | 9 +- 30 files changed, 429 insertions(+), 334 deletions(-) diff --git a/pingora-core/src/connectors/mod.rs b/pingora-core/src/connectors/mod.rs index a0f0a980..b3078fdf 100644 --- a/pingora-core/src/connectors/mod.rs +++ b/pingora-core/src/connectors/mod.rs @@ -28,10 +28,10 @@ use offload::OffloadRuntime; use pingora_error::{ErrorType::*, OrErr, Result}; use pingora_pool::{ConnectionMeta, ConnectionPool}; -use crate::connectors::tls::{Connector, do_connect}; +use crate::connectors::tls::{do_connect, Connector}; use crate::protocols::Stream; use crate::server::configuration::ServerConf; -use crate::upstreams::peer::{ALPN, Peer}; +use crate::upstreams::peer::{Peer, ALPN}; pub mod http; mod l4; @@ -271,9 +271,6 @@ impl TransportConnector { } } - - - struct PreferredHttpVersion { // TODO: shard to avoid the global lock versions: RwLock>, // diff --git a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs index 48408a16..bd41a690 100644 --- a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs @@ -19,27 +19,27 @@ use std::sync::{Arc, Once}; use log::debug; -use pingora_error::{Error, OrErr, Result}; use pingora_error::ErrorType::{ConnectTimedout, InternalError}; +use pingora_error::{Error, OrErr, Result}; use crate::connectors::ConnectorOptions; use crate::listeners::ALPN; -use crate::protocols::IO; use crate::protocols::tls::boringssl_openssl::client::handshake; use crate::protocols::tls::TlsStream; +use crate::protocols::IO; use crate::tls::ext::{ add_host, clear_error_stack, ssl_add_chain_cert, ssl_set_groups_list, ssl_set_renegotiate_mode_freely, ssl_set_verify_cert_store, ssl_use_certificate, ssl_use_private_key, ssl_use_second_key_share, }; -use crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion}; #[cfg(feature = "boringssl")] use crate::tls::ssl::SslCurve; +use crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion}; use crate::tls::x509::store::X509StoreBuilder; use crate::upstreams::peer::Peer; use crate::utils::tls::boringssl_openssl::{der_to_private_key, der_to_x509}; -use super::{Connector, replace_leftmost_underscore, TlsConnectorContext}; +use super::{replace_leftmost_underscore, Connector, TlsConnectorContext}; const CIPHER_LIST: &str = "AES-128-GCM-SHA256\ :AES-256-GCM-SHA384\ @@ -103,7 +103,7 @@ impl TlsConnectorContext for TlsConnectorCtx { fn build_connector(options: Option) -> Connector where - Self: Sized + Self: Sized, { let mut builder = SslConnector::builder(SslMethod::tls()).unwrap(); // TODO: make these conf @@ -168,9 +168,9 @@ pub(super) async fn connect( alpn_override: Option, tls_ctx: &Arc, ) -> Result> - where - T: IO, - P: Peer + Send + Sync +where + T: IO, + P: Peer + Send + Sync, { let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); let mut ssl_conf = ctx.0.configure().unwrap(); @@ -193,11 +193,9 @@ pub(super) async fn connect( if let Some(key_pair) = peer.get_client_cert_key() { debug!("setting client cert and key"); let leaf = der_to_x509(&*key_pair.leaf())?; - ssl_use_certificate(&mut ssl_conf, &leaf) - .or_err(InternalError, "invalid client cert")?; + ssl_use_certificate(&mut ssl_conf, &leaf).or_err(InternalError, "invalid client cert")?; let key = der_to_private_key(&*key_pair.key())?; - ssl_use_private_key(&mut ssl_conf, &key) - .or_err(InternalError, "invalid client key")?; + ssl_use_private_key(&mut ssl_conf, &key).or_err(InternalError, "invalid client key")?; let intermediates = key_pair.intermediates(); if !intermediates.is_empty() { @@ -275,4 +273,4 @@ pub(super) async fn connect( }, None => connect_future.await, } -} \ No newline at end of file +} diff --git a/pingora-core/src/connectors/tls/mod.rs b/pingora-core/src/connectors/tls/mod.rs index 8bce8a95..01167850 100644 --- a/pingora-core/src/connectors/tls/mod.rs +++ b/pingora-core/src/connectors/tls/mod.rs @@ -16,8 +16,8 @@ use std::any::Any; use std::net::SocketAddr; use std::sync::Arc; -use pingora_error::{Error, Result}; use pingora_error::ErrorType::ConnectTimedout; +use pingora_error::{Error, Result}; use crate::connectors::l4::connect as l4_connect; #[cfg(not(feature = "rustls"))] @@ -29,7 +29,7 @@ use crate::connectors::tls::rustls::connect as tls_connect; #[cfg(feature = "rustls")] use crate::connectors::tls::rustls::TlsConnectorCtx; use crate::protocols::Stream; -use crate::upstreams::peer::{ALPN, Peer}; +use crate::upstreams::peer::{Peer, ALPN}; use super::ConnectorOptions; @@ -53,14 +53,15 @@ pub(crate) trait TlsConnectorContext { fn as_any(&self) -> &dyn Any; fn build_connector(options: Option) -> Connector - where Self: Sized; + where + Self: Sized; } pub(super) async fn do_connect( peer: &P, bind_to: Option, alpn_override: Option, - tls_ctx: &Arc + tls_ctx: &Arc, ) -> Result { // Create the future that does the connections, but don't evaluate it until // we decide if we need a timeout or not @@ -82,7 +83,7 @@ async fn do_connect_inner( peer: &P, bind_to: Option, alpn_override: Option, - tls_ctx: &Arc + tls_ctx: &Arc, ) -> Result { let stream = l4_connect(peer, bind_to).await?; if peer.tls() { @@ -138,17 +139,24 @@ mod tests { ]; for case in none_cases { - assert!(super::replace_leftmost_underscore(case).is_none(), "{}", case); + assert!( + super::replace_leftmost_underscore(case).is_none(), + "{}", + case + ); } assert_eq!( - Some("bb-b.some.com".to_string()), super::replace_leftmost_underscore("bb_b.some.com") + Some("bb-b.some.com".to_string()), + super::replace_leftmost_underscore("bb_b.some.com") ); assert_eq!( - Some("a-a-a.some.com".to_string()), super::replace_leftmost_underscore("a_a_a.some.com") + Some("a-a-a.some.com".to_string()), + super::replace_leftmost_underscore("a_a_a.some.com") ); assert_eq!( - Some("-.some.com".to_string()), super::replace_leftmost_underscore("_.some.com") + Some("-.some.com".to_string()), + super::replace_leftmost_underscore("_.some.com") ); } -} \ No newline at end of file +} diff --git a/pingora-core/src/connectors/tls/rustls/mod.rs b/pingora-core/src/connectors/tls/rustls/mod.rs index 88e04793..453c0ab7 100644 --- a/pingora-core/src/connectors/tls/rustls/mod.rs +++ b/pingora-core/src/connectors/tls/rustls/mod.rs @@ -19,29 +19,31 @@ use std::sync::Arc; use log::debug; -use pingora_error::{Error, OrErr, Result}; use pingora_error::ErrorType::{ConnectTimedout, InvalidCert}; -use pingora_rustls::{load_ca_file_into_store, load_certs_key_file, load_platform_certs_incl_env_into_store}; +use pingora_error::{Error, OrErr, Result}; +use pingora_rustls::version; use pingora_rustls::CertificateDer; use pingora_rustls::ClientConfig; use pingora_rustls::ClientConfig as RusTlsClientConfig; use pingora_rustls::PrivateKeyDer; use pingora_rustls::RootCertStore; use pingora_rustls::TlsConnector as RusTlsConnector; -use pingora_rustls::version; +use pingora_rustls::{ + load_ca_file_into_store, load_certs_key_file, load_platform_certs_incl_env_into_store, +}; use crate::connectors::ConnectorOptions; use crate::listeners::ALPN; -use crate::protocols::IO; use crate::protocols::tls::rustls::client::handshake; use crate::protocols::tls::TlsStream; +use crate::protocols::IO; use crate::upstreams::peer::Peer; -use super::{Connector, replace_leftmost_underscore, TlsConnectorContext}; +use super::{replace_leftmost_underscore, Connector, TlsConnectorContext}; pub(crate) struct TlsConnectorCtx { config: RusTlsClientConfig, - ca_certs: RootCertStore + ca_certs: RootCertStore, } impl TlsConnectorContext for TlsConnectorCtx { fn as_any(&self) -> &dyn Any { @@ -50,7 +52,7 @@ impl TlsConnectorContext for TlsConnectorCtx { fn build_connector(options: Option) -> Connector where - Self: Sized + Self: Sized, { // NOTE: Rustls only supports TLS 1.2 & 1.3 @@ -88,37 +90,35 @@ impl TlsConnectorContext for TlsConnectorCtx { let config = match certs_key { Some((certs, key)) => { match builder.with_client_auth_cert(certs.clone(), key.clone_key()) { - Ok(config) => { config } + Ok(config) => config, Err(err) => { // TODO: is there a viable alternative to the panic? // falling back to no client auth... does not seem to be reasonable. - panic!("{}", format!("Failed to configure client auth cert/key. Error: {}", err)); + panic!( + "{}", + format!("Failed to configure client auth cert/key. Error: {}", err) + ); } } } - None => { - builder.with_no_client_auth() - } + None => builder.with_no_client_auth(), }; Connector { - ctx: Arc::new(TlsConnectorCtx { - config, - ca_certs - }), + ctx: Arc::new(TlsConnectorCtx { config, ca_certs }), } } } -pub(super) async fn connect( +pub(super) async fn connect( stream: T, peer: &P, alpn_override: Option, - tls_ctx: &Arc + tls_ctx: &Arc, ) -> Result> where T: IO, - P: Peer + Send + Sync + P: Peer + Send + Sync, { let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); let mut config = ctx.config.clone(); @@ -129,7 +129,7 @@ where let key_pair = peer.get_client_cert_key(); let updated_config: Option = match key_pair { - None => { None } + None => None, Some(key_arc) => { debug!("setting client cert and key"); @@ -138,20 +138,34 @@ where cert_chain.push(key_arc.leaf().to_owned()); debug!("adding intermediate certificates to mTLS cert chain"); - key_arc.intermediates().to_owned().iter() + key_arc + .intermediates() + .to_owned() + .iter() .map(|i| i.to_vec()) .for_each(|i| cert_chain.push(i)); - let certs: Vec = cert_chain.into_iter() - .map(|c| c.as_slice().to_owned().into()).collect(); - let private_key: PrivateKeyDer = key_arc.key().as_slice().to_owned().try_into().unwrap(); - - let builder = - ClientConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) - .with_root_certificates(ctx.ca_certs.clone()); - - let updated_config = builder.with_client_auth_cert(certs, private_key) - .explain_err(InvalidCert, |e| format!("Failed to use peer cert/key to update Rustls config: {:?}",e))?; + let certs: Vec = cert_chain + .into_iter() + .map(|c| c.as_slice().to_owned().into()) + .collect(); + let private_key: PrivateKeyDer = + key_arc.key().as_slice().to_owned().try_into().unwrap(); + + let builder = ClientConfig::builder_with_protocol_versions(&vec![ + &version::TLS12, + &version::TLS13, + ]) + .with_root_certificates(ctx.ca_certs.clone()); + + let updated_config = builder + .with_client_auth_cert(certs, private_key) + .explain_err(InvalidCert, |e| { + format!( + "Failed to use peer cert/key to update Rustls config: {:?}", + e + ) + })?; Some(updated_config) } }; @@ -212,4 +226,4 @@ where }, None => connect_future.await, } -} \ No newline at end of file +} diff --git a/pingora-core/src/lib.rs b/pingora-core/src/lib.rs index 35c81393..a4d131f7 100644 --- a/pingora-core/src/lib.rs +++ b/pingora-core/src/lib.rs @@ -57,10 +57,18 @@ pub use pingora_error::{ErrorType::*, *}; #[cfg(all(not(feature = "rustls"), feature = "boringssl"))] pub use pingora_boringssl as tls; -#[cfg(all(not(feature = "rustls"), not(feature = "boringssl"), feature = "openssl"))] +#[cfg(all( + not(feature = "rustls"), + not(feature = "boringssl"), + feature = "openssl" +))] pub use pingora_openssl as tls; -#[cfg(all(not(feature = "boringssl"), not(feature = "openssl"), feature = "rustls"))] +#[cfg(all( + not(feature = "boringssl"), + not(feature = "openssl"), + feature = "rustls" +))] pub use pingora_rustls as tls; pub mod prelude { diff --git a/pingora-core/src/listeners/mod.rs b/pingora-core/src/listeners/mod.rs index 4637229a..9c5c69a3 100644 --- a/pingora-core/src/listeners/mod.rs +++ b/pingora-core/src/listeners/mod.rs @@ -19,15 +19,15 @@ use std::{fs::Permissions, sync::Arc}; use l4::{ListenerEndpoint, Stream as L4Stream}; pub use l4::{ServerAddress, TcpSocketOptions}; use pingora_error::Result; -pub use tls::{ALPN, TlsSettings}; use tls::Acceptor; +pub use tls::{TlsSettings, ALPN}; -use crate::protocols::{IO, Stream}; +pub use crate::protocols::tls::server::TlsAccept; #[cfg(not(feature = "rustls"))] use crate::protocols::tls::TlsStream as TlsStreamProvider; #[cfg(feature = "rustls")] use crate::protocols::tls::TlsStream as TlsStreamProvider; -pub use crate::protocols::tls::server::TlsAccept; +use crate::protocols::{Stream, IO}; use crate::server::ListenFds; mod l4; @@ -85,7 +85,8 @@ pub(crate) struct UninitializedStream { impl UninitializedStream { pub async fn handshake(self) -> Result { if let Some(tls) = self.tls { - let tls_stream : TlsStreamProvider> = tls.handshake(Box::new(self.l4)).await?; + let tls_stream: TlsStreamProvider> = + tls.handshake(Box::new(self.l4)).await?; Ok(Box::new(tls_stream)) } else { Ok(Box::new(self.l4)) @@ -187,7 +188,7 @@ impl Listeners { mod test { use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; - use tokio::time::{Duration, sleep}; + use tokio::time::{sleep, Duration}; use super::*; diff --git a/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs index 054b7505..c6b4b9d1 100644 --- a/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs @@ -14,12 +14,12 @@ //! BoringSSL & OpenSSL listener specific implementation -use core::any::Any; -use async_trait::async_trait; -use pingora_error::{ErrorType, OrErr, Result}; -use crate::listeners::{ALPN, TlsSettings}; use crate::listeners::tls::{NativeBuilder, TlsAcceptor, TlsAcceptorBuilder}; +use crate::listeners::{TlsSettings, ALPN}; use crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; +use async_trait::async_trait; +use core::any::Any; +use pingora_error::{ErrorType, OrErr, Result}; const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); struct TlsAcc(SslAcceptor); @@ -47,7 +47,9 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { } fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result - where Self: Sized { + where + Self: Sized, + { let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( TLS_CONF_ERR, "fail to create mozilla_intermediate_v5 Acceptor", @@ -64,7 +66,9 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { } fn acceptor_with_callbacks() -> Result - where Self: Sized { + where + Self: Sized, + { let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err( TLS_CONF_ERR, "fail to create mozilla_intermediate_v5 Acceptor", @@ -96,7 +100,7 @@ impl From for TlsSettings { mod alpn { use crate::protocols::ALPN; - use crate::tls::ssl::{AlpnError, select_next_proto, SslRef}; + use crate::tls::ssl::{select_next_proto, AlpnError, SslRef}; // A standard implementation provided by the SSL lib is used below @@ -120,4 +124,4 @@ mod alpn { _ => Err(AlpnError::ALERT_FATAL), // cannot agree } } -} \ No newline at end of file +} diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs index 08961caa..80cb5547 100644 --- a/pingora-core/src/listeners/tls/mod.rs +++ b/pingora-core/src/listeners/tls/mod.rs @@ -17,19 +17,19 @@ use std::any::Any; use async_trait::async_trait; use log::debug; -use pingora_error::Result; #[cfg(not(feature = "rustls"))] use crate::listeners::tls::boringssl_openssl::TlsAcceptorBuil; #[cfg(feature = "rustls")] use crate::listeners::tls::rustls::TlsAcceptorBuil; -use crate::protocols::IO; -pub use crate::protocols::tls::ALPN; #[cfg(not(feature = "rustls"))] use crate::protocols::tls::boringssl_openssl::server::{handshake, handshake_with_callback}; #[cfg(feature = "rustls")] use crate::protocols::tls::rustls::server::{handshake, handshake_with_callback}; use crate::protocols::tls::server::TlsAcceptCallbacks; use crate::protocols::tls::TlsStream; +pub use crate::protocols::tls::ALPN; +use crate::protocols::IO; +use pingora_error::Result; #[cfg(not(feature = "rustls"))] pub mod boringssl_openssl; @@ -56,9 +56,11 @@ pub trait TlsAcceptorBuilder: Any { fn build(self: Box) -> Box; fn set_alpn(&mut self, alpn: ALPN); fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result - where Self: Sized; + where + Self: Sized; fn acceptor_with_callbacks() -> Result - where Self: Sized; + where + Self: Sized; fn as_any(&mut self) -> &mut dyn Any; } @@ -111,7 +113,10 @@ impl TlsSettings { } impl Acceptor { - pub async fn handshake(&self, stream: Box) -> Result>> { + pub async fn handshake( + &self, + stream: Box, + ) -> Result>> { debug!("new tls session"); // TODO: be able to offload this handshake in a thread pool if let Some(cb) = self.callbacks.as_ref() { diff --git a/pingora-core/src/listeners/tls/rustls/mod.rs b/pingora-core/src/listeners/tls/rustls/mod.rs index 87103df8..debec796 100644 --- a/pingora-core/src/listeners/tls/rustls/mod.rs +++ b/pingora-core/src/listeners/tls/rustls/mod.rs @@ -19,23 +19,23 @@ use std::sync::Arc; use async_trait::async_trait; -use pingora_error::{Error, ErrorSource, ImmutStr, OrErr, Result}; use pingora_error::ErrorType::InternalError; -use pingora_rustls::{TlsAcceptor as RusTlsAcceptor, version}; +use pingora_error::{Error, ErrorSource, ImmutStr, OrErr, Result}; use pingora_rustls::load_certs_key_file; use pingora_rustls::ServerConfig; +use pingora_rustls::{version, TlsAcceptor as RusTlsAcceptor}; -use crate::listeners::ALPN; use crate::listeners::tls::{TlsAcceptor, TlsAcceptorBuilder}; +use crate::listeners::ALPN; pub(super) struct TlsAcceptorBuil { alpn_protocols: Option>>, cert_path: String, - key_path: String + key_path: String, } struct TlsAcc { - acceptor: RusTlsAcceptor + acceptor: RusTlsAcceptor, } #[async_trait] @@ -47,21 +47,29 @@ impl TlsAcceptor for TlsAcc { impl TlsAcceptorBuilder for TlsAcceptorBuil { fn build(self: Box) -> Box { - let (certs, key) = load_certs_key_file(&self.cert_path, &self.key_path) - .expect(format!("Failed to load provided certificates \"{}\" or key \"{}\".", self.cert_path, self.key_path).as_str()); + let (certs, key) = load_certs_key_file(&self.cert_path, &self.key_path).expect( + format!( + "Failed to load provided certificates \"{}\" or key \"{}\".", + self.cert_path, self.key_path + ) + .as_str(), + ); - let mut config = ServerConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) - .with_no_client_auth() - .with_single_cert(certs, key) - .explain_err(InternalError, |e| format!("Failed to create server listener config: {}", e)) - .unwrap(); + let mut config = + ServerConfig::builder_with_protocol_versions(&vec![&version::TLS12, &version::TLS13]) + .with_no_client_auth() + .with_single_cert(certs, key) + .explain_err(InternalError, |e| { + format!("Failed to create server listener config: {}", e) + }) + .unwrap(); if let Some(alpn_protocols) = self.alpn_protocols { config.alpn_protocols = alpn_protocols; } Box::new(TlsAcc { - acceptor: RusTlsAcceptor::from(Arc::new(config)) + acceptor: RusTlsAcceptor::from(Arc::new(config)), }) } fn set_alpn(&mut self, alpn: ALPN) { @@ -69,25 +77,32 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { } fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result - where Self: Sized { + where + Self: Sized, + { Ok(TlsAcceptorBuil { alpn_protocols: None, cert_path: cert_path.to_string(), - key_path: key_path.to_string() + key_path: key_path.to_string(), }) } fn acceptor_with_callbacks() -> Result - where Self: Sized { + where + Self: Sized, + { // TODO: verify if/how callback in handshake can be done using Rustls - Err(Error::create(InternalError, - ErrorSource::Internal, - Some(ImmutStr::from("Certificate callbacks are not supported with feature \"rustls\".")), - None + Err(Error::create( + InternalError, + ErrorSource::Internal, + Some(ImmutStr::from( + "Certificate callbacks are not supported with feature \"rustls\".", + )), + None, )) } fn as_any(&mut self) -> &mut dyn Any { self as &mut dyn Any } -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/l4/stream.rs b/pingora-core/src/protocols/l4/stream.rs index fa0a1685..70ecdb31 100644 --- a/pingora-core/src/protocols/l4/stream.rs +++ b/pingora-core/src/protocols/l4/stream.rs @@ -28,7 +28,10 @@ use tokio::net::{TcpStream, UnixStream}; use crate::protocols::l4::ext::{set_tcp_keepalive, TcpKeepalive}; use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, IO, Shutdown, SocketDigest, Ssl, TimingDigest, UniqueID}; +use crate::protocols::{ + GetProxyDigest, GetSocketDigest, GetTimingDigest, Shutdown, SocketDigest, Ssl, TimingDigest, + UniqueID, IO, +}; use crate::upstreams::peer::Tracer; #[derive(Debug)] diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs index 49d35231..7dbf96d2 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs @@ -16,8 +16,8 @@ use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use crate::protocols::IO; use crate::protocols::tls::boringssl_openssl::TlsStream; +use crate::protocols::IO; use crate::tls::ssl::ConnectConfiguration; /// Perform the TLS handshake for the given connection with the given configuration @@ -32,16 +32,13 @@ pub async fn handshake( let mut stream = TlsStream::new(ssl, io) .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; - stream.connect().await - .map_err(|e| { - let err_msg = format!("TLS connect() failed: {e}, SNI: {domain}"); - if let Some(context) = e.context { - Error::explain(e.etype, - format!("{}, {}", err_msg, context.as_str())) - } else { - Error::explain(e.etype, err_msg) - } - })?; + stream.connect().await.map_err(|e| { + let err_msg = format!("TLS connect() failed: {e}, SNI: {domain}"); + if let Some(context) = e.context { + Error::explain(e.etype, format!("{}, {}", err_msg, context.as_str())) + } else { + Error::explain(e.etype, err_msg) + } + })?; Ok(stream) } - diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs index 47b35a6f..796f11fc 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs @@ -21,11 +21,11 @@ use std::task::{Context, Poll}; use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; use pingora_error::ErrorType::TLSHandshakeFailure; -use pingora_error::{Result, OrErr}; +use pingora_error::{OrErr, Result}; -use crate::protocols::{ALPN, Ssl, UniqueID}; use crate::protocols::tls::boringssl_openssl::stream::InnerStream; use crate::protocols::tls::SslDigest; +use crate::protocols::{Ssl, UniqueID, ALPN}; use crate::tls::hash::MessageDigest; use crate::tls::ssl; use crate::tls::ssl::SslRef; @@ -33,13 +33,13 @@ use crate::utils::tls::boringssl_openssl::{get_x509_organization, get_x509_seria use super::TlsStream; -pub(super) mod stream; pub mod client; pub mod server; +pub(super) mod stream; impl TlsStream where - T: AsyncRead + AsyncWrite + Unpin + Send + T: AsyncRead + AsyncWrite + Unpin + Send, { /// Create a new TLS connection from the given `stream` /// @@ -168,4 +168,4 @@ impl SslDigest { cert_digest, } } -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs index 639bbcb9..621b63ef 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -19,13 +19,13 @@ use std::pin::Pin; use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncWrite}; -use pingora_error::{OrErr, Result}; use pingora_error::ErrorType::{TLSHandshakeFailure, TLSWantX509Lookup}; +use pingora_error::{OrErr, Result}; use crate::listeners::tls::Acceptor; -use crate::protocols::{IO, Ssl}; use crate::protocols::tls::boringssl_openssl::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; +use crate::protocols::{Ssl, IO}; use crate::tls::ext; use crate::tls::ext::ssl_from_acceptor; use crate::tls::ssl::SslAcceptor; @@ -59,14 +59,17 @@ impl ResumableAccept for TlsStream } fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { - let ssl_acceptor = acceptor.inner().downcast_ref::().unwrap(); + let ssl_acceptor = acceptor.inner().downcast_ref::().unwrap(); let ssl = ssl_from_acceptor(ssl_acceptor) .explain_err(TLSHandshakeFailure, |e| format!("ssl_acceptor error: {e}"))?; TlsStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(acceptor: &Acceptor, io: Box) -> Result>> { +pub async fn handshake( + acceptor: &Acceptor, + io: Box, +) -> Result>> { let mut stream = prepare_tls_stream(acceptor, io)?; stream .accept() @@ -82,9 +85,7 @@ pub async fn handshake_with_callback( callbacks: &TlsAcceptCallbacks, ) -> pingora_error::Result>> { let mut tls_stream = prepare_tls_stream(acceptor, io)?; - let done = Pin::new(&mut tls_stream) - .start_accept() - .await?; + let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { // safety: we do hold a mut ref of tls_stream let ssl_mut = unsafe { ext::ssl_mut(tls_stream.0.ssl()) }; @@ -101,12 +102,12 @@ pub async fn handshake_with_callback( #[tokio::test] async fn test_async_cert() { - use crate::tls::ssl; - use tokio::io::AsyncReadExt; - use crate::tls::ssl::SslRef; - use crate::listeners::TlsAccept; use crate::listeners::tls::TlsSettings; + use crate::listeners::TlsAccept; use crate::protocols::tls::server::TlsAcceptCallbacks; + use crate::tls::ssl; + use crate::tls::ssl::SslRef; + use tokio::io::AsyncReadExt; struct Callback; #[async_trait] @@ -148,4 +149,4 @@ async fn test_async_cert() { let acceptor = TlsSettings::with_callbacks(cb).unwrap().build(); acceptor.handshake(Box::new(server)).await.unwrap(); -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs index 9a3a07c9..ef416816 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -14,24 +14,24 @@ //! BoringSSL & OpenSSL TLS stream specific implementation -use std::pin::Pin; -use std::sync::Arc; use async_trait::async_trait; use log::warn; +use std::pin::Pin; +use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use pingora_error::{Error, ErrorType::*, OrErr, Result}; use crate::listeners::ALPN; -use crate::protocols::{GetProxyDigest, GetTimingDigest}; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::tls::InnerTlsStream; use crate::protocols::tls::SslDigest; -use crate::tls::tokio_ssl::SslStream; +use crate::protocols::{GetProxyDigest, GetTimingDigest}; use crate::tls::error::ErrorStack; use crate::tls::ext; use crate::tls::tokio_ssl; +use crate::tls::tokio_ssl::SslStream; use crate::tls::{ssl, ssl::SslRef, ssl_sys::X509_V_ERR_INVALID_CALL}; #[derive(Debug)] @@ -64,12 +64,8 @@ impl InnerTlsStream for InnerStream async fn connect(&mut self) -> Result<()> { Self::clear_error(); match Pin::new(&mut self.0).connect().await { - Ok(_) => { - Ok(()) - } - Err(err) => { - self.transform_ssl_error(err) - } + Ok(_) => Ok(()), + Err(err) => self.transform_ssl_error(err), } } @@ -77,12 +73,8 @@ impl InnerTlsStream for InnerStream async fn accept(&mut self) -> Result<()> { Self::clear_error(); match Pin::new(&mut self.0).accept().await { - Ok(_) => { - Ok(()) - } - Err(err) => { - self.transform_ssl_error(err) - } + Ok(_) => Ok(()), + Err(err) => self.transform_ssl_error(err), } } @@ -136,8 +128,8 @@ impl InnerStream { } impl GetSocketDigest for InnerStream - where - S: GetSocketDigest, +where + S: GetSocketDigest, { fn get_socket_digest(&self) -> Option> { self.0.get_ref().get_socket_digest() @@ -148,8 +140,8 @@ impl GetSocketDigest for InnerStream } impl GetTimingDigest for InnerStream - where - S: GetTimingDigest, +where + S: GetTimingDigest, { fn get_timing_digest(&self) -> Vec> { self.0.get_ref().get_timing_digest() @@ -157,10 +149,10 @@ impl GetTimingDigest for InnerStream } impl GetProxyDigest for InnerStream - where - S: GetProxyDigest, +where + S: GetProxyDigest, { fn get_proxy_digest(&self) -> Option> { self.0.get_ref().get_proxy_digest() } -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs index 77203f81..f9b1c06d 100644 --- a/pingora-core/src/protocols/tls/mod.rs +++ b/pingora-core/src/protocols/tls/mod.rs @@ -14,29 +14,30 @@ //! The TLS layer implementations +use async_trait::async_trait; +use pingora_error::Result; use std::ops::{Deref, DerefMut}; use std::sync::Arc; use std::time::{Duration, SystemTime}; -use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncWrite}; -use pingora_error::Result; -use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, IO, SocketDigest, UniqueID}; use crate::protocols::digest::TimingDigest; use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::{ + GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest, UniqueID, IO, +}; -pub mod server; #[cfg(not(feature = "rustls"))] pub(crate) mod boringssl_openssl; #[cfg(feature = "rustls")] pub(crate) mod rustls; +pub mod server; #[cfg(not(feature = "rustls"))] use boringssl_openssl::stream::InnerStream; #[cfg(feature = "rustls")] use rustls::stream::InnerStream; - /// The TLS connection #[derive(Debug)] pub struct TlsStream { @@ -57,9 +58,7 @@ pub trait InnerTlsStream { fn selected_alpn_proto(&mut self) -> Option; } - -impl GetSocketDigest for Box -{ +impl GetSocketDigest for Box { fn get_socket_digest(&self) -> Option> { (**self).get_socket_digest() } @@ -68,28 +67,24 @@ impl GetSocketDigest for Box } } -impl GetTimingDigest for Box -{ +impl GetTimingDigest for Box { fn get_timing_digest(&self) -> Vec> { vec![] } } -impl GetProxyDigest for Box -{ +impl GetProxyDigest for Box { fn get_proxy_digest(&self) -> Option> { (**self).get_proxy_digest() } } -impl UniqueID for Box -{ +impl UniqueID for Box { fn id(&self) -> i32 { (**self).id() } } - /// The protocol for Application-Layer Protocol Negotiation #[derive(Hash, Clone, Debug)] pub enum ALPN { @@ -155,7 +150,7 @@ impl ALPN { match self { ALPN::H1 => vec![b"http/1.1".to_vec()], ALPN::H2 => vec![b"h2".to_vec()], - ALPN::H2H1 => vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ALPN::H2H1 => vec![b"h2".to_vec(), b"http/1.1".to_vec()], } } @@ -183,7 +178,6 @@ pub struct SslDigest { pub cert_digest: Vec, } - impl GetSocketDigest for TlsStream where S: GetSocketDigest, @@ -231,7 +225,7 @@ impl TlsStream { impl TlsStream where - T: AsyncRead + AsyncWrite + Unpin + Send + T: AsyncRead + AsyncWrite + Unpin + Send, { /// Connect to the remote TLS server as a client pub(crate) async fn connect(&mut self) -> Result<()> { diff --git a/pingora-core/src/protocols/tls/rustls/client.rs b/pingora-core/src/protocols/tls/rustls/client.rs index 9ed65ad3..bb9edc27 100644 --- a/pingora-core/src/protocols/tls/rustls/client.rs +++ b/pingora-core/src/protocols/tls/rustls/client.rs @@ -14,11 +14,11 @@ //! Rustls TLS client specific implementation -use pingora_error::{Error, OrErr, Result}; +use crate::protocols::tls::rustls::TlsStream; +use crate::protocols::IO; use pingora_error::ErrorType::TLSHandshakeFailure; +use pingora_error::{Error, OrErr, Result}; use pingora_rustls::TlsConnector; -use crate::protocols::IO; -use crate::protocols::tls::rustls::TlsStream; // Perform the TLS handshake for the given connection with the given configuration pub async fn handshake( @@ -26,8 +26,11 @@ pub async fn handshake( domain: &str, io: S, ) -> Result> { - let mut stream = TlsStream::from_connector(connector, domain, io).await - .explain_err(TLSHandshakeFailure, |e| format!("tip: tls stream error: {e}"))?; + let mut stream = TlsStream::from_connector(connector, domain, io) + .await + .explain_err(TLSHandshakeFailure, |e| { + format!("tip: tls stream error: {e}") + })?; let handshake_result = stream.connect().await; match handshake_result { @@ -38,4 +41,3 @@ pub async fn handshake( } } } - diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs index 5d59603b..4dbae68e 100644 --- a/pingora-core/src/protocols/tls/rustls/mod.rs +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -18,19 +18,19 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; -use x509_parser::nom::AsBytes; -use pingora_error::{OrErr, Result}; use pingora_error::ErrorType::{InternalError, TLSHandshakeFailure}; -use pingora_rustls::{hash_certificate, ServerName, TlsConnector}; +use pingora_error::{OrErr, Result}; use pingora_rustls::TlsStream as RusTlsStream; +use pingora_rustls::{hash_certificate, ServerName, TlsConnector}; +use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; +use x509_parser::nom::AsBytes; -use crate::utils::tls::rustls::{get_organization_serial}; +use crate::utils::tls::rustls::get_organization_serial; -use crate::listeners::tls::{Acceptor}; -use crate::protocols::{ALPN, Ssl, UniqueID}; +use crate::listeners::tls::Acceptor; use crate::protocols::tls::rustls::stream::InnerStream; use crate::protocols::tls::SslDigest; +use crate::protocols::{Ssl, UniqueID, ALPN}; use super::TlsStream; @@ -49,10 +49,13 @@ where pub async fn from_connector(connector: &TlsConnector, domain: &str, stream: T) -> Result { let server = ServerName::try_from(domain) .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) - .explain_err(InternalError, |e| format!("failed to parse domain: {}, error: {}", domain, e))? + .explain_err(InternalError, |e| { + format!("failed to parse domain: {}, error: {}", domain, e) + })? .to_owned(); - let tls = InnerStream::from_connector(connector, server, stream).await + let tls = InnerStream::from_connector(connector, server, stream) + .await .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; Ok(TlsStream { @@ -67,7 +70,8 @@ where /// Using RustTLS the stream is only returned after the handshake. /// The caller does therefor not need to perform [`Self::accept()`]. pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { - let tls = InnerStream::from_acceptor(acceptor, stream).await + let tls = InnerStream::from_acceptor(acceptor, stream) + .await .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; Ok(TlsStream { @@ -143,10 +147,8 @@ impl Ssl for TlsStream { if let Some(stream) = st { let proto = stream.get_ref().1.alpn_protocol(); match proto { - None => { None } - Some(raw) => { - ALPN::from_wire_selected(raw) - } + None => None, + Some(raw) => ALPN::from_wire_selected(raw), } } else { None @@ -163,44 +165,36 @@ impl SslDigest { let peer_certificates = session.peer_certificates(); let cipher = match cipher_suite { - Some(suite) => { - match suite.suite().as_str() { - Some(suite_str) => suite_str, - None => "", - } + Some(suite) => match suite.suite().as_str() { + Some(suite_str) => suite_str, + None => "", }, None => "", }; let version = match protocol { - Some(proto) => { - match proto.as_str() { - Some(ver) => ver, - None => "", - } + Some(proto) => match proto.as_str() { + Some(ver) => ver, + None => "", }, None => "", }; let cert_digest = match peer_certificates { - Some(certs) => { - match certs.first() { - Some(cert) => hash_certificate(cert.clone()), - None => vec![], - } + Some(certs) => match certs.first() { + Some(cert) => hash_certificate(cert.clone()), + None => vec![], }, None => vec![], }; let (organization, serial_number) = match peer_certificates { - Some(certs) => { - match certs.first() { - Some(cert) => { - let (organization, serial) = get_organization_serial(cert.as_bytes()); - (organization, Some(serial)) - }, - None => (None, None), + Some(certs) => match certs.first() { + Some(cert) => { + let (organization, serial) = get_organization_serial(cert.as_bytes()); + (organization, Some(serial)) } + None => (None, None), }, None => (None, None), }; @@ -213,4 +207,4 @@ impl SslDigest { cert_digest, } } -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs index ad67fa58..83a9d7df 100644 --- a/pingora-core/src/protocols/tls/rustls/server.rs +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -14,15 +14,15 @@ //! Rustls TLS server specific implementation -use std::pin::Pin; -use async_trait::async_trait; -use log::warn; -use tokio::io::{AsyncRead, AsyncWrite}; +use crate::listeners::tls::Acceptor; use crate::protocols::tls::rustls::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; -use pingora_error::{ErrorType::*, OrErr, Result}; use crate::protocols::IO; -use crate::listeners::tls::Acceptor; +use async_trait::async_trait; +use log::warn; +use pingora_error::{ErrorType::*, OrErr, Result}; +use std::pin::Pin; +use tokio::io::{AsyncRead, AsyncWrite}; #[async_trait] impl ResumableAccept for TlsStream { @@ -49,12 +49,16 @@ impl ResumableAccept for TlsStream } async fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { - TlsStream::from_acceptor(acceptor, io).await + TlsStream::from_acceptor(acceptor, io) + .await .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(acceptor: &Acceptor, io: Box) -> Result>> { +pub async fn handshake( + acceptor: &Acceptor, + io: Box, +) -> Result>> { let mut stream = prepare_tls_stream(acceptor, io).await?; stream .accept() @@ -71,9 +75,7 @@ pub async fn handshake_with_callback( _callbacks: &TlsAcceptCallbacks, ) -> Result>> { let mut tls_stream = prepare_tls_stream(acceptor, io).await?; - let done = Pin::new(&mut tls_stream) - .start_accept() - .await?; + let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { // TODO: verify if/how callback in handshake can be done using Rustls warn!("Callacks are not supported with feature \"rustls\"."); @@ -92,4 +94,4 @@ pub async fn handshake_with_callback( #[tokio::test] async fn test_async_cert() { todo!("callback support and test for Rustls") -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index 4c3556d5..349e056a 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -12,24 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Debug; -use std::sync::Arc; use async_trait::async_trait; -use tokio::io::{AsyncRead, AsyncWrite}; -use pingora_error::{Error, ImmutStr, OrErr, Result}; use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; -use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; +use pingora_error::{Error, ImmutStr, OrErr, Result}; +use pingora_rustls::NoDebug; use pingora_rustls::TlsAcceptor as RusTlsAcceptor; use pingora_rustls::TlsStream as RusTlsStream; -use pingora_rustls::NoDebug; +use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; +use std::fmt::Debug; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; -use crate::listeners::ALPN; use crate::listeners::tls::Acceptor; -use crate::protocols::{GetProxyDigest, GetTimingDigest}; +use crate::listeners::ALPN; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::tls::InnerTlsStream; use crate::protocols::tls::SslDigest; +use crate::protocols::{GetProxyDigest, GetTimingDigest}; #[derive(Debug)] pub struct InnerStream { @@ -43,7 +43,11 @@ impl InnerStream { /// /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS /// handshake after. - pub(crate) async fn from_connector(connector: &TlsConnector, server: ServerName<'_>, stream: T) -> Result { + pub(crate) async fn from_connector( + connector: &TlsConnector, + server: ServerName<'_>, + stream: T, + ) -> Result { let connect = connector.connect(server.to_owned(), stream); Ok(InnerStream { accept: None.into(), @@ -53,7 +57,7 @@ impl InnerStream { } pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { - let tls_acceptor = acceptor.inner().downcast_ref::().unwrap(); + let tls_acceptor = acceptor.inner().downcast_ref::().unwrap(); let accept = tls_acceptor.accept(stream); Ok(InnerStream { @@ -71,14 +75,18 @@ impl InnerTlsStream for InnerStream let connect = &mut (*self.connect); if let Some(ref mut connect) = connect { - let stream = connect.await + let stream = connect + .await .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; self.stream = Some(RusTlsStream::Client(stream)); self.connect = None.into(); Ok(()) } else { - Err(Error::explain(ConnectError, ImmutStr::from("TLS connect not available to perform handshake."))) + Err(Error::explain( + ConnectError, + ImmutStr::from("TLS connect not available to perform handshake."), + )) } } @@ -88,14 +96,18 @@ impl InnerTlsStream for InnerStream let accept = &mut (*self.accept); if let Some(ref mut accept) = accept { - let stream = accept.await + let stream = accept + .await .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; self.stream = Some(RusTlsStream::Server(stream)); self.connect = None.into(); Ok(()) } else { - Err(Error::explain(AcceptError, ImmutStr::from("TLS accept not available to perform handshake."))) + Err(Error::explain( + AcceptError, + ImmutStr::from("TLS accept not available to perform handshake."), + )) } } @@ -107,10 +119,8 @@ impl InnerTlsStream for InnerStream if let Some(stream) = self.stream.as_ref() { let proto = stream.get_ref().1.alpn_protocol(); match proto { - None => { None } - Some(raw) => { - ALPN::from_wire_selected(raw) - } + None => None, + Some(raw) => ALPN::from_wire_selected(raw), } } else { None @@ -118,10 +128,9 @@ impl InnerTlsStream for InnerStream } } - impl GetSocketDigest for InnerStream - where - S: GetSocketDigest, +where + S: GetSocketDigest, { fn get_socket_digest(&self) -> Option> { if let Some(stream) = self.stream.as_ref() { @@ -131,22 +140,32 @@ impl GetSocketDigest for InnerStream } } fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - self.stream.as_mut().unwrap().get_mut().0.set_socket_digest(socket_digest) + self.stream + .as_mut() + .unwrap() + .get_mut() + .0 + .set_socket_digest(socket_digest) } } impl GetTimingDigest for InnerStream - where - S: GetTimingDigest, +where + S: GetTimingDigest, { fn get_timing_digest(&self) -> Vec> { - self.stream.as_ref().unwrap().get_ref().0.get_timing_digest() + self.stream + .as_ref() + .unwrap() + .get_ref() + .0 + .get_timing_digest() } } impl GetProxyDigest for InnerStream - where - S: GetProxyDigest, +where + S: GetProxyDigest, { fn get_proxy_digest(&self) -> Option> { if let Some(stream) = self.stream.as_ref() { @@ -155,4 +174,4 @@ impl GetProxyDigest for InnerStream None } } -} \ No newline at end of file +} diff --git a/pingora-core/src/protocols/tls/server.rs b/pingora-core/src/protocols/tls/server.rs index 948d40fd..c0834af4 100644 --- a/pingora-core/src/protocols/tls/server.rs +++ b/pingora-core/src/protocols/tls/server.rs @@ -22,8 +22,8 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use pingora_error::Result; -use crate::protocols::{IO, Shutdown}; use crate::protocols::tls::TlsStream; +use crate::protocols::{Shutdown, IO}; #[cfg(not(feature = "rustls"))] use crate::tls::ssl::SslRef; @@ -58,8 +58,7 @@ where } #[async_trait] -impl Shutdown for Box -{ +impl Shutdown for Box { async fn shutdown(&mut self) { match ::shutdown(self).await { Ok(()) => {} @@ -85,4 +84,4 @@ pub trait ResumableAccept { /// /// This function should be called after the certificate is provided. async fn resume_accept(self: Pin<&mut Self>) -> Result<()>; -} \ No newline at end of file +} diff --git a/pingora-core/src/upstreams/mod.rs b/pingora-core/src/upstreams/mod.rs index 663a4973..7352b615 100644 --- a/pingora-core/src/upstreams/mod.rs +++ b/pingora-core/src/upstreams/mod.rs @@ -14,4 +14,4 @@ //! The interface to connect to a remote server -pub mod peer; \ No newline at end of file +pub mod peer; diff --git a/pingora-core/src/upstreams/peer.rs b/pingora-core/src/upstreams/peer.rs index 340311a2..848c47a9 100644 --- a/pingora-core/src/upstreams/peer.rs +++ b/pingora-core/src/upstreams/peer.rs @@ -32,11 +32,11 @@ use std::time::Duration; use crate::protocols::l4::socket::SocketAddr; use crate::protocols::ConnFdReusable; use crate::protocols::TcpKeepalive; -use crate::utils::tls::CertKey; #[cfg(not(feature = "rustls"))] -use crate::utils::tls::boringssl_openssl::{get_organizational_unit, get_not_after}; +use crate::utils::tls::boringssl_openssl::{get_not_after, get_organizational_unit}; #[cfg(feature = "rustls")] -use crate::utils::tls::rustls::{get_organizational_unit, get_not_after}; +use crate::utils::tls::rustls::{get_not_after, get_organizational_unit}; +use crate::utils::tls::CertKey; pub use crate::protocols::tls::ALPN; diff --git a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs index a6b68667..9909fea9 100644 --- a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs @@ -87,13 +87,21 @@ pub fn get_x509_serial(cert: &X509) -> pingora_error::Result { } pub fn der_to_x509(ca: &[u8]) -> pingora_error::Result { - let cert = X509::from_der(&*ca) - .explain_err(InvalidCert, |e| format!("Failed to convert ca certificate in DER form to X509 cert. Error: {:?}", e))?; + let cert = X509::from_der(&*ca).explain_err(InvalidCert, |e| { + format!( + "Failed to convert ca certificate in DER form to X509 cert. Error: {:?}", + e + ) + })?; Ok(cert) } pub fn der_to_private_key(key: &[u8]) -> pingora_error::Result> { - let key = PKey::private_key_from_der(key) - .explain_err(InternalError, |e| format!("Failed to convert private key in DER form to Pkey. Error: {:?}", e))?; + let key = PKey::private_key_from_der(key).explain_err(InternalError, |e| { + format!( + "Failed to convert private key in DER form to Pkey. Error: {:?}", + e + ) + })?; Ok(key) -} \ No newline at end of file +} diff --git a/pingora-core/src/utils/tls/mod.rs b/pingora-core/src/utils/tls/mod.rs index bab25a19..fc4447ad 100644 --- a/pingora-core/src/utils/tls/mod.rs +++ b/pingora-core/src/utils/tls/mod.rs @@ -20,11 +20,15 @@ pub mod boringssl_openssl; #[cfg(feature = "rustls")] pub mod rustls; -use std::hash::{Hash, Hasher}; #[cfg(not(feature = "rustls"))] -use boringssl_openssl::{get_organization, get_serial, get_common_name, get_organizational_unit, get_not_after}; +use boringssl_openssl::{ + get_common_name, get_not_after, get_organization, get_organizational_unit, get_serial, +}; #[cfg(feature = "rustls")] -use rustls::{get_organization, get_serial, get_common_name, get_organizational_unit, get_not_after}; +use rustls::{ + get_common_name, get_not_after, get_organization, get_organizational_unit, get_serial, +}; +use std::hash::{Hash, Hasher}; /// This type contains a list of one or more certificates and an associated private key. The leaf /// certificate should always be first. The certificates and keys are stored in Vec DER encoded @@ -105,4 +109,4 @@ impl Hash for CertKey { } } } -} \ No newline at end of file +} diff --git a/pingora-core/src/utils/tls/rustls/mod.rs b/pingora-core/src/utils/tls/rustls/mod.rs index 7a857858..41520d79 100644 --- a/pingora-core/src/utils/tls/rustls/mod.rs +++ b/pingora-core/src/utils/tls/rustls/mod.rs @@ -14,8 +14,8 @@ //! This module contains various helpers that make it easier to work with X509 certificates. -use x509_parser::prelude::FromDer; use pingora_error::Result; +use x509_parser::prelude::FromDer; pub fn get_organization_serial(cert: &[u8]) -> (Option, String) { let serial = get_serial(cert).expect("Failed to get serial for certificate."); @@ -32,7 +32,9 @@ pub fn get_serial(cert: &[u8]) -> Result { pub fn get_organization(cert: &[u8]) -> Option { let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) .expect("Failed to parse certificate from DER format."); - x509cert.subject.iter_organization() + x509cert + .subject + .iter_organization() .filter_map(|a| a.as_str().ok()) .map(|a| a.to_string()) .reduce(|cur, next| cur + &next) @@ -42,7 +44,9 @@ pub fn get_organization(cert: &[u8]) -> Option { pub fn get_organizational_unit(cert: &[u8]) -> Option { let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) .expect("Failed to parse certificate from DER format."); - x509cert.subject.iter_organizational_unit() + x509cert + .subject + .iter_organizational_unit() .filter_map(|a| a.as_str().ok()) .map(|a| a.to_string()) .reduce(|cur, next| cur + &next) @@ -58,9 +62,10 @@ pub fn get_not_after(cert: &[u8]) -> String { pub fn get_common_name(cert: &[u8]) -> Option { let (_, x509cert) = x509_parser::certificate::X509Certificate::from_der(cert) .expect("Failed to parse certificate from DER format."); - x509cert.subject.iter_common_name() + x509cert + .subject + .iter_common_name() .filter_map(|a| a.as_str().ok()) .map(|a| a.to_string()) .reduce(|cur, next| cur + &next) } - diff --git a/pingora-proxy/tests/test_basic.rs b/pingora-proxy/tests/test_basic.rs index 6647d8a7..34e5933a 100644 --- a/pingora-proxy/tests/test_basic.rs +++ b/pingora-proxy/tests/test_basic.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hyper::{Body, body::HttpBody, Client, header::HeaderValue}; +use hyper::{body::HttpBody, header::HeaderValue, Body, Client}; use hyperlocal::{UnixClientExt, Uri}; use reqwest::{header, StatusCode}; @@ -73,9 +73,12 @@ async fn test_h2_to_h1() { .build() .unwrap(); - let res = client.get("https://127.0.0.1:6150") + let res = client + .get("https://127.0.0.1:6150") .header("sni", "openrusty.org") - .send().await.unwrap(); + .send() + .await + .unwrap(); assert_eq!(res.status(), reqwest::StatusCode::OK); assert_eq!(res.version(), reqwest::Version::HTTP_2); @@ -294,7 +297,10 @@ async fn test_simple_proxy_uds_peer() { assert!(is_specified_port(sockaddr.port())); assert_eq!(headers["x-upstream-client-addr"], "unset"); // unnamed UDS - assert_eq!(headers["x-upstream-server-addr"], "/tmp/pingora_nginx_test.sock"); + assert_eq!( + headers["x-upstream-server-addr"], + "/tmp/pingora_nginx_test.sock" + ); let body = res.text().await.unwrap(); assert_eq!(body, "Hello World!\n"); diff --git a/pingora-proxy/tests/utils/cert.rs b/pingora-proxy/tests/utils/cert.rs index ce47a874..0ff149c3 100644 --- a/pingora-proxy/tests/utils/cert.rs +++ b/pingora-proxy/tests/utils/cert.rs @@ -15,10 +15,13 @@ use once_cell::sync::Lazy; use std::fs; -#[cfg(not(feature = "rustls"))] -use pingora_core::tls::{pkey::{PKey, Private}, x509::X509}; #[cfg(feature = "rustls")] use pingora_core::tls::{load_pem_file_ca, load_pem_file_private_key}; +#[cfg(not(feature = "rustls"))] +use pingora_core::tls::{ + pkey::{PKey, Private}, + x509::X509, +}; //pub static ROOT_CERT: Lazy = Lazy::new(|| load_cert("keys/root.crt")); //pub static ROOT_KEY: Lazy> = Lazy::new(|| load_key("keys/root.key")); @@ -47,7 +50,10 @@ fn load_cert(path: &str) -> Vec { fn load_key(path: &str) -> Vec { let path = format!("{}/{path}", super::conf_dir()); let key_bytes = fs::read(path).unwrap(); - PKey::private_key_from_pem(&key_bytes).unwrap().private_key_to_der().unwrap() + PKey::private_key_from_pem(&key_bytes) + .unwrap() + .private_key_to_der() + .unwrap() } #[cfg(feature = "rustls")] diff --git a/pingora-proxy/tests/utils/mock_origin.rs b/pingora-proxy/tests/utils/mock_origin.rs index 5d266096..de13be69 100644 --- a/pingora-proxy/tests/utils/mock_origin.rs +++ b/pingora-proxy/tests/utils/mock_origin.rs @@ -13,21 +13,30 @@ // limitations under the License. use once_cell::sync::Lazy; +use std::path::Path; use std::process; use std::{thread, time}; -use std::path::Path; pub static MOCK_ORIGIN: Lazy = Lazy::new(init); fn init() -> bool { #[cfg(feature = "rustls")] - let src_cert_path = format!("{}/tests/utils/conf/keys/server_rustls.crt", env!("CARGO_MANIFEST_DIR")); + let src_cert_path = format!( + "{}/tests/utils/conf/keys/server_rustls.crt", + env!("CARGO_MANIFEST_DIR") + ); #[cfg(not(feature = "rustls"))] - let src_cert_path = format!("{}/tests/utils/conf/keys/server_boringssl_openssl.crt", env!("CARGO_MANIFEST_DIR")); + let src_cert_path = format!( + "{}/tests/utils/conf/keys/server_boringssl_openssl.crt", + env!("CARGO_MANIFEST_DIR") + ); let mut dst_cert_path = format!("{}/tests/keys/server.crt", env!("CARGO_MANIFEST_DIR")); std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); - dst_cert_path = format!("{}/tests/utils/conf/keys/server.crt", env!("CARGO_MANIFEST_DIR")); + dst_cert_path = format!( + "{}/tests/utils/conf/keys/server.crt", + env!("CARGO_MANIFEST_DIR") + ); std::fs::copy(Path::new(&src_cert_path), Path::new(&dst_cert_path)); // TODO: figure out a way to kill openresty when exiting diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs index bb52474e..2f7c6ecb 100644 --- a/pingora-rustls/src/lib.rs +++ b/pingora-rustls/src/lib.rs @@ -18,29 +18,34 @@ use std::fs::File; use std::io::BufReader; use log::{error, warn}; -pub use rustls::{ClientConfig, RootCertStore, ServerConfig, Stream, version}; +pub use no_debug::{Ellipses, NoDebug, WithTypeInfo}; +pub use rustls::{version, ClientConfig, RootCertStore, ServerConfig, Stream}; pub use rustls_native_certs::load_native_certs; use rustls_pemfile::Item; pub use rustls_pki_types::{CertificateDer, PrivateKeyDer, ServerName}; -pub use tokio_rustls::{Accept, Connect, TlsAcceptor, TlsConnector, TlsStream}; pub use tokio_rustls::client::TlsStream as ClientTlsStream; pub use tokio_rustls::server::TlsStream as ServerTlsStream; -pub use no_debug::{NoDebug, WithTypeInfo, Ellipses}; +pub use tokio_rustls::{Accept, Connect, TlsAcceptor, TlsConnector, TlsStream}; fn load_file(path: &String) -> BufReader { let file = File::open(path).expect("io error"); BufReader::new(file) } fn load_pem_file(path: &String) -> Result, std::io::Error> { - let iter: Vec = rustls_pemfile::read_all(&mut load_file(path)).filter_map(|f| { - if f.is_ok() { - Some(f.unwrap()) - } else { - let err = f.err().unwrap(); - warn!("Skipping PEM element in file \"{}\" due to error \"{}\"", path, err); - None - } - }).collect(); + let iter: Vec = rustls_pemfile::read_all(&mut load_file(path)) + .filter_map(|f| { + if f.is_ok() { + Some(f.unwrap()) + } else { + let err = f.err().unwrap(); + warn!( + "Skipping PEM element in file \"{}\" due to error \"{}\"", + path, err + ); + None + } + }) + .collect(); Ok(iter) } @@ -51,12 +56,12 @@ pub fn load_ca_file_into_store(path: &String, cert_store: &mut RootCertStore) { cas.into_iter().for_each(|pem_item| { // only loading certificates, handling a CA file match pem_item { - Item::X509Certificate(content) => { - match cert_store.add(content) { - Ok(_) => {} - Err(err) => { error!("{}", err) } + Item::X509Certificate(content) => match cert_store.add(content) { + Ok(_) => {} + Err(err) => { + error!("{}", err) } - } + }, Item::Pkcs1Key(_) => {} Item::Pkcs8Key(_) => {} Item::Sec1Key(_) => {} @@ -67,7 +72,10 @@ pub fn load_ca_file_into_store(path: &String, cert_store: &mut RootCertStore) { }); } Err(err) => { - error!("Failed to load configured ca file located at \"{}\", error: \"{}\"", path, err); + error!( + "Failed to load configured ca file located at \"{}\", error: \"{}\"", + path, err + ); } } } @@ -82,38 +90,34 @@ pub fn load_platform_certs_incl_env_into_store(ca_certs: &mut RootCertStore) { } } Err(err) => { - error!("Failed to load native platform ca-certificates: \"{:?}\". Continuing without ...", err); + error!( + "Failed to load native platform ca-certificates: \"{:?}\". Continuing without ...", + err + ); } } } -pub fn load_certs_key_file<'a>(cert: &String, key: &String) -> Option<(Vec>, PrivateKeyDer<'a>)> { +pub fn load_certs_key_file<'a>( + cert: &String, + key: &String, +) -> Option<(Vec>, PrivateKeyDer<'a>)> { let certs_file = load_pem_file(cert) .expect(format!("Failed to load configured cert file located at {}.", cert).as_str()); let key_file = load_pem_file(key) .expect(format!("Failed to load configured key file located at {}.", cert).as_str()); let mut certs: Vec> = vec![]; - certs_file.into_iter().for_each(|i| { - match i { - Item::X509Certificate(cert) => { - certs.push(cert) - } - _ => {} - } + certs_file.into_iter().for_each(|i| match i { + Item::X509Certificate(cert) => certs.push(cert), + _ => {} }); let private_key = match key_file.into_iter().next()? { - Item::Pkcs1Key(key) => { - Some(PrivateKeyDer::from(key)) - } - Item::Pkcs8Key(key) => { - Some(PrivateKeyDer::from(key)) - } - Item::Sec1Key(key) => { - Some(PrivateKeyDer::from(key)) - } - _ => { None } + Item::Pkcs1Key(key) => Some(PrivateKeyDer::from(key)), + Item::Pkcs8Key(key) => Some(PrivateKeyDer::from(key)), + Item::Sec1Key(key) => Some(PrivateKeyDer::from(key)), + _ => None, }; if certs.is_empty() || private_key.is_none() { @@ -134,10 +138,8 @@ pub fn load_pem_file_ca(path: &String) -> Vec { } }); match ca { - None => { Vec::new() } - Some(ca) => { - ca.to_vec() - } + None => Vec::new(), + Some(ca) => ca.to_vec(), } } @@ -151,7 +153,6 @@ pub fn load_pem_file_private_key(path: &String) -> Vec { Vec::new() } - pub fn hash_certificate(cert: CertificateDer) -> Vec { let hash = ring::digest::digest(&ring::digest::SHA256, cert.as_ref()); hash.as_ref().to_vec() diff --git a/pingora/examples/server.rs b/pingora/examples/server.rs index 083e6a73..e7924311 100644 --- a/pingora/examples/server.rs +++ b/pingora/examples/server.rs @@ -52,9 +52,9 @@ impl BackgroundService for ExampleBackgroundService { #[cfg(not(feature = "rustls"))] mod boringssl_openssl { + use super::*; use pingora::tls::pkey::{PKey, Private}; use pingora::tls::x509::X509; - use super::*; pub(super) struct DynamicCert { cert: X509, @@ -146,13 +146,16 @@ pub fn main() { let dynamic_cert = boringssl_openssl::DynamicCert::new(&cert_path, &key_path); tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo - tls_settings.get_builder().native() + tls_settings + .get_builder() + .native() .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) .unwrap(); } #[cfg(feature = "rustls")] { - tls_settings = pingora::listeners::TlsSettings::intermediate(&cert_path, &key_path).unwrap(); + tls_settings = + pingora::listeners::TlsSettings::intermediate(&cert_path, &key_path).unwrap(); } tls_settings.enable_h2(); echo_service_http.add_tls_with_settings("0.0.0.0:6148", None, tls_settings); From b8183bbcc3912177106f099769532f667b093146 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Wed, 24 Jul 2024 19:02:04 +0200 Subject: [PATCH 03/10] Fix visibility issue, update rustls & several clippy warnings --- .../connectors/tls/boringssl_openssl/mod.rs | 6 +++--- pingora-core/src/listeners/tls/mod.rs | 2 +- pingora-core/src/protocols/http/server.rs | 2 +- pingora-core/src/upstreams/peer.rs | 2 +- .../src/utils/tls/boringssl_openssl/mod.rs | 2 +- pingora-rustls/Cargo.toml | 2 +- pingora-rustls/src/lib.rs | 21 +++++++++---------- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs index bd41a690..dd3dfc33 100644 --- a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs @@ -182,7 +182,7 @@ where if let Some(ca_list) = peer.get_ca() { let mut store_builder = X509StoreBuilder::new().unwrap(); for ca in &**ca_list { - let cert = der_to_x509(&**ca)?; + let cert = der_to_x509(ca)?; store_builder.add_cert(cert).unwrap(); } ssl_set_verify_cert_store(&mut ssl_conf, &store_builder.build()) @@ -192,9 +192,9 @@ where // Set up client cert/key if let Some(key_pair) = peer.get_client_cert_key() { debug!("setting client cert and key"); - let leaf = der_to_x509(&*key_pair.leaf())?; + let leaf = der_to_x509(key_pair.leaf())?; ssl_use_certificate(&mut ssl_conf, &leaf).or_err(InternalError, "invalid client cert")?; - let key = der_to_private_key(&*key_pair.key())?; + let key = der_to_private_key(key_pair.key())?; ssl_use_private_key(&mut ssl_conf, &key).or_err(InternalError, "invalid client key")?; let intermediates = key_pair.intermediates(); diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs index 80cb5547..0c8ac0de 100644 --- a/pingora-core/src/listeners/tls/mod.rs +++ b/pingora-core/src/listeners/tls/mod.rs @@ -36,7 +36,7 @@ pub mod boringssl_openssl; #[cfg(feature = "rustls")] pub(crate) mod rustls; -pub(crate) struct Acceptor { +pub struct Acceptor { ssl_acceptor: Box, callbacks: Option, } diff --git a/pingora-core/src/protocols/http/server.rs b/pingora-core/src/protocols/http/server.rs index c6479e7a..56ae2c94 100644 --- a/pingora-core/src/protocols/http/server.rs +++ b/pingora-core/src/protocols/http/server.rs @@ -53,7 +53,7 @@ impl Session { /// else with the session. /// - `Ok(true)`: successful /// - `Ok(false)`: client exit without sending any bytes. This is normal on reused connection. - /// In this case the user should give up this session. + /// In this case the user should give up this session. pub async fn read_request(&mut self) -> Result { match self { Self::H1(s) => { diff --git a/pingora-core/src/upstreams/peer.rs b/pingora-core/src/upstreams/peer.rs index 848c47a9..0c60974c 100644 --- a/pingora-core/src/upstreams/peer.rs +++ b/pingora-core/src/upstreams/peer.rs @@ -388,7 +388,7 @@ impl Display for PeerOptions { write!( f, "CA: {}, expire: {},", - get_organizational_unit(&**ca).unwrap_or_default(), + get_organizational_unit(ca).unwrap_or_default(), get_not_after(ca), )?; } diff --git a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs index 9909fea9..0882cd1c 100644 --- a/pingora-core/src/utils/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/utils/tls/boringssl_openssl/mod.rs @@ -87,7 +87,7 @@ pub fn get_x509_serial(cert: &X509) -> pingora_error::Result { } pub fn der_to_x509(ca: &[u8]) -> pingora_error::Result { - let cert = X509::from_der(&*ca).explain_err(InvalidCert, |e| { + let cert = X509::from_der(ca).explain_err(InvalidCert, |e| { format!( "Failed to convert ca certificate in DER form to X509 cert. Error: {:?}", e diff --git a/pingora-rustls/Cargo.toml b/pingora-rustls/Cargo.toml index f68d8da0..aef66661 100644 --- a/pingora-rustls/Cargo.toml +++ b/pingora-rustls/Cargo.toml @@ -19,7 +19,7 @@ path = "src/lib.rs" [dependencies] log = "0.4.21" ring = "0.17.8" -rustls = "0.23.10" +rustls = "0.23.12" rustls-native-certs = "0.7.1" rustls-pemfile = "2.1.2" rustls-pki-types = "1.7.0" diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs index 2f7c6ecb..e4d5de0e 100644 --- a/pingora-rustls/src/lib.rs +++ b/pingora-rustls/src/lib.rs @@ -34,8 +34,8 @@ fn load_file(path: &String) -> BufReader { fn load_pem_file(path: &String) -> Result, std::io::Error> { let iter: Vec = rustls_pemfile::read_all(&mut load_file(path)) .filter_map(|f| { - if f.is_ok() { - Some(f.unwrap()) + if let Ok(f) = f { + Some(f) } else { let err = f.err().unwrap(); warn!( @@ -103,14 +103,15 @@ pub fn load_certs_key_file<'a>( key: &String, ) -> Option<(Vec>, PrivateKeyDer<'a>)> { let certs_file = load_pem_file(cert) - .expect(format!("Failed to load configured cert file located at {}.", cert).as_str()); + .unwrap_or_else(|_| panic!("Failed to load configured cert file located at {}.", cert)); let key_file = load_pem_file(key) - .expect(format!("Failed to load configured key file located at {}.", cert).as_str()); + .unwrap_or_else(|_| panic!("Failed to load configured key file located at {}.", cert)); let mut certs: Vec> = vec![]; - certs_file.into_iter().for_each(|i| match i { - Item::X509Certificate(cert) => certs.push(cert), - _ => {} + certs_file.into_iter().for_each(|i| { + if let Item::X509Certificate(cert) = i { + certs.push(cert) + } }); let private_key = match key_file.into_iter().next()? { @@ -145,10 +146,8 @@ pub fn load_pem_file_ca(path: &String) -> Vec { pub fn load_pem_file_private_key(path: &String) -> Vec { let key = rustls_pemfile::private_key(&mut load_file(path)); - if let Ok(key) = key { - if let Some(key) = key { - return key.secret_der().to_vec(); - } + if let Ok(Some(key)) = key { + return key.secret_der().to_vec(); } Vec::new() } From f56d04e738614e62d9a09c546f11d23e3f1e9dbe Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Wed, 24 Jul 2024 20:08:44 +0200 Subject: [PATCH 04/10] Fix clippy warning --- pingora-core/src/protocols/tls/boringssl_openssl/stream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs index ef416816..63df200d 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -98,7 +98,7 @@ impl InnerStream { ssl::ErrorCode::SSL => { // Unify the return type of `verify_result` for openssl #[cfg(not(feature = "boringssl"))] - fn verify_result(ssl: &SslRef) -> Result<(), i32> { + fn verify_result(ssl: &SslRef) -> Result<(), i32> { match ssl.verify_result().as_raw() { crate::tls::ssl_sys::X509_V_OK => Ok(()), e => Err(e), @@ -107,11 +107,11 @@ impl InnerStream { // Unify the return type of `verify_result` for boringssl #[cfg(feature = "boringssl")] - fn verify_result(ssl: &SslRef) -> Result<(), i32> { + fn verify_result(ssl: &SslRef) -> Result<(), i32> { ssl.verify_result().map_err(|e| e.as_raw()) } - match verify_result::(self.0.ssl()) { + match verify_result(self.0.ssl()) { Ok(()) => Error::e_explain(TLSHandshakeFailure, context), // X509_V_ERR_INVALID_CALL in case verify result was never set Err(X509_V_ERR_INVALID_CALL) => { From 022252e6d7dc94fc4a655b07056fb2394188a6a3 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Fri, 23 Aug 2024 19:10:35 +0200 Subject: [PATCH 05/10] fix: remove several Box types in handshake use generics instead as used within the original version remove trait implementations for Box as no longer required --- pingora-core/src/listeners/mod.rs | 9 ++---- pingora-core/src/listeners/tls/mod.rs | 5 +-- pingora-core/src/protocols/l4/stream.rs | 3 +- .../protocols/tls/boringssl_openssl/server.rs | 13 +++----- pingora-core/src/protocols/tls/mod.rs | 31 +------------------ .../src/protocols/tls/rustls/server.rs | 11 +++---- pingora-core/src/protocols/tls/server.rs | 14 +-------- 7 files changed, 15 insertions(+), 71 deletions(-) diff --git a/pingora-core/src/listeners/mod.rs b/pingora-core/src/listeners/mod.rs index 9c5c69a3..025ae932 100644 --- a/pingora-core/src/listeners/mod.rs +++ b/pingora-core/src/listeners/mod.rs @@ -23,11 +23,7 @@ use tls::Acceptor; pub use tls::{TlsSettings, ALPN}; pub use crate::protocols::tls::server::TlsAccept; -#[cfg(not(feature = "rustls"))] -use crate::protocols::tls::TlsStream as TlsStreamProvider; -#[cfg(feature = "rustls")] -use crate::protocols::tls::TlsStream as TlsStreamProvider; -use crate::protocols::{Stream, IO}; +use crate::protocols::Stream; use crate::server::ListenFds; mod l4; @@ -85,8 +81,7 @@ pub(crate) struct UninitializedStream { impl UninitializedStream { pub async fn handshake(self) -> Result { if let Some(tls) = self.tls { - let tls_stream: TlsStreamProvider> = - tls.handshake(Box::new(self.l4)).await?; + let tls_stream = tls.handshake(self.l4).await?; Ok(Box::new(tls_stream)) } else { Ok(Box::new(self.l4)) diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs index 0c8ac0de..b556e171 100644 --- a/pingora-core/src/listeners/tls/mod.rs +++ b/pingora-core/src/listeners/tls/mod.rs @@ -113,10 +113,7 @@ impl TlsSettings { } impl Acceptor { - pub async fn handshake( - &self, - stream: Box, - ) -> Result>> { + pub async fn handshake(&self, stream: S) -> Result> { debug!("new tls session"); // TODO: be able to offload this handshake in a thread pool if let Some(cb) = self.callbacks.as_ref() { diff --git a/pingora-core/src/protocols/l4/stream.rs b/pingora-core/src/protocols/l4/stream.rs index 70ecdb31..ee91a8aa 100644 --- a/pingora-core/src/protocols/l4/stream.rs +++ b/pingora-core/src/protocols/l4/stream.rs @@ -30,7 +30,7 @@ use crate::protocols::l4::ext::{set_tcp_keepalive, TcpKeepalive}; use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::{ GetProxyDigest, GetSocketDigest, GetTimingDigest, Shutdown, SocketDigest, Ssl, TimingDigest, - UniqueID, IO, + UniqueID, }; use crate::upstreams::peer::Tracer; @@ -208,7 +208,6 @@ impl UniqueID for Stream { } impl Ssl for Stream {} -impl Ssl for Box {} #[async_trait] impl Shutdown for Stream { diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs index 621b63ef..249374be 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -66,10 +66,7 @@ fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake( - acceptor: &Acceptor, - io: Box, -) -> Result>> { +pub async fn handshake(acceptor: &Acceptor, io: S) -> Result> { let mut stream = prepare_tls_stream(acceptor, io)?; stream .accept() @@ -79,11 +76,11 @@ pub async fn handshake( } /// Perform TLS handshake for the given connection with the given configuration and callbacks -pub async fn handshake_with_callback( +pub async fn handshake_with_callback( acceptor: &Acceptor, - io: Box, + io: S, callbacks: &TlsAcceptCallbacks, -) -> pingora_error::Result>> { +) -> pingora_error::Result> { let mut tls_stream = prepare_tls_stream(acceptor, io)?; let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { @@ -148,5 +145,5 @@ async fn test_async_cert() { }); let acceptor = TlsSettings::with_callbacks(cb).unwrap().build(); - acceptor.handshake(Box::new(server)).await.unwrap(); + acceptor.handshake(server).await.unwrap(); } diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs index f9b1c06d..0bff20c3 100644 --- a/pingora-core/src/protocols/tls/mod.rs +++ b/pingora-core/src/protocols/tls/mod.rs @@ -23,9 +23,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; use crate::protocols::digest::TimingDigest; use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{ - GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest, UniqueID, IO, -}; +use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest}; #[cfg(not(feature = "rustls"))] pub(crate) mod boringssl_openssl; @@ -58,33 +56,6 @@ pub trait InnerTlsStream { fn selected_alpn_proto(&mut self) -> Option; } -impl GetSocketDigest for Box { - fn get_socket_digest(&self) -> Option> { - (**self).get_socket_digest() - } - fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - (**self).set_socket_digest(socket_digest) - } -} - -impl GetTimingDigest for Box { - fn get_timing_digest(&self) -> Vec> { - vec![] - } -} - -impl GetProxyDigest for Box { - fn get_proxy_digest(&self) -> Option> { - (**self).get_proxy_digest() - } -} - -impl UniqueID for Box { - fn id(&self) -> i32 { - (**self).id() - } -} - /// The protocol for Application-Layer Protocol Negotiation #[derive(Hash, Clone, Debug)] pub enum ALPN { diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs index 83a9d7df..d520dbbe 100644 --- a/pingora-core/src/protocols/tls/rustls/server.rs +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -55,10 +55,7 @@ async fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result, -) -> Result>> { +pub async fn handshake(acceptor: &Acceptor, io: S) -> Result> { let mut stream = prepare_tls_stream(acceptor, io).await?; stream .accept() @@ -69,11 +66,11 @@ pub async fn handshake( /// Perform TLS handshake for the given connection with the given configuration and callbacks /// callbacks are currently not supported within pingora Rustls and are ignored -pub async fn handshake_with_callback( +pub async fn handshake_with_callback( acceptor: &Acceptor, - io: Box, + io: S, _callbacks: &TlsAcceptCallbacks, -) -> Result>> { +) -> Result> { let mut tls_stream = prepare_tls_stream(acceptor, io).await?; let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { diff --git a/pingora-core/src/protocols/tls/server.rs b/pingora-core/src/protocols/tls/server.rs index c0834af4..a4bd2e49 100644 --- a/pingora-core/src/protocols/tls/server.rs +++ b/pingora-core/src/protocols/tls/server.rs @@ -23,7 +23,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use pingora_error::Result; use crate::protocols::tls::TlsStream; -use crate::protocols::{Shutdown, IO}; +use crate::protocols::Shutdown; #[cfg(not(feature = "rustls"))] use crate::tls::ssl::SslRef; @@ -57,18 +57,6 @@ where } } -#[async_trait] -impl Shutdown for Box { - async fn shutdown(&mut self) { - match ::shutdown(self).await { - Ok(()) => {} - Err(e) => { - warn!("TLS shutdown failed, {e}"); - } - } - } -} - /// Resumable TLS server side handshake. #[async_trait] pub trait ResumableAccept { From 97a3df73ba290b58964b1db599d884c565bc3695 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Thu, 5 Sep 2024 16:10:38 +0200 Subject: [PATCH 06/10] fix: remove indirection for protocols::tls::TlsStream.tls --- .../protocols/tls/boringssl_openssl/stream.rs | 17 +++---------- pingora-core/src/protocols/tls/mod.rs | 6 ++--- .../src/protocols/tls/rustls/stream.rs | 25 +++---------------- 3 files changed, 11 insertions(+), 37 deletions(-) diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs index 63df200d..89a8dca6 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -14,7 +14,6 @@ //! BoringSSL & OpenSSL TLS stream specific implementation -use async_trait::async_trait; use log::warn; use std::pin::Pin; use std::sync::Arc; @@ -22,10 +21,8 @@ use tokio::io::{AsyncRead, AsyncWrite}; use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use crate::listeners::ALPN; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::tls::InnerTlsStream; use crate::protocols::tls::SslDigest; use crate::protocols::{GetProxyDigest, GetTimingDigest}; use crate::tls::error::ErrorStack; @@ -58,10 +55,9 @@ impl InnerStream { } } -#[async_trait] -impl InnerTlsStream for InnerStream { +impl InnerStream { /// Connect to the remote TLS server as a client - async fn connect(&mut self) -> Result<()> { + pub(crate) async fn connect(&mut self) -> Result<()> { Self::clear_error(); match Pin::new(&mut self.0).connect().await { Ok(_) => Ok(()), @@ -70,7 +66,7 @@ impl InnerTlsStream for InnerStream } /// Finish the TLS handshake from client as a server - async fn accept(&mut self) -> Result<()> { + pub(crate) async fn accept(&mut self) -> Result<()> { Self::clear_error(); match Pin::new(&mut self.0).accept().await { Ok(_) => Ok(()), @@ -78,14 +74,9 @@ impl InnerTlsStream for InnerStream } } - fn digest(&mut self) -> Option> { + pub(crate) fn digest(&mut self) -> Option> { Some(Arc::new(SslDigest::from_ssl(self.0.ssl()))) } - - fn selected_alpn_proto(&mut self) -> Option { - let ssl = self.0.ssl(); - ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) - } } impl InnerStream { diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs index 0bff20c3..9eaf3afe 100644 --- a/pingora-core/src/protocols/tls/mod.rs +++ b/pingora-core/src/protocols/tls/mod.rs @@ -44,6 +44,9 @@ pub struct TlsStream { timing: TimingDigest, } +// NOTE: keeping trait for documentation purpose +// switched to direct implementations to eliminate redirections in within the call-graph +// the below trait is required for InnerStream to be implemented #[async_trait] pub trait InnerTlsStream { async fn connect(&mut self) -> Result<()>; @@ -51,9 +54,6 @@ pub trait InnerTlsStream { /// Return the [`ssl::SslDigest`] for logging fn digest(&mut self) -> Option>; - - /// Return selected ALPN if any - fn selected_alpn_proto(&mut self) -> Option; } /// The protocol for Application-Layer Protocol Negotiation diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index 349e056a..28c8219b 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use async_trait::async_trait; use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; use pingora_error::{Error, ImmutStr, OrErr, Result}; use pingora_rustls::NoDebug; @@ -24,10 +23,8 @@ use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use crate::listeners::tls::Acceptor; -use crate::listeners::ALPN; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::tls::InnerTlsStream; use crate::protocols::tls::SslDigest; use crate::protocols::{GetProxyDigest, GetTimingDigest}; @@ -67,11 +64,9 @@ impl InnerStream { }) } } - -#[async_trait] -impl InnerTlsStream for InnerStream { +impl InnerStream { /// Connect to the remote TLS server as a client - async fn connect(&mut self) -> Result<()> { + pub(crate) async fn connect(&mut self) -> Result<()> { let connect = &mut (*self.connect); if let Some(ref mut connect) = connect { @@ -92,7 +87,7 @@ impl InnerTlsStream for InnerStream /// Finish the TLS handshake from client as a server /// no-op implementation within Rustls, handshake is performed during creation of stream. - async fn accept(&mut self) -> Result<()> { + pub(crate) async fn accept(&mut self) -> Result<()> { let accept = &mut (*self.accept); if let Some(ref mut accept) = accept { @@ -111,21 +106,9 @@ impl InnerTlsStream for InnerStream } } - fn digest(&mut self) -> Option> { + pub(crate) fn digest(&mut self) -> Option> { Some(Arc::new(SslDigest::from_stream(&self.stream))) } - - fn selected_alpn_proto(&mut self) -> Option { - if let Some(stream) = self.stream.as_ref() { - let proto = stream.get_ref().1.alpn_protocol(); - match proto { - None => None, - Some(raw) => ALPN::from_wire_selected(raw), - } - } else { - None - } - } } impl GetSocketDigest for InnerStream From bc83020d1942dcc8b7fa6c29a296f2eb8fe99ba9 Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Sat, 7 Sep 2024 20:06:23 +0200 Subject: [PATCH 07/10] fix: remove indirection in connectors:tls:Connector replace direct field access with getter method that returns &Arc, store concrete type in now private struct field using impl Trait in return value keeps method signature stable across features; allows for static dispatch adjust visibility for TlsConnectorCtx structs from pub(crate) to pub(super) --- pingora-core/src/connectors/mod.rs | 12 ++++---- .../connectors/tls/boringssl_openssl/mod.rs | 7 +++-- pingora-core/src/connectors/tls/mod.rs | 28 +++++++++++++------ pingora-core/src/connectors/tls/rustls/mod.rs | 7 +++-- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/pingora-core/src/connectors/mod.rs b/pingora-core/src/connectors/mod.rs index b3078fdf..38da0cd9 100644 --- a/pingora-core/src/connectors/mod.rs +++ b/pingora-core/src/connectors/mod.rs @@ -174,11 +174,13 @@ impl TransportConnector { let stream = if let Some(rt) = rt { let peer = peer.clone(); let tls_ctx = self.tls_ctx.clone(); - rt.spawn(async move { do_connect(&peer, bind_to, alpn_override, &tls_ctx.ctx).await }) - .await - .or_err(InternalError, "offload runtime failure")?? + rt.spawn( + async move { do_connect(&peer, bind_to, alpn_override, tls_ctx.context()).await }, + ) + .await + .or_err(InternalError, "offload runtime failure")?? } else { - do_connect(peer, bind_to, alpn_override, &self.tls_ctx.ctx).await? + do_connect(peer, bind_to, alpn_override, self.tls_ctx.context()).await? }; Ok(stream) @@ -437,7 +439,7 @@ mod tests { /// the decomposed error type and message async fn get_do_connect_failure_with_peer(peer: &BasicPeer) -> (ErrorType, String) { let connector = Connector::new(None); - let stream = do_connect(peer, None, None, &connector.ctx).await; + let stream = do_connect(peer, None, None, connector.context()).await; match stream { Ok(_) => panic!("should throw an error"), Err(e) => ( diff --git a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs index dd3dfc33..a16c61c5 100644 --- a/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs @@ -94,7 +94,7 @@ fn init_ssl_cert_env_vars() { INIT_CA_ENV.call_once(openssl_probe::init_ssl_cert_env_vars); } -pub(crate) struct TlsConnectorCtx(pub(crate) SslConnector); +pub(super) struct TlsConnectorCtx(SslConnector); impl TlsConnectorContext for TlsConnectorCtx { fn as_any(&self) -> &dyn Any { @@ -162,15 +162,16 @@ impl TlsConnectorContext for TlsConnectorCtx { } } -pub(super) async fn connect( +pub(super) async fn connect( stream: T, peer: &P, alpn_override: Option, - tls_ctx: &Arc, + tls_ctx: &Arc, ) -> Result> where T: IO, P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, { let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); let mut ssl_conf = ctx.0.configure().unwrap(); diff --git a/pingora-core/src/connectors/tls/mod.rs b/pingora-core/src/connectors/tls/mod.rs index 01167850..9ede8049 100644 --- a/pingora-core/src/connectors/tls/mod.rs +++ b/pingora-core/src/connectors/tls/mod.rs @@ -40,16 +40,20 @@ pub(crate) mod rustls; #[derive(Clone)] pub struct Connector { - pub(crate) ctx: Arc, // Arc to support clone + ctx: Arc, // Arc to support clone } impl Connector { pub fn new(options: Option) -> Self { TlsConnectorCtx::build_connector(options) } + + pub fn context(&self) -> &Arc { + &self.ctx + } } -pub(crate) trait TlsConnectorContext { +pub trait TlsConnectorContext { fn as_any(&self) -> &dyn Any; fn build_connector(options: Option) -> Connector @@ -57,12 +61,16 @@ pub(crate) trait TlsConnectorContext { Self: Sized; } -pub(super) async fn do_connect( +pub(super) async fn do_connect( peer: &P, bind_to: Option, alpn_override: Option, - tls_ctx: &Arc, -) -> Result { + tls_ctx: &Arc, +) -> Result +where + P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, +{ // Create the future that does the connections, but don't evaluate it until // we decide if we need a timeout or not let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx); @@ -79,12 +87,16 @@ pub(super) async fn do_connect( } } -async fn do_connect_inner( +async fn do_connect_inner( peer: &P, bind_to: Option, alpn_override: Option, - tls_ctx: &Arc, -) -> Result { + tls_ctx: &Arc, +) -> Result +where + P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, +{ let stream = l4_connect(peer, bind_to).await?; if peer.tls() { let tls_stream = tls_connect(stream, peer, alpn_override, tls_ctx).await?; diff --git a/pingora-core/src/connectors/tls/rustls/mod.rs b/pingora-core/src/connectors/tls/rustls/mod.rs index 453c0ab7..9953e7a3 100644 --- a/pingora-core/src/connectors/tls/rustls/mod.rs +++ b/pingora-core/src/connectors/tls/rustls/mod.rs @@ -41,7 +41,7 @@ use crate::upstreams::peer::Peer; use super::{replace_leftmost_underscore, Connector, TlsConnectorContext}; -pub(crate) struct TlsConnectorCtx { +pub(super) struct TlsConnectorCtx { config: RusTlsClientConfig, ca_certs: RootCertStore, } @@ -110,15 +110,16 @@ impl TlsConnectorContext for TlsConnectorCtx { } } -pub(super) async fn connect( +pub(super) async fn connect( stream: T, peer: &P, alpn_override: Option, - tls_ctx: &Arc, + tls_ctx: &Arc, ) -> Result> where T: IO, P: Peer + Send + Sync, + C: TlsConnectorContext + Send + Sync, { let ctx = tls_ctx.as_any().downcast_ref::().unwrap(); let mut config = ctx.config.clone(); From a42eaf1dd48daa58a2d19cbf10dc805a639f9d3f Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Tue, 10 Sep 2024 10:33:12 +0200 Subject: [PATCH 08/10] fix: rustls remove NoDebug, impl Debug --- .../src/protocols/tls/rustls/stream.rs | 47 +++++++++++++------ pingora-rustls/Cargo.toml | 1 - pingora-rustls/src/lib.rs | 1 - 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index 28c8219b..358f79d9 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::fmt; +use core::fmt::Formatter; use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; use pingora_error::{Error, ImmutStr, OrErr, Result}; -use pingora_rustls::NoDebug; use pingora_rustls::TlsAcceptor as RusTlsAcceptor; use pingora_rustls::TlsStream as RusTlsStream; use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; @@ -28,11 +29,31 @@ use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::tls::SslDigest; use crate::protocols::{GetProxyDigest, GetTimingDigest}; -#[derive(Debug)] pub struct InnerStream { pub(crate) stream: Option>, - connect: NoDebug>>, - accept: NoDebug>>, + connect: Option>, + accept: Option>, +} + +impl Debug for InnerStream { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + fmt.debug_struct("InnerStream") + .field("stream", &self.stream) + .field("connect", { + if self.connect.is_some() { + &"Some(Connect)" + } else { + &"None" + } + }) + .field("accept", { + if self.accept.is_some() { + &"Some(Accept)" + } else { + &"None" + } + }).finish() + } } impl InnerStream { @@ -47,8 +68,8 @@ impl InnerStream { ) -> Result { let connect = connector.connect(server.to_owned(), stream); Ok(InnerStream { - accept: None.into(), - connect: Some(connect).into(), + accept: None, + connect: Some(connect), stream: None, }) } @@ -58,8 +79,8 @@ impl InnerStream { let accept = tls_acceptor.accept(stream); Ok(InnerStream { - accept: Some(accept).into(), - connect: None.into(), + accept: Some(accept), + connect: None, stream: None, }) } @@ -67,14 +88,14 @@ impl InnerStream { impl InnerStream { /// Connect to the remote TLS server as a client pub(crate) async fn connect(&mut self) -> Result<()> { - let connect = &mut (*self.connect); + let connect = &mut self.connect; if let Some(ref mut connect) = connect { let stream = connect .await .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; self.stream = Some(RusTlsStream::Client(stream)); - self.connect = None.into(); + self.connect = None; Ok(()) } else { @@ -88,14 +109,12 @@ impl InnerStream { /// Finish the TLS handshake from client as a server /// no-op implementation within Rustls, handshake is performed during creation of stream. pub(crate) async fn accept(&mut self) -> Result<()> { - let accept = &mut (*self.accept); - - if let Some(ref mut accept) = accept { + if let Some(ref mut accept) = &mut self.accept { let stream = accept .await .explain_err(TLSHandshakeFailure, |e| format!("tls connect error: {e}"))?; self.stream = Some(RusTlsStream::Server(stream)); - self.connect = None.into(); + self.connect = None; Ok(()) } else { diff --git a/pingora-rustls/Cargo.toml b/pingora-rustls/Cargo.toml index aef66661..c4f6134e 100644 --- a/pingora-rustls/Cargo.toml +++ b/pingora-rustls/Cargo.toml @@ -24,7 +24,6 @@ rustls-native-certs = "0.7.1" rustls-pemfile = "2.1.2" rustls-pki-types = "1.7.0" tokio-rustls = "0.26.0" -no_debug = "3.1.0" [dev-dependencies] tokio-test = "0.4.3" diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs index e4d5de0e..be6a42c8 100644 --- a/pingora-rustls/src/lib.rs +++ b/pingora-rustls/src/lib.rs @@ -18,7 +18,6 @@ use std::fs::File; use std::io::BufReader; use log::{error, warn}; -pub use no_debug::{Ellipses, NoDebug, WithTypeInfo}; pub use rustls::{version, ClientConfig, RootCertStore, ServerConfig, Stream}; pub use rustls_native_certs::load_native_certs; use rustls_pemfile::Item; From b84ad211e8d5c2cfcfc5fd65f517b1090b452f0e Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Tue, 10 Sep 2024 11:40:39 +0200 Subject: [PATCH 09/10] fix: remove indirections in listener::tls::Acceptor & TlsSettings --- .../listeners/tls/boringssl_openssl/mod.rs | 57 ++++++-------- pingora-core/src/listeners/tls/mod.rs | 78 +++++++++---------- pingora-core/src/listeners/tls/rustls/mod.rs | 35 ++------- .../protocols/tls/boringssl_openssl/server.rs | 15 ++-- pingora-core/src/protocols/tls/rustls/mod.rs | 5 +- .../src/protocols/tls/rustls/server.rs | 10 +-- .../src/protocols/tls/rustls/stream.rs | 12 ++- pingora/examples/server.rs | 7 +- 8 files changed, 93 insertions(+), 126 deletions(-) diff --git a/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs index c6b4b9d1..b5d0556c 100644 --- a/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/listeners/tls/boringssl_openssl/mod.rs @@ -14,31 +14,22 @@ //! BoringSSL & OpenSSL listener specific implementation -use crate::listeners::tls::{NativeBuilder, TlsAcceptor, TlsAcceptorBuilder}; use crate::listeners::{TlsSettings, ALPN}; use crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; -use async_trait::async_trait; -use core::any::Any; use pingora_error::{ErrorType, OrErr, Result}; -const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); +use std::ops::{Deref, DerefMut}; -struct TlsAcc(SslAcceptor); -pub(super) struct TlsAcceptorBuil(SslAcceptorBuilder); +const TLS_CONF_ERR: ErrorType = ErrorType::Custom("TLSConfigError"); -#[async_trait] -impl TlsAcceptor for TlsAcc { - fn get_acceptor(&self) -> &dyn Any { - &self.0 - } -} +pub struct TlsAcceptorBuil(SslAcceptorBuilder); +pub(super) struct TlsAcc(pub(super) SslAcceptor); -impl TlsAcceptorBuilder for TlsAcceptorBuil { - fn build(self: Box) -> Box { - let builder = (*self).0; - Box::new(TlsAcc(SslAcceptorBuilder::build(builder))) +impl TlsAcceptorBuil { + pub(super) fn build(self) -> TlsAcc { + TlsAcc(SslAcceptorBuilder::build(self.0)) } - fn set_alpn(&mut self, alpn: ALPN) { + pub(super) fn set_alpn(&mut self, alpn: ALPN) { match alpn { ALPN::H2H1 => self.0.set_alpn_select_callback(alpn::prefer_h2), ALPN::H1 => self.0.set_alpn_select_callback(alpn::h1_only), @@ -46,7 +37,7 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { } } - fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + pub(super) fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result where Self: Sized, { @@ -65,7 +56,7 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { Ok(TlsAcceptorBuil(accept_builder)) } - fn acceptor_with_callbacks() -> Result + pub(super) fn acceptor_with_callbacks() -> Result where Self: Sized, { @@ -75,24 +66,12 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { )?; Ok(TlsAcceptorBuil(accept_builder)) } - - fn as_any(&mut self) -> &mut dyn Any { - self as &mut dyn Any - } -} - -impl NativeBuilder for Box { - type Builder = SslAcceptorBuilder; - - fn native(&mut self) -> &mut Self::Builder { - self.as_any().downcast_mut::().unwrap() - } } impl From for TlsSettings { fn from(settings: SslAcceptorBuilder) -> Self { TlsSettings { - accept_builder: Box::new(TlsAcceptorBuil(settings)), + accept_builder: TlsAcceptorBuil(settings), callbacks: None, } } @@ -125,3 +104,17 @@ mod alpn { } } } + +impl Deref for TlsAcceptorBuil { + type Target = SslAcceptorBuilder; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TlsAcceptorBuil { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/pingora-core/src/listeners/tls/mod.rs b/pingora-core/src/listeners/tls/mod.rs index b556e171..9340a151 100644 --- a/pingora-core/src/listeners/tls/mod.rs +++ b/pingora-core/src/listeners/tls/mod.rs @@ -12,48 +12,46 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; - -use async_trait::async_trait; -use log::debug; - -#[cfg(not(feature = "rustls"))] -use crate::listeners::tls::boringssl_openssl::TlsAcceptorBuil; -#[cfg(feature = "rustls")] -use crate::listeners::tls::rustls::TlsAcceptorBuil; -#[cfg(not(feature = "rustls"))] -use crate::protocols::tls::boringssl_openssl::server::{handshake, handshake_with_callback}; -#[cfg(feature = "rustls")] -use crate::protocols::tls::rustls::server::{handshake, handshake_with_callback}; use crate::protocols::tls::server::TlsAcceptCallbacks; use crate::protocols::tls::TlsStream; pub use crate::protocols::tls::ALPN; use crate::protocols::IO; +#[cfg(not(feature = "rustls"))] +use crate::{ + listeners::tls::boringssl_openssl::{TlsAcc, TlsAcceptorBuil}, + protocols::tls::boringssl_openssl::server::{handshake, handshake_with_callback}, +}; +#[cfg(feature = "rustls")] +use crate::{ + listeners::tls::rustls::{TlsAcc, TlsAcceptorBuil}, + protocols::tls::rustls::server::{handshake, handshake_with_callback}, +}; +use log::debug; use pingora_error::Result; +use std::ops::{Deref, DerefMut}; #[cfg(not(feature = "rustls"))] -pub mod boringssl_openssl; +pub(crate) mod boringssl_openssl; #[cfg(feature = "rustls")] pub(crate) mod rustls; pub struct Acceptor { - ssl_acceptor: Box, + tls_acceptor: TlsAcc, callbacks: Option, } -#[async_trait] -pub trait TlsAcceptor { - fn get_acceptor(&self) -> &dyn Any; -} - /// The TLS settings of a listening endpoint pub struct TlsSettings { - accept_builder: Box, + accept_builder: TlsAcceptorBuil, callbacks: Option, } -pub trait TlsAcceptorBuilder: Any { - fn build(self: Box) -> Box; +// NOTE: keeping trait for documentation purpose +// switched to direct implementations to eliminate redirections in within the call-graph +// the below trait needs to be implemented for TlsAcceptorBuil +pub trait TlsAcceptorBuilder { + type TlsAcceptor; // TlsAcc + fn build(self) -> Self::TlsAcceptor; fn set_alpn(&mut self, alpn: ALPN); fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result where @@ -61,12 +59,6 @@ pub trait TlsAcceptorBuilder: Any { fn acceptor_with_callbacks() -> Result where Self: Sized; - fn as_any(&mut self) -> &mut dyn Any; -} - -pub trait NativeBuilder { - type Builder; - fn native(&mut self) -> &mut Self::Builder; } impl TlsSettings { @@ -75,7 +67,7 @@ impl TlsSettings { /// Return error if the provided certificate and private key are invalid or not found. pub fn intermediate(cert_path: &str, key_path: &str) -> Result { Ok(TlsSettings { - accept_builder: Box::new(TlsAcceptorBuil::acceptor_intermediate(cert_path, key_path)?), + accept_builder: TlsAcceptorBuil::acceptor_intermediate(cert_path, key_path)?, callbacks: None, }) } @@ -84,7 +76,7 @@ impl TlsSettings { /// is needed to provide the certificate during the TLS handshake. pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result { Ok(TlsSettings { - accept_builder: Box::new(TlsAcceptorBuil::acceptor_with_callbacks()?), + accept_builder: TlsAcceptorBuil::acceptor_with_callbacks()?, callbacks: Some(callbacks), }) } @@ -102,14 +94,10 @@ impl TlsSettings { pub(crate) fn build(self) -> Acceptor { Acceptor { - ssl_acceptor: self.accept_builder.build(), + tls_acceptor: self.accept_builder.build(), callbacks: self.callbacks, } } - - pub fn get_builder(&mut self) -> &mut Box { - &mut (self.accept_builder) - } } impl Acceptor { @@ -117,13 +105,23 @@ impl Acceptor { debug!("new tls session"); // TODO: be able to offload this handshake in a thread pool if let Some(cb) = self.callbacks.as_ref() { - handshake_with_callback(self, stream, cb).await + handshake_with_callback(&self.tls_acceptor.0, stream, cb).await } else { - handshake(self, stream).await + handshake(&self.tls_acceptor.0, stream).await } } +} + +impl Deref for TlsSettings { + type Target = TlsAcceptorBuil; + + fn deref(&self) -> &Self::Target { + &self.accept_builder + } +} - pub(crate) fn inner(&self) -> &dyn Any { - self.ssl_acceptor.get_acceptor() +impl DerefMut for TlsSettings { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.accept_builder } } diff --git a/pingora-core/src/listeners/tls/rustls/mod.rs b/pingora-core/src/listeners/tls/rustls/mod.rs index debec796..2e36913c 100644 --- a/pingora-core/src/listeners/tls/rustls/mod.rs +++ b/pingora-core/src/listeners/tls/rustls/mod.rs @@ -14,39 +14,26 @@ //! Rustls TLS listener specific implementation -use std::any::Any; use std::sync::Arc; -use async_trait::async_trait; - use pingora_error::ErrorType::InternalError; use pingora_error::{Error, ErrorSource, ImmutStr, OrErr, Result}; use pingora_rustls::load_certs_key_file; use pingora_rustls::ServerConfig; use pingora_rustls::{version, TlsAcceptor as RusTlsAcceptor}; -use crate::listeners::tls::{TlsAcceptor, TlsAcceptorBuilder}; use crate::listeners::ALPN; -pub(super) struct TlsAcceptorBuil { +pub struct TlsAcceptorBuil { alpn_protocols: Option>>, cert_path: String, key_path: String, } -struct TlsAcc { - acceptor: RusTlsAcceptor, -} - -#[async_trait] -impl TlsAcceptor for TlsAcc { - fn get_acceptor(&self) -> &dyn Any { - &self.acceptor - } -} +pub(super) struct TlsAcc(pub(super) RusTlsAcceptor); -impl TlsAcceptorBuilder for TlsAcceptorBuil { - fn build(self: Box) -> Box { +impl TlsAcceptorBuil { + pub(super) fn build(self) -> TlsAcc { let (certs, key) = load_certs_key_file(&self.cert_path, &self.key_path).expect( format!( "Failed to load provided certificates \"{}\" or key \"{}\".", @@ -68,15 +55,13 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { config.alpn_protocols = alpn_protocols; } - Box::new(TlsAcc { - acceptor: RusTlsAcceptor::from(Arc::new(config)), - }) + TlsAcc(RusTlsAcceptor::from(Arc::new(config))) } - fn set_alpn(&mut self, alpn: ALPN) { + pub(super) fn set_alpn(&mut self, alpn: ALPN) { self.alpn_protocols = Some(alpn.to_wire_protocols()); } - fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result + pub(super) fn acceptor_intermediate(cert_path: &str, key_path: &str) -> Result where Self: Sized, { @@ -87,7 +72,7 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { }) } - fn acceptor_with_callbacks() -> Result + pub(super) fn acceptor_with_callbacks() -> Result where Self: Sized, { @@ -101,8 +86,4 @@ impl TlsAcceptorBuilder for TlsAcceptorBuil { None, )) } - - fn as_any(&mut self) -> &mut dyn Any { - self as &mut dyn Any - } } diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs index 249374be..e11f921d 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -17,12 +17,10 @@ use std::pin::Pin; use async_trait::async_trait; -use tokio::io::{AsyncRead, AsyncWrite}; - use pingora_error::ErrorType::{TLSHandshakeFailure, TLSWantX509Lookup}; use pingora_error::{OrErr, Result}; +use tokio::io::{AsyncRead, AsyncWrite}; -use crate::listeners::tls::Acceptor; use crate::protocols::tls::boringssl_openssl::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; use crate::protocols::{Ssl, IO}; @@ -58,15 +56,14 @@ impl ResumableAccept for TlsStream } } -fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { - let ssl_acceptor = acceptor.inner().downcast_ref::().unwrap(); - let ssl = ssl_from_acceptor(ssl_acceptor) +fn prepare_tls_stream(acceptor: &SslAcceptor, io: S) -> Result> { + let ssl = ssl_from_acceptor(&acceptor) .explain_err(TLSHandshakeFailure, |e| format!("ssl_acceptor error: {e}"))?; TlsStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(acceptor: &Acceptor, io: S) -> Result> { +pub(crate) async fn handshake(acceptor: &SslAcceptor, io: S) -> Result> { let mut stream = prepare_tls_stream(acceptor, io)?; stream .accept() @@ -76,8 +73,8 @@ pub async fn handshake(acceptor: &Acceptor, io: S) -> Result } /// Perform TLS handshake for the given connection with the given configuration and callbacks -pub async fn handshake_with_callback( - acceptor: &Acceptor, +pub(crate) async fn handshake_with_callback( + acceptor: &SslAcceptor, io: S, callbacks: &TlsAcceptCallbacks, ) -> pingora_error::Result> { diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs index 4dbae68e..c4536413 100644 --- a/pingora-core/src/protocols/tls/rustls/mod.rs +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -20,14 +20,13 @@ use std::task::{Context, Poll}; use pingora_error::ErrorType::{InternalError, TLSHandshakeFailure}; use pingora_error::{OrErr, Result}; -use pingora_rustls::TlsStream as RusTlsStream; use pingora_rustls::{hash_certificate, ServerName, TlsConnector}; +use pingora_rustls::{TlsAcceptor, TlsStream as RusTlsStream}; use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; use x509_parser::nom::AsBytes; use crate::utils::tls::rustls::get_organization_serial; -use crate::listeners::tls::Acceptor; use crate::protocols::tls::rustls::stream::InnerStream; use crate::protocols::tls::SslDigest; use crate::protocols::{Ssl, UniqueID, ALPN}; @@ -69,7 +68,7 @@ where /// /// Using RustTLS the stream is only returned after the handshake. /// The caller does therefor not need to perform [`Self::accept()`]. - pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { + pub(crate) async fn from_acceptor(acceptor: &TlsAcceptor, stream: T) -> Result { let tls = InnerStream::from_acceptor(acceptor, stream) .await .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs index d520dbbe..353ccde2 100644 --- a/pingora-core/src/protocols/tls/rustls/server.rs +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -14,13 +14,13 @@ //! Rustls TLS server specific implementation -use crate::listeners::tls::Acceptor; use crate::protocols::tls::rustls::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; use crate::protocols::IO; use async_trait::async_trait; use log::warn; use pingora_error::{ErrorType::*, OrErr, Result}; +use pingora_rustls::TlsAcceptor; use std::pin::Pin; use tokio::io::{AsyncRead, AsyncWrite}; @@ -48,14 +48,14 @@ impl ResumableAccept for TlsStream } } -async fn prepare_tls_stream(acceptor: &Acceptor, io: S) -> Result> { +async fn prepare_tls_stream(acceptor: &TlsAcceptor, io: S) -> Result> { TlsStream::from_acceptor(acceptor, io) .await .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } /// Perform TLS handshake for the given connection with the given configuration -pub async fn handshake(acceptor: &Acceptor, io: S) -> Result> { +pub(crate) async fn handshake(acceptor: &TlsAcceptor, io: S) -> Result> { let mut stream = prepare_tls_stream(acceptor, io).await?; stream .accept() @@ -66,8 +66,8 @@ pub async fn handshake(acceptor: &Acceptor, io: S) -> Result /// Perform TLS handshake for the given connection with the given configuration and callbacks /// callbacks are currently not supported within pingora Rustls and are ignored -pub async fn handshake_with_callback( - acceptor: &Acceptor, +pub(crate) async fn handshake_with_callback( + acceptor: &TlsAcceptor, io: S, _callbacks: &TlsAcceptCallbacks, ) -> Result> { diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index 358f79d9..7c55e3d5 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -16,14 +16,12 @@ use core::fmt; use core::fmt::Formatter; use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; use pingora_error::{Error, ImmutStr, OrErr, Result}; -use pingora_rustls::TlsAcceptor as RusTlsAcceptor; -use pingora_rustls::TlsStream as RusTlsStream; use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; +use pingora_rustls::{TlsAcceptor, TlsStream as RusTlsStream}; use std::fmt::Debug; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use crate::listeners::tls::Acceptor; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::tls::SslDigest; @@ -52,7 +50,8 @@ impl Debug for InnerStream { } else { &"None" } - }).finish() + }) + .finish() } } @@ -74,9 +73,8 @@ impl InnerStream { }) } - pub(crate) async fn from_acceptor(acceptor: &Acceptor, stream: T) -> Result { - let tls_acceptor = acceptor.inner().downcast_ref::().unwrap(); - let accept = tls_acceptor.accept(stream); + pub(crate) async fn from_acceptor(acceptor: &TlsAcceptor, stream: T) -> Result { + let accept = acceptor.accept(stream); Ok(InnerStream { accept: Some(accept), diff --git a/pingora/examples/server.rs b/pingora/examples/server.rs index e7924311..54d781d9 100644 --- a/pingora/examples/server.rs +++ b/pingora/examples/server.rs @@ -141,14 +141,15 @@ pub fn main() { // NOTE: dynamic certificate callback is only supported with BoringSSL/OpenSSL #[cfg(not(feature = "rustls"))] { - use pingora_core::listeners::tls::NativeBuilder; + use std::ops::DerefMut; let dynamic_cert = boringssl_openssl::DynamicCert::new(&cert_path, &key_path); tls_settings = pingora::listeners::TlsSettings::with_callbacks(dynamic_cert).unwrap(); // by default intermediate supports both TLS 1.2 and 1.3. We force to tls 1.2 just for the demo + tls_settings - .get_builder() - .native() + .deref_mut() + .deref_mut() .set_max_proto_version(Some(pingora::tls::ssl::SslVersion::TLS1_2)) .unwrap(); } From 90a823a3ce4026bc13086056b589310b4a5bed9c Mon Sep 17 00:00:00 2001 From: Harald Gutmann Date: Tue, 10 Sep 2024 11:40:39 +0200 Subject: [PATCH 10/10] fix: remove indirections in protocols:tls:TlsStream --- .../protocols/tls/boringssl_openssl/client.rs | 2 +- .../protocols/tls/boringssl_openssl/mod.rs | 153 ------------ .../protocols/tls/boringssl_openssl/server.rs | 9 +- .../protocols/tls/boringssl_openssl/stream.rs | 174 ++++++++++++-- pingora-core/src/protocols/tls/mod.rs | 115 +--------- .../src/protocols/tls/rustls/client.rs | 6 +- pingora-core/src/protocols/tls/rustls/mod.rs | 191 --------------- .../src/protocols/tls/rustls/server.rs | 2 +- .../src/protocols/tls/rustls/stream.rs | 217 +++++++++++++++--- 9 files changed, 346 insertions(+), 523 deletions(-) diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs index 7dbf96d2..37d2160b 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/client.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/client.rs @@ -16,7 +16,7 @@ use pingora_error::{Error, ErrorType::*, OrErr, Result}; -use crate::protocols::tls::boringssl_openssl::TlsStream; +use crate::protocols::tls::TlsStream; use crate::protocols::IO; use crate::tls::ssl::ConnectConfiguration; diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs index 796f11fc..a4c52fb5 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/mod.rs @@ -13,159 +13,6 @@ // limitations under the License. //! BoringSSL & OpenSSL TLS specific implementation - -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; - -use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; - -use pingora_error::ErrorType::TLSHandshakeFailure; -use pingora_error::{OrErr, Result}; - -use crate::protocols::tls::boringssl_openssl::stream::InnerStream; -use crate::protocols::tls::SslDigest; -use crate::protocols::{Ssl, UniqueID, ALPN}; -use crate::tls::hash::MessageDigest; -use crate::tls::ssl; -use crate::tls::ssl::SslRef; -use crate::utils::tls::boringssl_openssl::{get_x509_organization, get_x509_serial}; - -use super::TlsStream; - pub mod client; pub mod server; pub(super) mod stream; - -impl TlsStream -where - T: AsyncRead + AsyncWrite + Unpin + Send, -{ - /// Create a new TLS connection from the given `stream` - /// - /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS - /// handshake after. - pub fn new(ssl: ssl::Ssl, stream: T) -> Result { - let tls = InnerStream::new(ssl, stream) - .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; - Ok(TlsStream { - tls, - digest: None, - timing: Default::default(), - }) - } -} - -impl AsyncRead for TlsStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Self::clear_error(&self); - Pin::new(&mut self.tls.0).poll_read(cx, buf) - } -} - -impl TlsStream { - #[inline] - fn clear_error(&self) { - InnerStream::::clear_error() - } -} - -impl AsyncWrite for TlsStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context, - buf: &[u8], - ) -> Poll> { - Self::clear_error(&self); - Pin::new(&mut self.tls.0).poll_write(cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(&self); - Pin::new(&mut self.tls.0).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Self::clear_error(&self); - Pin::new(&mut self.tls.0).poll_shutdown(cx) - } - - fn poll_write_vectored( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Self::clear_error(&self); - Pin::new(&mut self.tls.0).poll_write_vectored(cx, bufs) - } - - fn is_write_vectored(&self) -> bool { - true - } -} - -impl UniqueID for TlsStream -where - T: UniqueID, -{ - fn id(&self) -> i32 { - self.tls.0.get_ref().id() - } -} - -impl Ssl for TlsStream { - fn get_ssl(&self) -> Option<&ssl::SslRef> { - Some(self.tls.0.ssl()) - } - - fn get_ssl_digest(&self) -> Option> { - self.ssl_digest() - } - - fn selected_alpn_proto(&self) -> Option { - let ssl = self.tls.0.ssl(); - ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) - } -} - -impl SslDigest { - pub fn from_ssl(ssl: &SslRef) -> Self { - let cipher = match ssl.current_cipher() { - Some(c) => c.name(), - None => "", - }; - - let (cert_digest, org, sn) = match ssl.peer_certificate() { - Some(cert) => { - let cert_digest = match cert.digest(MessageDigest::sha256()) { - Ok(c) => c.as_ref().to_vec(), - Err(_) => Vec::new(), - }; - ( - cert_digest, - get_x509_organization(&cert), - get_x509_serial(&cert).ok(), - ) - } - None => (Vec::new(), None, None), - }; - - SslDigest { - cipher, - version: ssl.version_str(), - organization: org, - serial_number: sn, - cert_digest, - } - } -} diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs index e11f921d..6270614d 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/server.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/server.rs @@ -14,6 +14,7 @@ //! BoringSSL & OpenSSL TLS server specific implementation +use crate::protocols::Ssl; use std::pin::Pin; use async_trait::async_trait; @@ -21,9 +22,9 @@ use pingora_error::ErrorType::{TLSHandshakeFailure, TLSWantX509Lookup}; use pingora_error::{OrErr, Result}; use tokio::io::{AsyncRead, AsyncWrite}; -use crate::protocols::tls::boringssl_openssl::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; -use crate::protocols::{Ssl, IO}; +use crate::protocols::tls::TlsStream; +use crate::protocols::IO; use crate::tls::ext; use crate::tls::ext::ssl_from_acceptor; use crate::tls::ssl::SslAcceptor; @@ -57,7 +58,7 @@ impl ResumableAccept for TlsStream } fn prepare_tls_stream(acceptor: &SslAcceptor, io: S) -> Result> { - let ssl = ssl_from_acceptor(&acceptor) + let ssl = ssl_from_acceptor(acceptor) .explain_err(TLSHandshakeFailure, |e| format!("ssl_acceptor error: {e}"))?; TlsStream::new(ssl, io).explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}")) } @@ -82,7 +83,7 @@ pub(crate) async fn handshake_with_callback( let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { // safety: we do hold a mut ref of tls_stream - let ssl_mut = unsafe { ext::ssl_mut(tls_stream.0.ssl()) }; + let ssl_mut = unsafe { ext::ssl_mut(tls_stream.stream.ssl()) }; callbacks.certificate_callback(ssl_mut).await; Pin::new(&mut tls_stream) .resume_accept() diff --git a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs index 89a8dca6..700a4b22 100644 --- a/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs +++ b/pingora-core/src/protocols/tls/boringssl_openssl/stream.rs @@ -14,36 +14,47 @@ //! BoringSSL & OpenSSL TLS stream specific implementation +use crate::tls::hash::MessageDigest; use log::warn; use std::pin::Pin; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite}; - -use pingora_error::{Error, ErrorType::*, OrErr, Result}; +use std::task::{Context, Poll}; +use std::time::SystemTime; +use tokio::io; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use crate::listeners::ALPN; use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; use crate::protocols::raw_connect::ProxyDigest; use crate::protocols::tls::SslDigest; -use crate::protocols::{GetProxyDigest, GetTimingDigest}; +use crate::protocols::{GetProxyDigest, GetTimingDigest, Ssl, UniqueID}; use crate::tls::error::ErrorStack; use crate::tls::ext; -use crate::tls::tokio_ssl; use crate::tls::tokio_ssl::SslStream; use crate::tls::{ssl, ssl::SslRef, ssl_sys::X509_V_ERR_INVALID_CALL}; +use pingora_error::{Error, ErrorType::*, OrErr, Result}; #[derive(Debug)] -pub struct InnerStream(pub(crate) tokio_ssl::SslStream); +pub struct TlsStream { + pub(crate) stream: SslStream, + pub(super) digest: Option>, + pub(super) timing: TimingDigest, +} -impl InnerStream { +impl TlsStream { /// Create a new TLS connection from the given `stream` /// /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS /// handshake after. pub(crate) fn new(ssl: ssl::Ssl, stream: T) -> Result { - let ssl = SslStream::new(ssl, stream) + let stream = SslStream::new(ssl, stream) .explain_err(TLSHandshakeFailure, |e| format!("tls.rs stream error: {e}"))?; - Ok(InnerStream(ssl)) + Ok(TlsStream { + stream, + timing: Default::default(), + digest: None, + }) } #[inline] @@ -55,12 +66,16 @@ impl InnerStream { } } -impl InnerStream { +impl TlsStream { /// Connect to the remote TLS server as a client pub(crate) async fn connect(&mut self) -> Result<()> { Self::clear_error(); - match Pin::new(&mut self.0).connect().await { - Ok(_) => Ok(()), + match Pin::new(&mut self.stream).connect().await { + Ok(_) => { + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } Err(err) => self.transform_ssl_error(err), } } @@ -68,18 +83,20 @@ impl InnerStream { /// Finish the TLS handshake from client as a server pub(crate) async fn accept(&mut self) -> Result<()> { Self::clear_error(); - match Pin::new(&mut self.0).accept().await { - Ok(_) => Ok(()), + match Pin::new(&mut self.stream).accept().await { + Ok(_) => { + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); + Ok(()) + } Err(err) => self.transform_ssl_error(err), } } pub(crate) fn digest(&mut self) -> Option> { - Some(Arc::new(SslDigest::from_ssl(self.0.ssl()))) + Some(Arc::new(SslDigest::from_ssl(self.stream.ssl()))) } -} -impl InnerStream { fn transform_ssl_error(&self, e: ssl::Error) -> Result<()> { let context = format!("ssl::ErrorCode: {:?}", e.code()); if ext::is_suspended_for_cert(&e) { @@ -102,7 +119,7 @@ impl InnerStream { ssl.verify_result().map_err(|e| e.as_raw()) } - match verify_result(self.0.ssl()) { + match verify_result(self.stream.ssl()) { Ok(()) => Error::e_explain(TLSHandshakeFailure, context), // X509_V_ERR_INVALID_CALL in case verify result was never set Err(X509_V_ERR_INVALID_CALL) => { @@ -118,32 +135,139 @@ impl InnerStream { } } -impl GetSocketDigest for InnerStream +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Self::clear_error(); + Pin::new(&mut self.stream).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl Ssl for TlsStream { + fn get_ssl(&self) -> Option<&ssl::SslRef> { + Some(self.stream.ssl()) + } + + fn get_ssl_digest(&self) -> Option> { + self.digest.clone() + } + + fn selected_alpn_proto(&self) -> Option { + let ssl = self.stream.ssl(); + ALPN::from_wire_selected(ssl.selected_alpn_protocol()?) + } +} + +impl SslDigest { + pub fn from_ssl(ssl: &SslRef) -> Self { + let cipher = match ssl.current_cipher() { + Some(c) => c.name(), + None => "", + }; + + let (cert_digest, org, sn) = match ssl.peer_certificate() { + Some(cert) => { + let cert_digest = match cert.digest(MessageDigest::sha256()) { + Ok(c) => c.as_ref().to_vec(), + Err(_) => Vec::new(), + }; + ( + cert_digest, + crate::utils::tls::boringssl_openssl::get_x509_organization(&cert), + crate::utils::tls::boringssl_openssl::get_x509_serial(&cert).ok(), + ) + } + None => (Vec::new(), None, None), + }; + + SslDigest { + cipher, + version: ssl.version_str(), + organization: org, + serial_number: sn, + cert_digest, + } + } +} + +impl GetSocketDigest for TlsStream where S: GetSocketDigest, { fn get_socket_digest(&self) -> Option> { - self.0.get_ref().get_socket_digest() + self.stream.get_ref().get_socket_digest() } fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - self.0.get_mut().set_socket_digest(socket_digest) + self.stream.get_mut().set_socket_digest(socket_digest) } } -impl GetTimingDigest for InnerStream +impl GetTimingDigest for TlsStream where S: GetTimingDigest, { fn get_timing_digest(&self) -> Vec> { - self.0.get_ref().get_timing_digest() + self.stream.get_ref().get_timing_digest() } } -impl GetProxyDigest for InnerStream +impl GetProxyDigest for TlsStream where S: GetProxyDigest, { fn get_proxy_digest(&self) -> Option> { - self.0.get_ref().get_proxy_digest() + self.stream.get_ref().get_proxy_digest() + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.stream.get_ref().id() } } diff --git a/pingora-core/src/protocols/tls/mod.rs b/pingora-core/src/protocols/tls/mod.rs index 9eaf3afe..53c3ac87 100644 --- a/pingora-core/src/protocols/tls/mod.rs +++ b/pingora-core/src/protocols/tls/mod.rs @@ -14,17 +14,6 @@ //! The TLS layer implementations -use async_trait::async_trait; -use pingora_error::Result; -use std::ops::{Deref, DerefMut}; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use tokio::io::{AsyncRead, AsyncWrite}; - -use crate::protocols::digest::TimingDigest; -use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::{GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest}; - #[cfg(not(feature = "rustls"))] pub(crate) mod boringssl_openssl; #[cfg(feature = "rustls")] @@ -32,29 +21,9 @@ pub(crate) mod rustls; pub mod server; #[cfg(not(feature = "rustls"))] -use boringssl_openssl::stream::InnerStream; +pub(crate) use boringssl_openssl::stream::TlsStream; #[cfg(feature = "rustls")] -use rustls::stream::InnerStream; - -/// The TLS connection -#[derive(Debug)] -pub struct TlsStream { - tls: InnerStream, - digest: Option>, - timing: TimingDigest, -} - -// NOTE: keeping trait for documentation purpose -// switched to direct implementations to eliminate redirections in within the call-graph -// the below trait is required for InnerStream to be implemented -#[async_trait] -pub trait InnerTlsStream { - async fn connect(&mut self) -> Result<()>; - async fn accept(&mut self) -> Result<()>; - - /// Return the [`ssl::SslDigest`] for logging - fn digest(&mut self) -> Option>; -} +pub(crate) use rustls::stream::TlsStream; /// The protocol for Application-Layer Protocol Negotiation #[derive(Hash, Clone, Debug)] @@ -148,83 +117,3 @@ pub struct SslDigest { /// The digest of the peer's certificate pub cert_digest: Vec, } - -impl GetSocketDigest for TlsStream -where - S: GetSocketDigest, -{ - fn get_socket_digest(&self) -> Option> { - self.tls.get_socket_digest() - } - fn set_socket_digest(&mut self, socket_digest: SocketDigest) { - self.tls.set_socket_digest(socket_digest) - } -} - -impl GetTimingDigest for TlsStream -where - S: GetTimingDigest, -{ - fn get_timing_digest(&self) -> Vec> { - let mut ts_vec = self.tls.get_timing_digest(); - ts_vec.push(Some(self.timing.clone())); - ts_vec - } - fn get_read_pending_time(&self) -> Duration { - self.tls.get_read_pending_time() - } - - fn get_write_pending_time(&self) -> Duration { - self.tls.get_write_pending_time() - } -} - -impl GetProxyDigest for TlsStream -where - S: GetProxyDigest, -{ - fn get_proxy_digest(&self) -> Option> { - self.tls.get_proxy_digest() - } -} - -impl TlsStream { - pub fn ssl_digest(&self) -> Option> { - self.digest.clone() - } -} - -impl TlsStream -where - T: AsyncRead + AsyncWrite + Unpin + Send, -{ - /// Connect to the remote TLS server as a client - pub(crate) async fn connect(&mut self) -> Result<()> { - self.tls.connect().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = self.tls.digest(); - Ok(()) - } - - /// Finish the TLS handshake from client as a server - pub(crate) async fn accept(&mut self) -> Result<()> { - self.tls.accept().await?; - self.timing.established_ts = SystemTime::now(); - self.digest = self.tls.digest(); - Ok(()) - } -} - -impl Deref for TlsStream { - type Target = InnerStream; - - fn deref(&self) -> &Self::Target { - &self.tls - } -} - -impl DerefMut for TlsStream { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.tls - } -} diff --git a/pingora-core/src/protocols/tls/rustls/client.rs b/pingora-core/src/protocols/tls/rustls/client.rs index bb9edc27..3b66154f 100644 --- a/pingora-core/src/protocols/tls/rustls/client.rs +++ b/pingora-core/src/protocols/tls/rustls/client.rs @@ -14,7 +14,7 @@ //! Rustls TLS client specific implementation -use crate::protocols::tls::rustls::TlsStream; +use crate::protocols::tls::rustls::stream::TlsStream; use crate::protocols::IO; use pingora_error::ErrorType::TLSHandshakeFailure; use pingora_error::{Error, OrErr, Result}; @@ -28,9 +28,7 @@ pub async fn handshake( ) -> Result> { let mut stream = TlsStream::from_connector(connector, domain, io) .await - .explain_err(TLSHandshakeFailure, |e| { - format!("tip: tls stream error: {e}") - })?; + .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; let handshake_result = stream.connect().await; match handshake_result { diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs index c4536413..66e97d3d 100644 --- a/pingora-core/src/protocols/tls/rustls/mod.rs +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -13,197 +13,6 @@ // limitations under the License. //! Rustls TLS specific implementation - -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; - -use pingora_error::ErrorType::{InternalError, TLSHandshakeFailure}; -use pingora_error::{OrErr, Result}; -use pingora_rustls::{hash_certificate, ServerName, TlsConnector}; -use pingora_rustls::{TlsAcceptor, TlsStream as RusTlsStream}; -use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; -use x509_parser::nom::AsBytes; - -use crate::utils::tls::rustls::get_organization_serial; - -use crate::protocols::tls::rustls::stream::InnerStream; -use crate::protocols::tls::SslDigest; -use crate::protocols::{Ssl, UniqueID, ALPN}; - -use super::TlsStream; - pub mod client; pub mod server; pub(super) mod stream; - -impl TlsStream -where - T: AsyncRead + AsyncWrite + Unpin + Send, -{ - /// Create a new TLS connection from the given `stream` - /// - /// Using RustTLS the stream is only returned after the handshake. - /// The caller does therefor not need to perform [`Self::connect()`]. - pub async fn from_connector(connector: &TlsConnector, domain: &str, stream: T) -> Result { - let server = ServerName::try_from(domain) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) - .explain_err(InternalError, |e| { - format!("failed to parse domain: {}, error: {}", domain, e) - })? - .to_owned(); - - let tls = InnerStream::from_connector(connector, server, stream) - .await - .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; - - Ok(TlsStream { - tls, - digest: None, - timing: Default::default(), - }) - } - - /// Create a new TLS connection from the given `stream` - /// - /// Using RustTLS the stream is only returned after the handshake. - /// The caller does therefor not need to perform [`Self::accept()`]. - pub(crate) async fn from_acceptor(acceptor: &TlsAcceptor, stream: T) -> Result { - let tls = InnerStream::from_acceptor(acceptor, stream) - .await - .explain_err(TLSHandshakeFailure, |e| format!("tls stream error: {e}"))?; - - Ok(TlsStream { - tls, - digest: None, - timing: Default::default(), - }) - } -} - -impl AsyncRead for TlsStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_read(cx, buf) - } -} - -impl AsyncWrite for TlsStream -where - T: AsyncRead + AsyncWrite + Unpin, -{ - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context, - buf: &[u8], - ) -> Poll> { - Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_write(cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_shutdown(cx) - } - - fn poll_write_vectored( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Pin::new(&mut self.tls.stream.as_mut().unwrap()).poll_write_vectored(cx, bufs) - } - - fn is_write_vectored(&self) -> bool { - true - } -} - -impl UniqueID for TlsStream -where - T: UniqueID, -{ - fn id(&self) -> i32 { - self.tls.stream.as_ref().unwrap().get_ref().0.id() - } -} - -impl Ssl for TlsStream { - fn get_ssl_digest(&self) -> Option> { - self.ssl_digest() - } - - fn selected_alpn_proto(&self) -> Option { - let st = self.tls.stream.as_ref(); - if let Some(stream) = st { - let proto = stream.get_ref().1.alpn_protocol(); - match proto { - None => None, - Some(raw) => ALPN::from_wire_selected(raw), - } - } else { - None - } - } -} - -impl SslDigest { - fn from_stream(stream: &Option>) -> Self { - let stream = stream.as_ref().unwrap(); - let (_io, session) = stream.get_ref(); - let protocol = session.protocol_version(); - let cipher_suite = session.negotiated_cipher_suite(); - let peer_certificates = session.peer_certificates(); - - let cipher = match cipher_suite { - Some(suite) => match suite.suite().as_str() { - Some(suite_str) => suite_str, - None => "", - }, - None => "", - }; - - let version = match protocol { - Some(proto) => match proto.as_str() { - Some(ver) => ver, - None => "", - }, - None => "", - }; - - let cert_digest = match peer_certificates { - Some(certs) => match certs.first() { - Some(cert) => hash_certificate(cert.clone()), - None => vec![], - }, - None => vec![], - }; - - let (organization, serial_number) = match peer_certificates { - Some(certs) => match certs.first() { - Some(cert) => { - let (organization, serial) = get_organization_serial(cert.as_bytes()); - (organization, Some(serial)) - } - None => (None, None), - }, - None => (None, None), - }; - - SslDigest { - cipher: &cipher, - version: &version, - organization, - serial_number, - cert_digest, - } - } -} diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs index 353ccde2..98597394 100644 --- a/pingora-core/src/protocols/tls/rustls/server.rs +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -14,7 +14,7 @@ //! Rustls TLS server specific implementation -use crate::protocols::tls::rustls::TlsStream; +use crate::protocols::tls::rustls::stream::TlsStream; use crate::protocols::tls::server::{ResumableAccept, TlsAcceptCallbacks}; use crate::protocols::IO; use async_trait::async_trait; diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index 7c55e3d5..f6c9f3be 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -12,28 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::listeners::ALPN; +use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; +use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::SslDigest; +use crate::protocols::{GetProxyDigest, GetTimingDigest, Ssl, UniqueID}; +use crate::utils::tls::rustls::get_organization_serial; use core::fmt; use core::fmt::Formatter; -use pingora_error::ErrorType::{AcceptError, ConnectError, TLSHandshakeFailure}; +use pingora_error::ErrorType::{AcceptError, ConnectError, InternalError, TLSHandshakeFailure}; use pingora_error::{Error, ImmutStr, OrErr, Result}; -use pingora_rustls::{Accept, Connect, ServerName, TlsConnector}; +use pingora_rustls::{hash_certificate, Accept, Connect, ServerName, TlsConnector}; use pingora_rustls::{TlsAcceptor, TlsStream as RusTlsStream}; use std::fmt::Debug; +use std::pin::Pin; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite}; +use std::task::{Context, Poll}; +use std::time::SystemTime; +use tokio::io; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use x509_parser::nom::AsBytes; -use crate::protocols::digest::{GetSocketDigest, SocketDigest, TimingDigest}; -use crate::protocols::raw_connect::ProxyDigest; -use crate::protocols::tls::SslDigest; -use crate::protocols::{GetProxyDigest, GetTimingDigest}; - -pub struct InnerStream { +/// The TLS connection +pub struct TlsStream { pub(crate) stream: Option>, connect: Option>, accept: Option>, + pub(super) digest: Option>, + pub(super) timing: TimingDigest, } -impl Debug for InnerStream { +impl Debug for TlsStream { fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { fmt.debug_struct("InnerStream") .field("stream", &self.stream) @@ -55,35 +64,50 @@ impl Debug for InnerStream { } } -impl InnerStream { +impl TlsStream { /// Create a new TLS connection from the given `stream` /// - /// The caller needs to perform [`Self::connect()`] or [`Self::accept()`] to perform TLS - /// handshake after. - pub(crate) async fn from_connector( - connector: &TlsConnector, - server: ServerName<'_>, - stream: T, - ) -> Result { - let connect = connector.connect(server.to_owned(), stream); - Ok(InnerStream { - accept: None, - connect: Some(connect), - stream: None, - }) - } - + /// Using RustTLS the stream is only returned after the handshake. + /// + /// The caller needs to perform [`Self::accept()`] to perform the TLS handshake. pub(crate) async fn from_acceptor(acceptor: &TlsAcceptor, stream: T) -> Result { let accept = acceptor.accept(stream); - Ok(InnerStream { + Ok(TlsStream { accept: Some(accept), + digest: None, connect: None, stream: None, + timing: Default::default(), + }) + } + + /// Create a new TLS connection from the given `stream` + /// + /// Using RustTLS the stream is only returned after the handshake. + /// + /// The caller needs to perform [`Self::connect()`] to perform the TLS handshake. + pub async fn from_connector(connector: &TlsConnector, domain: &str, stream: T) -> Result { + let server = ServerName::try_from(domain) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) + .explain_err(InternalError, |e| { + format!("failed to parse domain: {}, error: {}", domain, e) + })? + .to_owned(); + + let connect = connector.connect(server.to_owned(), stream); + + Ok(TlsStream { + accept: None, + digest: None, + connect: Some(connect), + stream: None, + timing: Default::default(), }) } } -impl InnerStream { + +impl TlsStream { /// Connect to the remote TLS server as a client pub(crate) async fn connect(&mut self) -> Result<()> { let connect = &mut self.connect; @@ -95,6 +119,8 @@ impl InnerStream { self.stream = Some(RusTlsStream::Client(stream)); self.connect = None; + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); Ok(()) } else { Err(Error::explain( @@ -114,6 +140,8 @@ impl InnerStream { self.stream = Some(RusTlsStream::Server(stream)); self.connect = None; + self.timing.established_ts = SystemTime::now(); + self.digest = self.digest(); Ok(()) } else { Err(Error::explain( @@ -128,7 +156,7 @@ impl InnerStream { } } -impl GetSocketDigest for InnerStream +impl GetSocketDigest for TlsStream where S: GetSocketDigest, { @@ -149,7 +177,7 @@ where } } -impl GetTimingDigest for InnerStream +impl GetTimingDigest for TlsStream where S: GetTimingDigest, { @@ -163,7 +191,7 @@ where } } -impl GetProxyDigest for InnerStream +impl GetProxyDigest for TlsStream where S: GetProxyDigest, { @@ -175,3 +203,130 @@ where } } } + +impl AsyncRead for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsStream +where + T: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.stream.as_mut().unwrap()).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl Ssl for TlsStream { + fn get_ssl_digest(&self) -> Option> { + self.digest.clone() + } + + fn selected_alpn_proto(&self) -> Option { + let st = self.stream.as_ref(); + if let Some(stream) = st { + let proto = stream.get_ref().1.alpn_protocol(); + match proto { + None => None, + Some(raw) => ALPN::from_wire_selected(raw), + } + } else { + None + } + } +} + +impl SslDigest { + fn from_stream(stream: &Option>) -> Self { + let stream = stream.as_ref().unwrap(); + let (_io, session) = stream.get_ref(); + let protocol = session.protocol_version(); + let cipher_suite = session.negotiated_cipher_suite(); + let peer_certificates = session.peer_certificates(); + + let cipher = match cipher_suite { + Some(suite) => match suite.suite().as_str() { + Some(suite_str) => suite_str, + None => "", + }, + None => "", + }; + + let version = match protocol { + Some(proto) => match proto.as_str() { + Some(ver) => ver, + None => "", + }, + None => "", + }; + + let cert_digest = match peer_certificates { + Some(certs) => match certs.first() { + Some(cert) => hash_certificate(cert.clone()), + None => vec![], + }, + None => vec![], + }; + + let (organization, serial_number) = match peer_certificates { + Some(certs) => match certs.first() { + Some(cert) => { + let (organization, serial) = get_organization_serial(cert.as_bytes()); + (organization, Some(serial)) + } + None => (None, None), + }, + None => (None, None), + }; + + SslDigest { + cipher: &cipher, + version: &version, + organization, + serial_number, + cert_digest, + } + } +} + +impl UniqueID for TlsStream +where + T: UniqueID, +{ + fn id(&self) -> i32 { + self.stream.as_ref().unwrap().get_ref().0.id() + } +}