From 6ef949069588d8b9677cfa15dfe4b5a82b4d7779 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 25 Jun 2026 17:05:52 -0700 Subject: [PATCH 01/59] feat(middleware): add in-process supervisor middleware Signed-off-by: Piotr Mlocek --- Cargo.lock | 15 + Cargo.toml | 1 + crates/openshell-core/src/proto/mod.rs | 14 + crates/openshell-policy/Cargo.toml | 1 + crates/openshell-policy/src/compose.rs | 1 + crates/openshell-policy/src/lib.rs | 220 +++++- crates/openshell-policy/src/merge.rs | 24 + crates/openshell-providers/src/profiles.rs | 2 + .../Cargo.toml | 25 + .../src/builtins/mod.rs | 4 + .../src/builtins/secrets.rs | 83 +++ .../src/lib.rs | 437 ++++++++++++ .../src/service.rs | 75 ++ .../openshell-supervisor-network/Cargo.toml | 2 + .../data/sandbox-policy.rego | 22 +- .../src/l7/relay.rs | 478 ++++++++++++- .../src/l7/rest.rs | 122 ++++ .../openshell-supervisor-network/src/opa.rs | 638 +++++++++++++++++- .../src/policy_local.rs | 5 + .../openshell-supervisor-network/src/proxy.rs | 2 + proto/middleware.proto | 95 +++ proto/sandbox.proto | 29 +- 22 files changed, 2278 insertions(+), 17 deletions(-) create mode 100644 crates/openshell-supervisor-middleware/Cargo.toml create mode 100644 crates/openshell-supervisor-middleware/src/builtins/mod.rs create mode 100644 crates/openshell-supervisor-middleware/src/builtins/secrets.rs create mode 100644 crates/openshell-supervisor-middleware/src/lib.rs create mode 100644 crates/openshell-supervisor-middleware/src/service.rs create mode 100644 proto/middleware.proto diff --git a/Cargo.lock b/Cargo.lock index 13b670f558..6f664acbad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3771,6 +3771,7 @@ version = "0.0.0" dependencies = [ "miette", "openshell-core", + "prost-types", "serde", "serde_json", "serde_yml", @@ -3926,6 +3927,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "openshell-supervisor-middleware" +version = "0.0.0" +dependencies = [ + "miette", + "openshell-core", + "prost-types", + "regex", + "tokio", + "tonic", +] + [[package]] name = "openshell-supervisor-network" version = "0.0.0" @@ -3948,6 +3961,8 @@ dependencies = [ "openshell-ocsf", "openshell-policy", "openshell-router", + "openshell-supervisor-middleware", + "prost-types", "rcgen", "regorus", "reqwest 0.12.28", diff --git a/Cargo.toml b/Cargo.toml index f450cd5c8c..fd3641d681 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,7 @@ serde_yml = "0.0.12" toml = "0.8" apollo-parser = "0.8.5" tower-mcp-types = "0.12.0" +regex = "1" # HTTP client reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } diff --git a/crates/openshell-core/src/proto/mod.rs b/crates/openshell-core/src/proto/mod.rs index 08b062d2e5..1ac6fc94c4 100644 --- a/crates/openshell-core/src/proto/mod.rs +++ b/crates/openshell-core/src/proto/mod.rs @@ -79,8 +79,22 @@ pub mod inference { } } +#[allow( + clippy::all, + clippy::pedantic, + clippy::nursery, + unused_qualifications, + rust_2018_idioms +)] +pub mod middleware { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/openshell.middleware.v1.rs")); + } +} + pub use datamodel::v1::*; pub use inference::v1::*; +pub use middleware::v1::*; pub use openshell::*; pub use sandbox::v1::*; pub use test::ObjectForTest; diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 16719de13d..50bea5b32e 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +prost-types = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yml = { workspace = true } diff --git a/crates/openshell-policy/src/compose.rs b/crates/openshell-policy/src/compose.rs index 7ca8584d9d..1ad0d46171 100644 --- a/crates/openshell-policy/src/compose.rs +++ b/crates/openshell-policy/src/compose.rs @@ -115,6 +115,7 @@ mod tests { ..Default::default() }], binaries: Vec::new(), + middleware: Vec::new(), } } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index f1721146ed..0aa43c30d7 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -19,8 +19,8 @@ use std::path::Path; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ FilesystemPolicy, GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, - LandlockPolicy, McpOptions, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, ProcessPolicy, - SandboxPolicy, + LandlockPolicy, MiddlewareEndpointSelector, NetworkBinary, NetworkEndpoint, + NetworkMiddlewareConfig, NetworkPolicyRule, ProcessPolicy, SandboxPolicy, McpOptions, }; use serde::{Deserialize, Serialize}; @@ -49,6 +49,8 @@ struct PolicyFile { process: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] network_policies: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + network_middlewares: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -87,6 +89,30 @@ struct NetworkPolicyRuleDef { endpoints: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] binaries: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + middleware: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct NetworkMiddlewareConfigDef { + name: String, + middleware: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + config: BTreeMap, + #[serde(default, skip_serializing_if = "String::is_empty")] + on_error: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + endpoints: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MiddlewareEndpointSelectorDef { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + include: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + exclude: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -148,6 +174,8 @@ struct NetworkEndpointDef { json_rpc: Option, #[serde(default, skip_serializing_if = "Option::is_none")] mcp: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + middleware: Vec, } // Signature dictated by serde's `skip_serializing_if`, which requires `&T`. @@ -672,6 +700,21 @@ fn yaml_mcp_method( } fn to_proto(raw: PolicyFile) -> SandboxPolicy { + let network_middlewares = raw + .network_middlewares + .into_iter() + .map(|mw| NetworkMiddlewareConfig { + name: mw.name, + middleware: mw.middleware, + config: Some(json_map_to_struct(mw.config)), + on_error: mw.on_error, + endpoints: mw.endpoints.map(|selector| MiddlewareEndpointSelector { + include: selector.include, + exclude: selector.exclude, + }), + }) + .collect(); + let network_policies = raw .network_policies .into_iter() @@ -745,6 +788,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { signing_region: e.signing_region, json_rpc_max_body_bytes: json_rpc_max_body_bytes(&e.json_rpc, &e.mcp), mcp: mcp_options(&e.mcp), + middleware: e.middleware, } }) .collect(), @@ -756,6 +800,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { ..Default::default() }) .collect(), + middleware: rule.middleware, }; (key, proto_rule) }) @@ -776,6 +821,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { run_as_group: p.run_as_group, }), network_policies, + network_middlewares, } } @@ -892,6 +938,7 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { signing_region: e.signing_region.clone(), json_rpc, mcp, + middleware: e.middleware.clone(), } }) .collect(), @@ -903,17 +950,103 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { harness: false, }) .collect(), + middleware: rule.middleware.clone(), }; (key.clone(), yaml_rule) }) .collect(); + let network_middlewares = policy + .network_middlewares + .iter() + .map(|mw| NetworkMiddlewareConfigDef { + name: mw.name.clone(), + middleware: mw.middleware.clone(), + config: mw + .config + .as_ref() + .map(struct_to_json_map) + .unwrap_or_default(), + on_error: mw.on_error.clone(), + endpoints: mw + .endpoints + .as_ref() + .map(|selector| MiddlewareEndpointSelectorDef { + include: selector.include.clone(), + exclude: selector.exclude.clone(), + }), + }) + .collect(); + PolicyFile { version: policy.version, filesystem_policy, landlock, process, network_policies, + network_middlewares, + } +} + +fn json_map_to_struct(map: BTreeMap) -> prost_types::Struct { + prost_types::Struct { + fields: map + .into_iter() + .map(|(key, value)| (key, json_to_protobuf_value(value))) + .collect(), + } +} + +fn json_to_protobuf_value(value: serde_json::Value) -> prost_types::Value { + use prost_types::{ListValue, Struct, Value, value::Kind}; + Value { + kind: Some(match value { + serde_json::Value::Null => Kind::NullValue(0), + serde_json::Value::Bool(value) => Kind::BoolValue(value), + serde_json::Value::Number(value) => { + Kind::NumberValue(value.as_f64().unwrap_or_default()) + } + serde_json::Value::String(value) => Kind::StringValue(value), + serde_json::Value::Array(values) => Kind::ListValue(ListValue { + values: values.into_iter().map(json_to_protobuf_value).collect(), + }), + serde_json::Value::Object(values) => Kind::StructValue(Struct { + fields: values + .into_iter() + .map(|(key, value)| (key, json_to_protobuf_value(value))) + .collect(), + }), + }), + } +} + +fn struct_to_json_map(config: &prost_types::Struct) -> BTreeMap { + config + .fields + .iter() + .map(|(key, value)| (key.clone(), protobuf_value_to_json(value))) + .collect() +} + +fn protobuf_value_to_json(value: &prost_types::Value) -> serde_json::Value { + match value.kind.as_ref() { + Some(prost_types::value::Kind::NullValue(_)) | None => serde_json::Value::Null, + Some(prost_types::value::Kind::BoolValue(value)) => serde_json::Value::Bool(*value), + Some(prost_types::value::Kind::NumberValue(value)) => serde_json::Number::from_f64(*value) + .map_or(serde_json::Value::Null, serde_json::Value::Number), + Some(prost_types::value::Kind::StringValue(value)) => { + serde_json::Value::String(value.clone()) + } + Some(prost_types::value::Kind::ListValue(value)) => { + serde_json::Value::Array(value.values.iter().map(protobuf_value_to_json).collect()) + } + Some(prost_types::value::Kind::StructValue(value)) => serde_json::Value::Object( + value + .fields + .iter() + .map(|(key, value)| (key.clone(), protobuf_value_to_json(value))) + .collect(), + ), } } @@ -1064,6 +1197,7 @@ pub fn restrictive_default_policy() -> SandboxPolicy { run_as_group: "sandbox".into(), }), network_policies: HashMap::new(), + network_middlewares: vec![], } } @@ -1438,6 +1572,87 @@ network_policies: assert_eq!(proto2.network_policies["my_api"].name, "my-custom-api-name"); } + #[test] + fn round_trip_preserves_network_middlewares() { + let yaml = r#" +version: 1 +network_middlewares: + - name: global-redactor + middleware: openshell/secrets + on_error: fail_open + endpoints: + include: ["api.example.com", "*.service.test"] + exclude: ["internal.example.com"] + config: + secrets: ["api_key", "authorization"] + service: + mode: redact + max_matches: 2 + - name: endpoint-redactor + middleware: openshell/secrets +network_policies: + api: + name: api + middleware: ["global-redactor"] + endpoints: + - host: api.example.com + port: 443 + protocol: rest + middleware: ["endpoint-redactor"] + binaries: + - path: /usr/bin/curl +"#; + let proto = parse_sandbox_policy(yaml).expect("parse failed"); + assert_eq!(proto.network_middlewares.len(), 2); + assert_eq!(proto.network_middlewares[0].name, "global-redactor"); + assert_eq!(proto.network_middlewares[0].middleware, "openshell/secrets"); + assert_eq!(proto.network_middlewares[0].on_error, "fail_open"); + assert_eq!( + proto.network_middlewares[0] + .endpoints + .as_ref() + .expect("selector") + .include, + vec!["api.example.com", "*.service.test"] + ); + assert_eq!( + proto.network_middlewares[0] + .endpoints + .as_ref() + .expect("selector") + .exclude, + vec!["internal.example.com"] + ); + assert!( + proto.network_middlewares[0] + .config + .as_ref() + .expect("config") + .fields + .contains_key("service") + ); + assert_eq!( + proto.network_policies["api"].middleware, + vec!["global-redactor"] + ); + assert_eq!( + proto.network_policies["api"].endpoints[0].middleware, + vec!["endpoint-redactor"] + ); + + let yaml_out = serialize_sandbox_policy(&proto).expect("serialize failed"); + let reparsed = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); + assert_eq!(reparsed.network_middlewares, proto.network_middlewares); + assert_eq!( + reparsed.network_policies["api"].middleware, + vec!["global-redactor"] + ); + assert_eq!( + reparsed.network_policies["api"].endpoints[0].middleware, + vec!["endpoint-redactor"] + ); + } + #[test] fn restrictive_default_has_no_network_policies() { let policy = restrictive_default_policy(); @@ -1753,6 +1968,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), + network_middlewares: Vec::new(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs index 04f390198d..1c63e6ebc4 100644 --- a/crates/openshell-policy/src/merge.rs +++ b/crates/openshell-policy/src/merge.rs @@ -989,6 +989,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1007,6 +1008,7 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], + ..Default::default() }; let result = merge_policy( @@ -1035,6 +1037,7 @@ mod tests { name: "existing".to_string(), endpoints: vec![endpoint("api.github.com", 443)], binaries: vec![advisor_binary("/usr/bin/curl")], + ..Default::default() }, ); @@ -1045,6 +1048,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let result = merge_policy( @@ -1076,6 +1080,7 @@ mod tests { ..Default::default() }, ], + ..Default::default() }; let result = merge_policy( @@ -1107,6 +1112,7 @@ mod tests { path: "/usr/bin/python".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1120,6 +1126,7 @@ mod tests { ..Default::default() }], binaries: vec![advisor_binary("/usr/bin/python")], + ..Default::default() }; let result = merge_policy( @@ -1447,6 +1454,7 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1471,6 +1479,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let merged = merge_policy( @@ -1494,6 +1503,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; // Merge an *unrelated* rule for a different host. The proposed rule @@ -1524,6 +1534,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1536,6 +1547,7 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1567,6 +1579,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; // Endpoint exists in the policy but with a *different* binary. The @@ -1582,6 +1595,7 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1618,6 +1632,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1637,6 +1652,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1664,6 +1680,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1686,6 +1703,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1709,6 +1727,7 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], + ..Default::default() }; let merged = merge_policy( @@ -1733,6 +1752,7 @@ mod tests { name: "any_binary_rule".to_string(), endpoints: vec![endpoint("api.github.com", 443)], binaries: vec![], + ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1745,6 +1765,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -1802,6 +1823,7 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], + ..Default::default() }; let composed = compose_effective_policy( &SandboxPolicy::default(), @@ -1833,6 +1855,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let result = merge_policy( composed, @@ -1901,6 +1924,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let result = merge_policy( policy, diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index ddfbcaf7dc..1eb1b54d23 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -450,6 +450,7 @@ impl ProviderTypeProfile { NetworkPolicyRule { name: rule_name.to_string(), endpoints: self.endpoints.iter().map(endpoint_to_proto).collect(), + middleware: Vec::new(), binaries: self.binaries.iter().map(binary_to_proto).collect(), } } @@ -787,6 +788,7 @@ fn endpoint_to_proto(endpoint: &EndpointProfile) -> NetworkEndpoint { request_body_credential_rewrite: endpoint.request_body_credential_rewrite, advisor_proposed: false, persisted_queries: endpoint.persisted_queries.clone(), + middleware: Vec::new(), graphql_persisted_queries: endpoint .graphql_persisted_queries .iter() diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml new file mode 100644 index 0000000000..fdaeb2e824 --- /dev/null +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor-middleware" +description = "In-process supervisor middleware contract and built-ins for OpenShell" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core" } + +miette = { workspace = true } +prost-types = { workspace = true } +regex = { workspace = true } +tonic = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-supervisor-middleware/src/builtins/mod.rs b/crates/openshell-supervisor-middleware/src/builtins/mod.rs new file mode 100644 index 0000000000..60572d3e82 --- /dev/null +++ b/crates/openshell-supervisor-middleware/src/builtins/mod.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod secrets; diff --git a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs new file mode 100644 index 0000000000..6c94eb4396 --- /dev/null +++ b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use miette::{Result, miette}; +use openshell_core::proto::{Decision, Finding, HttpRequestEvaluation, HttpRequestResult}; +use regex::Regex; + +use crate::BUILTIN_SECRETS; + +pub(crate) fn validate_config(config: &prost_types::Struct) -> Result<()> { + let mode = config + .fields + .get("secrets") + .and_then(|value| match value.kind.as_ref() { + Some(prost_types::value::Kind::StringValue(value)) => Some(value.as_str()), + _ => None, + }) + .unwrap_or("redact"); + if mode != "redact" { + return Err(miette!( + "{} only supports config.secrets: redact in phase 1", + BUILTIN_SECRETS + )); + } + Ok(()) +} + +pub(crate) fn evaluate_http_request( + evaluation: &HttpRequestEvaluation, +) -> Result { + let default_config = prost_types::Struct::default(); + validate_config(evaluation.config.as_ref().unwrap_or(&default_config))?; + let text = String::from_utf8(evaluation.body.clone()) + .map_err(|_| miette!("{} requires UTF-8 request bodies", BUILTIN_SECRETS))?; + let (body, count) = redact_common_secrets(&text)?; + let mut result = HttpRequestResult { + decision: Decision::Allow as i32, + reason: String::new(), + body: body.into_bytes(), + has_body: count > 0, + add_headers: HashMap::new(), + findings: Vec::new(), + metadata: HashMap::new(), + }; + if count > 0 { + result.findings.push(Finding { + r#type: "secret.common".into(), + label: "common secret pattern".into(), + count, + confidence: "medium".into(), + severity: "medium".into(), + }); + result + .metadata + .insert("secrets_redacted".into(), count.to_string()); + } + Ok(result) +} + +fn redact_common_secrets(input: &str) -> Result<(String, u32)> { + let patterns = [ + r#"(?i)(api[_-]?key|access[_-]?token|secret|password)(["']?\s*[:=]\s*["'])[^"',\s}]+(["']?)"#, + r#"(sk-[A-Za-z0-9_-]{16,})"#, + ]; + let mut output = input.to_string(); + let mut count = 0u32; + for pattern in patterns { + let regex = Regex::new(pattern).map_err(|e| miette!("{e}"))?; + count = count.saturating_add(regex.find_iter(&output).count() as u32); + output = regex + .replace_all(&output, |captures: ®ex::Captures<'_>| { + if captures.len() >= 4 { + format!("{}{}[REDACTED]{}", &captures[1], &captures[2], &captures[3]) + } else { + "[REDACTED]".to_string() + } + }) + .into_owned(); + } + Ok((output, count)) +} diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs new file mode 100644 index 0000000000..7d9161fcf3 --- /dev/null +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -0,0 +1,437 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! In-process supervisor middleware chain execution. + +mod builtins; +mod service; + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use miette::{Result, miette}; +pub use service::InProcessMiddlewareService; + +use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; +use openshell_core::proto::{ + Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, NetworkMiddlewareConfig, Process, + RequestContext, +}; +use tonic::Request; + +pub const API_VERSION: &str = "openshell.middleware.v1"; +pub const HTTP_REQUEST_OPERATION: &str = "HttpRequest"; +pub const PRE_CREDENTIALS_PHASE: &str = "pre_credentials"; +pub const BUILTIN_SECRETS: &str = "openshell/secrets"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OnError { + FailClosed, + FailOpen, +} + +impl OnError { + pub fn parse(value: &str) -> Result { + match value { + "" | "fail_closed" => Ok(Self::FailClosed), + "fail_open" => Ok(Self::FailOpen), + other => Err(miette!( + "invalid middleware on_error '{other}', expected fail_closed or fail_open" + )), + } + } +} + +#[derive(Debug, Clone)] +pub struct ChainEntry { + pub name: String, + pub implementation: String, + pub config: prost_types::Struct, + pub on_error: OnError, +} + +impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { + type Error = miette::Report; + + fn try_from(value: &NetworkMiddlewareConfig) -> Result { + if value.name.is_empty() { + return Err(miette!("middleware config name cannot be empty")); + } + if value.middleware.is_empty() { + return Err(miette!( + "middleware config '{}' must name an implementation", + value.name + )); + } + Ok(Self { + name: value.name.clone(), + implementation: value.middleware.clone(), + config: value.config.clone().unwrap_or_default(), + on_error: OnError::parse(&value.on_error)?, + }) + } +} + +#[derive(Debug, Clone)] +pub struct HttpRequestInput { + pub request_id: String, + pub sandbox_id: String, + pub binary: String, + pub pid: u32, + pub ancestors: Vec, + pub scheme: String, + pub host: String, + pub port: u16, + pub method: String, + pub path: String, + pub query: String, + pub headers: BTreeMap, + pub body: Vec, +} + +#[derive(Debug, Clone)] +pub struct ChainOutcome { + pub allowed: bool, + pub reason: String, + pub body: Vec, + pub added_headers: BTreeMap, + pub findings: Vec, + pub metadata: BTreeMap>, + pub applied: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct NamespacedFinding { + pub middleware: String, + pub finding: Finding, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MiddlewareInvocation { + pub name: String, + pub implementation: String, + pub decision: Decision, + pub transformed: bool, +} + +#[derive(Clone)] +pub struct ChainRunner { + service: Arc, +} + +impl Default for ChainRunner { + fn default() -> Self { + Self::new(Arc::new(InProcessMiddlewareService)) + } +} + +impl ChainRunner { + pub fn new(service: Arc) -> Self { + Self { service } + } + + pub async fn evaluate( + &self, + entries: &[ChainEntry], + input: HttpRequestInput, + ) -> Result { + let mut headers = input.headers.clone(); + let mut body = input.body.clone(); + let mut added_headers = BTreeMap::new(); + let mut findings = Vec::new(); + let mut metadata = BTreeMap::new(); + let mut applied = Vec::new(); + + for entry in entries { + let evaluation = build_evaluation(entry, &input, &headers, &body); + let result = match self + .service + .evaluate_http_request(Request::new(evaluation)) + .await + { + Ok(result) => result.into_inner(), + Err(err) => match entry.on_error { + OnError::FailOpen => { + applied.push(MiddlewareInvocation { + name: entry.name.clone(), + implementation: entry.implementation.clone(), + decision: Decision::Allow, + transformed: false, + }); + continue; + } + OnError::FailClosed => { + return Ok(ChainOutcome { + allowed: false, + reason: format!("middleware_failed: {}", safe_reason(&err.to_string())), + body, + added_headers, + findings, + metadata, + applied, + }); + } + }, + }; + + validate_header_mutations(&headers, &result.add_headers)?; + for (name, value) in &result.add_headers { + headers.insert(name.to_ascii_lowercase(), value.clone()); + added_headers.insert(name.to_ascii_lowercase(), value.clone()); + } + let transformed = result.has_body; + if result.has_body { + body = result.body.clone(); + } + for finding in result.findings { + findings.push(NamespacedFinding { + middleware: entry.name.clone(), + finding, + }); + } + if !result.metadata.is_empty() { + metadata.insert( + entry.name.clone(), + result.metadata.clone().into_iter().collect(), + ); + } + applied.push(MiddlewareInvocation { + name: entry.name.clone(), + implementation: entry.implementation.clone(), + decision: Decision::try_from(result.decision).unwrap_or(Decision::Unspecified), + transformed, + }); + if result.decision == Decision::Deny as i32 { + return Ok(ChainOutcome { + allowed: false, + reason: safe_reason(&result.reason), + body, + added_headers, + findings, + metadata, + applied, + }); + } + } + + Ok(ChainOutcome { + allowed: true, + reason: String::new(), + body, + added_headers, + findings, + metadata, + applied, + }) + } +} + +fn build_evaluation( + entry: &ChainEntry, + input: &HttpRequestInput, + headers: &BTreeMap, + body: &[u8], +) -> HttpRequestEvaluation { + HttpRequestEvaluation { + api_version: API_VERSION.into(), + binding_id: entry.implementation.clone(), + phase: PRE_CREDENTIALS_PHASE.into(), + context: Some(RequestContext { + request_id: input.request_id.clone(), + sandbox_id: input.sandbox_id.clone(), + originating_process: Some(Process { + binary: input.binary.clone(), + pid: input.pid, + ancestors: input.ancestors.clone(), + }), + }), + config: Some(entry.config.clone()), + target: Some(HttpRequestTarget { + scheme: input.scheme.clone(), + host: input.host.clone(), + port: u32::from(input.port), + method: input.method.clone(), + path: input.path.clone(), + query: input.query.clone(), + }), + headers: headers.clone().into_iter().collect(), + body: body.to_vec(), + } +} + +fn validate_header_mutations( + existing_headers: &BTreeMap, + mutations: &HashMap, +) -> Result<()> { + let mut seen = HashSet::new(); + for name in mutations.keys() { + let lower = name.to_ascii_lowercase(); + if !seen.insert(lower.clone()) || existing_headers.contains_key(&lower) { + return Err(miette!( + "middleware cannot rewrite existing header '{name}'" + )); + } + if !is_safe_append_header(&lower) { + return Err(miette!("middleware cannot append unsafe header '{name}'")); + } + } + Ok(()) +} + +fn is_safe_append_header(name: &str) -> bool { + if name.is_empty() + || name.contains(':') + || name.bytes().any(|b| b <= 0x20 || b >= 0x7f) + || matches!( + name, + "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" + ) + || name.starts_with("x-amz-") + || name.starts_with("x-openshell-credential") + { + return false; + } + name.starts_with("x-openshell-middleware-") +} + +pub(crate) fn safe_reason(reason: &str) -> String { + reason + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | ':' | ' ')) + .take(160) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; + + fn entry(name: &str, on_error: OnError) -> ChainEntry { + ChainEntry { + name: name.into(), + implementation: BUILTIN_SECRETS.into(), + config: prost_types::Struct { + fields: [( + "secrets".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("redact".into())), + }, + )] + .into_iter() + .collect(), + }, + on_error, + } + } + + fn input(body: &str) -> HttpRequestInput { + HttpRequestInput { + request_id: "req".into(), + sandbox_id: "sbx".into(), + binary: "/usr/bin/curl".into(), + pid: 42, + ancestors: vec![], + scheme: "https".into(), + host: "api.example.com".into(), + port: 443, + method: "POST".into(), + path: "/v1".into(), + query: String::new(), + headers: BTreeMap::new(), + body: body.as_bytes().to_vec(), + } + } + + #[tokio::test] + async fn redacts_common_secret_patterns() { + let outcome = ChainRunner::default() + .evaluate( + &[entry("redact", OnError::FailClosed)], + input(r#"{"api_key":"sk-1234567890abcdef"}"#), + ) + .await + .expect("evaluate"); + assert!(outcome.allowed); + assert_eq!( + String::from_utf8(outcome.body).expect("utf8"), + r#"{"api_key":"[REDACTED]"}"# + ); + assert_eq!(outcome.findings[0].finding.count, 1); + } + + #[tokio::test] + async fn transformed_body_feeds_next_stage() { + let entries = [ + entry("first", OnError::FailClosed), + entry("second", OnError::FailClosed), + ]; + let outcome = ChainRunner::default() + .evaluate(&entries, input(r#"password="top-secret""#)) + .await + .expect("evaluate"); + assert!(outcome.allowed); + assert_eq!( + String::from_utf8(outcome.body).expect("utf8"), + r#"password="[REDACTED]""# + ); + assert_eq!(outcome.applied.len(), 2); + } + + #[tokio::test] + async fn fail_open_allows_unavailable_middleware() { + let unavailable = ChainEntry { + name: "missing".into(), + implementation: "third-party/missing".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }; + let outcome = ChainRunner::default() + .evaluate(&[unavailable], input("hello")) + .await + .expect("evaluate"); + assert!(outcome.allowed); + assert_eq!(outcome.body, b"hello"); + } + + #[tokio::test] + async fn fail_closed_denies_unavailable_middleware() { + let unavailable = ChainEntry { + name: "missing".into(), + implementation: "third-party/missing".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let outcome = ChainRunner::default() + .evaluate(&[unavailable], input("hello")) + .await + .expect("evaluate"); + assert!(!outcome.allowed); + assert!(outcome.reason.starts_with("middleware_failed:")); + } + + #[tokio::test] + async fn in_process_service_describes_builtin_binding() { + let manifest = InProcessMiddlewareService + .describe(Request::new(())) + .await + .expect("describe") + .into_inner(); + assert_eq!(manifest.api_version, API_VERSION); + assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); + assert_eq!(manifest.bindings[0].operation, HTTP_REQUEST_OPERATION); + assert_eq!(manifest.bindings[0].phase, PRE_CREDENTIALS_PHASE); + } + + #[test] + fn unsafe_header_mutation_is_rejected() { + let err = validate_header_mutations( + &BTreeMap::new(), + &[("Authorization".into(), "Bearer nope".into())] + .into_iter() + .collect(), + ) + .expect_err("unsafe header"); + assert!(err.to_string().contains("unsafe header")); + } +} diff --git a/crates/openshell-supervisor-middleware/src/service.rs b/crates/openshell-supervisor-middleware/src/service.rs new file mode 100644 index 0000000000..31cca5694b --- /dev/null +++ b/crates/openshell-supervisor-middleware/src/service.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; +use openshell_core::proto::{ + HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding, MiddlewareManifest, + ValidateConfigRequest, ValidateConfigResponse, +}; +use tonic::{Request, Response, Status}; + +use crate::{ + API_VERSION, BUILTIN_SECRETS, HTTP_REQUEST_OPERATION, PRE_CREDENTIALS_PHASE, builtins, + safe_reason, +}; + +#[derive(Debug, Default)] +pub struct InProcessMiddlewareService; + +#[tonic::async_trait] +impl SupervisorMiddleware for InProcessMiddlewareService { + async fn describe( + &self, + _request: Request<()>, + ) -> Result, Status> { + Ok(Response::new(MiddlewareManifest { + api_version: API_VERSION.into(), + name: "openshell/in-process".into(), + service_version: env!("CARGO_PKG_VERSION").into(), + bindings: vec![MiddlewareBinding { + id: BUILTIN_SECRETS.into(), + operation: HTTP_REQUEST_OPERATION.into(), + phase: PRE_CREDENTIALS_PHASE.into(), + }], + })) + } + + async fn validate_config( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let config = request.config.unwrap_or_default(); + let validation = match request.binding_id.as_str() { + BUILTIN_SECRETS => builtins::secrets::validate_config(&config), + other => Err(miette::miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + }; + Ok(Response::new(match validation { + Ok(()) => ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + Err(err) => ValidateConfigResponse { + valid: false, + reason: safe_reason(&err.to_string()), + }, + })) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let result = match request.binding_id.as_str() { + BUILTIN_SECRETS => builtins::secrets::evaluate_http_request(&request), + other => Err(miette::miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + } + .map_err(|err| Status::invalid_argument(safe_reason(&err.to_string())))?; + Ok(Response::new(result)) + } +} diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 7d0079f7b3..fd8fad5f75 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -15,6 +15,7 @@ openshell-core = { path = "../openshell-core" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } apollo-parser = { workspace = true } aws-sigv4 = { version = "1", features = ["sign-http", "http1"] } @@ -28,6 +29,7 @@ glob = { workspace = true } hex = "0.4" ipnet = "2" miette = { workspace = true } +prost-types = { workspace = true } rcgen = { workspace = true } regorus = { version = "0.9", default-features = false, features = ["std", "arc", "glob"] } reqwest = { workspace = true } diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index efcdf07320..afa4f69478 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -856,6 +856,22 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { count(_matching_endpoint_configs) > 0 } +network_middlewares := object.get(data, "network_middlewares", []) + +_matching_middleware_contexts := [ctx | + some pname + _matching_policy_names[pname] + policy := data.network_policies[pname] + some ep + ep := policy.endpoints[_] + endpoint_matches_request(ep, input.network) + ctx := { + "policy": pname, + "policy_middleware": object.get(policy, "middleware", []), + "endpoint": ep, + } +] + _policy_has_exact_declared_endpoint(policy) if { some ep ep := policy.endpoints[_] @@ -909,7 +925,7 @@ endpoint_path_matches_request(ep, request) if { } # An endpoint has extended config if it specifies L7 protocol, allowed_ips, -# or an explicit tls mode (e.g. tls: skip). +# middleware, or an explicit tls mode (e.g. tls: skip). endpoint_has_extended_config(ep) if { ep.protocol } @@ -918,6 +934,10 @@ endpoint_has_extended_config(ep) if { count(object.get(ep, "allowed_ips", [])) > 0 } +endpoint_has_extended_config(ep) if { + count(object.get(ep, "middleware", [])) > 0 +} + endpoint_has_extended_config(ep) if { ep.tls } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index ed2bde1135..4d501d0a39 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -15,9 +15,12 @@ use miette::{IntoDiagnostic, Result, miette}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::secrets::{self, SecretResolver}; use openshell_ocsf::{ - ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, - NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, + ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, + HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, + ocsf_emit, }; +use std::collections::BTreeMap; +use std::path::PathBuf; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; @@ -450,6 +453,37 @@ where let _ = &eval_target; if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { + let chain = + engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let req = + match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let outcome = crate::l7::rest::relay_http_request_with_options_guarded( &req, client, @@ -734,6 +768,167 @@ fn jsonrpc_engine_type(protocol: L7Protocol) -> &'static str { } } +enum MiddlewareApplyResult { + Allowed(crate::l7::provider::L7Request), + Denied(String), +} + +async fn apply_middleware_chain( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + generation_guard: &PolicyGenerationGuard, +) -> Result { + if chain.is_empty() { + return Ok(MiddlewareApplyResult::Allowed(req)); + } + let buffered = + crate::l7::rest::buffer_request_body_for_middleware(&req, client, Some(generation_guard)) + .await?; + let headers = safe_middleware_headers(&buffered.headers)?; + let input = openshell_supervisor_middleware::HttpRequestInput { + request_id: uuid::Uuid::new_v4().to_string(), + sandbox_id: String::new(), + binary: ctx.binary_path.clone(), + pid: 0, + ancestors: ctx.ancestors.clone(), + scheme: "https".into(), + host: ctx.host.clone(), + port: ctx.port, + method: req.action.clone(), + path: req.target.clone(), + query: String::new(), + headers, + body: buffered.body, + }; + let outcome = openshell_supervisor_middleware::ChainRunner::default() + .evaluate(&chain, input) + .await?; + emit_middleware_events(ctx, &req, &outcome); + let rebuilt = crate::l7::rest::rebuild_request_with_buffered_body( + &req, + &buffered.headers, + &outcome.body, + &outcome.added_headers, + )?; + if outcome.allowed { + Ok(MiddlewareApplyResult::Allowed(rebuilt)) + } else { + Ok(MiddlewareApplyResult::Denied(outcome.reason)) + } +} + +fn safe_middleware_headers(headers: &[u8]) -> Result> { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let mut out = BTreeMap::new(); + for line in header_str.lines().skip(1) { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let name = name.trim().to_ascii_lowercase(); + if name.is_empty() + || matches!( + name.as_str(), + "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" + ) + || name.starts_with("x-amz-") + || name.starts_with("x-openshell-credential") + { + continue; + } + out.insert(name, value.trim().to_string()); + } + Ok(out) +} + +fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { + crate::opa::NetworkInput { + host: ctx.host.clone(), + port: ctx.port, + binary_path: PathBuf::from(&ctx.binary_path), + binary_sha256: String::new(), + ancestors: ctx.ancestors.iter().map(PathBuf::from).collect(), + cmdline_paths: ctx.cmdline_paths.iter().map(PathBuf::from).collect(), + } +} + +fn emit_middleware_events( + ctx: &L7EvalContext, + req: &crate::l7::provider::L7Request, + outcome: &openshell_supervisor_middleware::ChainOutcome, +) { + for invocation in &outcome.applied { + let allowed = invocation.decision == openshell_core::proto::Decision::Allow; + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(if allowed { + ActionId::Allowed + } else { + ActionId::Denied + }) + .disposition(if allowed { + DispositionId::Allowed + } else { + DispositionId::Blocked + }) + .severity(if allowed { + SeverityId::Informational + } else { + SeverityId::Medium + }) + .http_request(HttpRequest::new( + &req.action, + OcsfUrl::new("http", &ctx.host, &req.target, ctx.port), + )) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .firewall_rule(&ctx.policy_name, "middleware") + .message(format!( + "MIDDLEWARE {} {} decision={:?} transformed={}", + invocation.name, + invocation.implementation, + invocation.decision, + invocation.transformed + )) + .build(); + ocsf_emit!(event); + } + if !outcome.allowed && outcome.reason.starts_with("middleware_failed:") { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::High) + .finding_info(FindingInfo::new( + "openshell.middleware.failure", + "Supervisor middleware failure", + )) + .message("Required supervisor middleware failed closed") + .build(); + ocsf_emit!(event); + } + for finding in &outcome.findings { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(match finding.finding.severity.as_str() { + "high" => SeverityId::High, + "low" => SeverityId::Low, + _ => SeverityId::Medium, + }) + .finding_info(FindingInfo::new( + &finding.finding.r#type, + &finding.finding.label, + )) + .evidence_pairs(&[ + ("middleware", &finding.middleware), + ("count", &finding.finding.count.to_string()), + ]) + .message(format!( + "Middleware finding {} count={}", + finding.finding.r#type, finding.finding.count + )) + .build(); + ocsf_emit!(event); + } +} + /// REST relay loop: parse request -> evaluate -> allow/deny -> relay response -> repeat. async fn relay_rest( config: &L7EndpointConfig, @@ -903,6 +1098,37 @@ where let _ = &eval_target; if allowed || config.enforcement == EnforcementMode::Audit { + let chain = + engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let req = + match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + provider + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let req_with_auth = match crate::l7::token_grant_injection::inject_if_needed(req, ctx).await { Ok(req) => req, @@ -1336,6 +1562,37 @@ where let _ = &eval_target; if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { + let chain = + engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let req = + match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let outcome = crate::l7::rest::relay_http_request_with_resolver_guarded( &req, client, @@ -1674,6 +1931,7 @@ pub async fn relay_passthrough_with_credentials( upstream: &mut U, ctx: &L7EvalContext, generation_guard: &PolicyGenerationGuard, + middleware_engine: Option<&crate::opa::OpaEngine>, ) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, @@ -1756,6 +2014,43 @@ where ocsf_emit!(event); } + let req = if let Some(engine) = middleware_engine { + let input = middleware_network_input(ctx); + let (chain, generation) = + engine.query_middleware_chain_with_generation(&input, &req.target)?; + if generation != generation_guard.captured_generation() { + return Ok(()); + } + match apply_middleware_chain(req, client, ctx, chain, generation_guard).await? { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: "HTTP".into(), + target: redacted_target.clone(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + } + } else { + req + }; + let req_with_auth = match crate::l7::token_grant_injection::inject_if_needed(req, ctx).await { Ok(req) => req, @@ -1901,6 +2196,63 @@ network_policies: (config, tunnel_engine, ctx, fixture) } + fn middleware_relay_context( + middleware_impl: &str, + on_error: &str, + ) -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + let data = format!( + r#" +network_middlewares: + - name: request-middleware + middleware: {middleware_impl} + on_error: {on_error} +network_policies: + rest_api: + name: rest_api + middleware: ["request-middleware"] + endpoints: + - host: api.example.test + port: 8080 + protocol: rest + enforcement: enforce + rules: + - allow: + method: POST + path: "/v1/**" + binaries: + - {{ path: /usr/bin/curl }} +"# + ); + let engine = OpaEngine::from_strings(TEST_POLICY, &data).unwrap(); + let input = NetworkInput { + host: "api.example.test".into(), + port: 8080, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&endpoint_config.unwrap()).unwrap(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 8080, + policy_name: "rest_api".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + + (config, tunnel_engine, ctx) + } + fn passthrough_token_grant_relay_context( resolver_response: std::result::Result<&str, &str>, ) -> ( @@ -2112,7 +2464,10 @@ network_policies: .unwrap(); let upstream_request = String::from_utf8_lossy(&upstream_request[..n]); - assert!(upstream_request.starts_with("GET /v1/projects HTTP/1.1\r\n")); + assert!( + upstream_request.starts_with("GET /v1/projects HTTP/1.1\r\n"), + "unexpected upstream request: {upstream_request:?}" + ); assert!(upstream_request.contains("Authorization: Bearer grant-token\r\n")); assert!(!upstream_request.contains("stale-token")); assert_eq!(authorization_header_count(&upstream_request), 1); @@ -2194,6 +2549,115 @@ network_policies: fixture.assert_one_request("api.example.test\t8080\t/v1/**\tprovider:access_token"); } + #[tokio::test] + async fn l7_rest_middleware_redacts_body_before_upstream() { + let (config, tunnel_engine, ctx) = + middleware_relay_context("openshell/secrets", "fail_closed"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"api_key":"sk-1234567890abcdef"}"#; + let request = format!( + "POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut upstream_request = [0u8; 1024]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_request), + ) + .await + .expect("request should reach upstream") + .unwrap(); + let upstream_request = String::from_utf8_lossy(&upstream_request[..n]); + assert!(upstream_request.contains(r#""api_key":"[REDACTED]""#)); + assert!(!upstream_request.contains("sk-1234567890abcdef")); + + upstream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut client_response = [0u8; 512]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + app.read(&mut client_response), + ) + .await + .expect("response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("204 No Content")); + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn l7_rest_middleware_fail_closed_does_not_reach_upstream() { + let (config, tunnel_engine, ctx) = + middleware_relay_context("example/unavailable", "fail_closed"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all( + b"POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + ) + .await + .unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden")); + assert!(response.contains("middleware_failed")); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive request bytes" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[tokio::test] async fn passthrough_relay_injects_token_grant_authorization_header() { let (generation_guard, ctx, fixture) = @@ -2206,6 +2670,7 @@ network_policies: &mut relay_upstream, &ctx, &generation_guard, + None, ) .await }); @@ -2268,6 +2733,7 @@ network_policies: &mut relay_upstream, &ctx, &generation_guard, + None, ) .await }); @@ -3173,7 +3639,10 @@ network_policies: .expect("first request should reach upstream") .unwrap(); let first_upstream = String::from_utf8_lossy(&first_upstream[..n]); - assert!(first_upstream.starts_with("POST /write HTTP/1.1")); + assert!( + first_upstream.starts_with("POST /write HTTP/1.1"), + "unexpected upstream request: {first_upstream:?}" + ); upstream .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: keep-alive\r\n\r\nOK") @@ -3243,6 +3712,7 @@ network_policies: &mut relay_upstream, &ctx, &generation_guard, + None, ) .await }); diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 0558a67e55..1a4036abd7 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,6 +27,7 @@ const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; /// Maximum body bytes for `SigV4` body-signing mode. Larger than the credential /// rewrite limit because Bedrock payloads can be several megabytes. const MAX_SIGV4_BODY_BYTES: usize = 10 * 1024 * 1024; +pub(crate) const MAX_MIDDLEWARE_BODY_BYTES: usize = MAX_REWRITE_BODY_BYTES; const RELAY_BUF_SIZE: usize = 8192; const HTTP_METHOD_PREFIXES: &[&[u8]] = &[ b"GET ", @@ -768,6 +769,83 @@ struct PreparedRequestBody { body: Vec, } +pub(crate) struct BufferedRequestBody { + pub(crate) headers: Vec, + pub(crate) body: Vec, +} + +pub(crate) async fn buffer_request_body_for_middleware( + req: &L7Request, + client: &mut C, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result { + let header_end = req + .raw_header + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(req.raw_header.len(), |p| p + 4); + let headers = req.raw_header[..header_end].to_vec(); + let already_read = &req.raw_header[header_end..]; + match req.body_length { + BodyLength::None => Ok(BufferedRequestBody { + headers, + body: already_read.to_vec(), + }), + BodyLength::ContentLength(len) => { + let len = usize::try_from(len) + .map_err(|_| miette!("request body is too large for middleware"))?; + if len > MAX_MIDDLEWARE_BODY_BYTES { + return Err(miette!( + "middleware buffers at most {MAX_MIDDLEWARE_BODY_BYTES} request body bytes" + )); + } + let initial_len = already_read.len().min(len); + let mut body = Vec::with_capacity(len); + body.extend_from_slice(&already_read[..initial_len]); + let mut remaining = len.saturating_sub(initial_len); + let mut buf = [0u8; RELAY_BUF_SIZE]; + while remaining > 0 { + let to_read = remaining.min(buf.len()); + let n = client.read(&mut buf[..to_read]).await.into_diagnostic()?; + if n == 0 { + return Err(miette!( + "Connection closed with {remaining} body bytes remaining" + )); + } + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + body.extend_from_slice(&buf[..n]); + remaining -= n; + } + Ok(BufferedRequestBody { headers, body }) + } + BodyLength::Chunked => { + let body = collect_chunked_body(client, already_read, generation_guard).await?; + Ok(BufferedRequestBody { headers, body }) + } + } +} + +pub(crate) fn rebuild_request_with_buffered_body( + req: &L7Request, + headers: &[u8], + body: &[u8], + add_headers: &std::collections::BTreeMap, +) -> Result { + let mut header_bytes = set_content_length(headers, body.len())?; + header_bytes = strip_header(&header_bytes, "transfer-encoding")?; + header_bytes = append_headers(&header_bytes, add_headers)?; + header_bytes.extend_from_slice(body); + Ok(L7Request { + action: req.action.clone(), + target: req.target.clone(), + query_params: req.query_params.clone(), + raw_header: header_bytes, + body_length: BodyLength::ContentLength(body.len() as u64), + }) +} + async fn collect_and_rewrite_request_body( req: &L7Request, client: &mut C, @@ -1160,6 +1238,50 @@ fn set_content_length(headers: &[u8], len: usize) -> Result> { Ok(out.into_bytes()) } +fn strip_header(headers: &[u8], strip_name: &str) -> Result> { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let mut out = String::with_capacity(header_str.len()); + for line in header_str.split("\r\n") { + if line.is_empty() { + out.push_str("\r\n"); + break; + } + if line + .split_once(':') + .is_some_and(|(name, _)| name.trim().eq_ignore_ascii_case(strip_name)) + { + continue; + } + out.push_str(line); + out.push_str("\r\n"); + } + Ok(out.into_bytes()) +} + +fn append_headers( + headers: &[u8], + add_headers: &std::collections::BTreeMap, +) -> Result> { + if add_headers.is_empty() { + return Ok(headers.to_vec()); + } + let split = headers + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(headers.len(), |pos| pos); + let mut out = Vec::with_capacity(headers.len() + add_headers.len() * 32); + out.extend_from_slice(&headers[..split]); + for (name, value) in add_headers { + out.extend_from_slice(b"\r\n"); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(b": "); + out.extend_from_slice(value.as_bytes()); + } + out.extend_from_slice(b"\r\n\r\n"); + Ok(out) +} + pub(crate) fn request_is_websocket_upgrade(raw_header: &[u8]) -> bool { let header_end = raw_header .windows(4) diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index fbab5fedd7..451c57e597 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -13,6 +13,8 @@ use openshell_core::policy::{ }; use openshell_core::proto::SandboxPolicy as ProtoSandboxPolicy; use openshell_policy::L7ConfigStanza; +use openshell_supervisor_middleware::ChainEntry; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{ Arc, Mutex, @@ -132,6 +134,19 @@ impl TunnelPolicyEngine { pub(crate) fn engine(&self) -> &Mutex { &self.engine } + + /// Query the ordered middleware chain for a request path within this tunnel. + pub fn query_middleware_chain( + &self, + input: &NetworkInput, + request_path: &str, + ) -> Result> { + let mut engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + query_middleware_chain_locked(&mut engine, input, request_path) + } } impl OpaEngine { @@ -200,6 +215,14 @@ impl OpaEngine { .map_err(|e| miette::miette!("internal: failed to parse proto JSON: {e}"))?; // Validate BEFORE expanding presets + let middleware_errors = validate_middleware_policies(&data); + if !middleware_errors.is_empty() { + return Err(miette::miette!( + "middleware policy validation failed:\n{}", + middleware_errors.join("\n") + )); + } + let (errors, warnings) = crate::l7::validate_l7_policies(&data); for w in &warnings { openshell_ocsf::ocsf_emit!( @@ -548,6 +571,21 @@ impl OpaEngine { } } + /// Query the ordered middleware chain for a parsed HTTP request path. + pub fn query_middleware_chain_with_generation( + &self, + input: &NetworkInput, + request_path: &str, + ) -> Result<(Vec, u64)> { + let mut engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let generation = self.current_generation(); + let chain = query_middleware_chain_locked(&mut engine, input, request_path)?; + Ok((chain, generation)) + } + /// Query `allowed_ips` from the matched endpoint config for a given request. /// /// Returns the list of CIDR/IP strings from the endpoint's `allowed_ips` @@ -687,6 +725,243 @@ fn get_str_array(val: ®orus::Value, key: &str) -> Vec { } } +fn network_input_json(input: &NetworkInput) -> serde_json::Value { + let ancestor_strs: Vec = input + .ancestors + .iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + let cmdline_strs: Vec = input + .cmdline_paths + .iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + serde_json::json!({ + "exec": { + "path": input.binary_path.to_string_lossy(), + "ancestors": ancestor_strs, + "cmdline_paths": cmdline_strs, + }, + "network": { + "host": input.host, + "port": input.port, + } + }) +} + +#[derive(Debug, Clone)] +struct MiddlewareContext { + policy_middleware: Vec, + endpoint_middleware: Vec, + endpoint_path: String, +} + +fn query_middleware_chain_locked( + engine: &mut regorus::Engine, + input: &NetworkInput, + request_path: &str, +) -> Result> { + engine + .set_input_json(&network_input_json(input).to_string()) + .map_err(|e| miette::miette!("{e}"))?; + + let configs_val = engine + .eval_rule("data.openshell.sandbox.network_middlewares".into()) + .map_err(|e| miette::miette!("{e}"))?; + let configs = parse_middleware_configs(&configs_val)?; + if configs.is_empty() { + return Ok(Vec::new()); + } + let contexts_val = engine + .eval_rule("data.openshell.sandbox._matching_middleware_contexts".into()) + .map_err(|e| miette::miette!("{e}"))?; + let contexts = parse_middleware_contexts(&contexts_val); + let Some(context) = select_middleware_context(&contexts, request_path) else { + return Ok(global_middleware_entries( + &configs, + &input.host, + &HashSet::new(), + )?); + }; + + let mut explicit = Vec::new(); + for name in context + .policy_middleware + .iter() + .chain(context.endpoint_middleware.iter()) + { + if !explicit.contains(name) { + explicit.push(name.clone()); + } + } + let explicit_set: HashSet = explicit.iter().cloned().collect(); + let mut ordered = global_middleware_entries(&configs, &input.host, &explicit_set)?; + for name in explicit { + if !ordered.iter().any(|entry| entry.name == name) { + let config = configs + .iter() + .find(|config| get_str(config, "name").as_deref() == Some(name.as_str())) + .ok_or_else(|| miette::miette!("unknown middleware config '{name}'"))?; + ordered.push(chain_entry_from_value(config)?); + } + } + Ok(ordered) +} + +fn parse_middleware_configs(value: ®orus::Value) -> Result> { + match value { + regorus::Value::Undefined => Ok(Vec::new()), + regorus::Value::Array(values) => Ok(values.to_vec()), + other => Err(miette::miette!( + "network_middlewares must be an array, got {other:?}" + )), + } +} + +fn parse_middleware_contexts(value: ®orus::Value) -> Vec { + let regorus::Value::Array(values) = value else { + return Vec::new(); + }; + values + .iter() + .filter_map(|value| { + let regorus::Value::Object(_) = value else { + return None; + }; + let endpoint = get_field(value, "endpoint")?; + Some(MiddlewareContext { + policy_middleware: get_str_array(value, "policy_middleware"), + endpoint_middleware: get_str_array(endpoint, "middleware"), + endpoint_path: get_str(endpoint, "path").unwrap_or_default(), + }) + }) + .collect() +} + +fn select_middleware_context<'a>( + contexts: &'a [MiddlewareContext], + request_path: &str, +) -> Option<&'a MiddlewareContext> { + contexts + .iter() + .filter(|context| crate::l7::endpoint_path_matches(&context.endpoint_path, request_path)) + .max_by_key(|context| { + if context.endpoint_path.is_empty() { + 0 + } else { + context.endpoint_path.chars().filter(|c| *c != '*').count() + } + }) +} + +fn global_middleware_entries( + configs: &[regorus::Value], + host: &str, + explicit: &HashSet, +) -> Result> { + let mut entries = Vec::new(); + for config in configs { + let name = get_str(config, "name").unwrap_or_default(); + if explicit.contains(&name) { + continue; + } + if middleware_selector_matches(config, host) { + entries.push(chain_entry_from_value(config)?); + } + } + Ok(entries) +} + +fn middleware_selector_matches(config: ®orus::Value, host: &str) -> bool { + let Some(selector) = get_field(config, "endpoints") else { + return false; + }; + let includes = get_str_array(selector, "include"); + let excludes = get_str_array(selector, "exclude"); + let included = + !includes.is_empty() && includes.iter().any(|pattern| host_matches(pattern, host)); + let excluded = excludes.iter().any(|pattern| host_matches(pattern, host)); + included && !excluded +} + +fn host_matches(pattern: &str, host: &str) -> bool { + if pattern == "*" || pattern == "**" { + return true; + } + if !pattern.contains('*') { + return pattern.eq_ignore_ascii_case(host); + } + glob::Pattern::new(&pattern.to_ascii_lowercase()) + .is_ok_and(|pattern| pattern.matches(&host.to_ascii_lowercase())) +} + +fn chain_entry_from_value(value: ®orus::Value) -> Result { + let name = get_str(value, "name").unwrap_or_default(); + let implementation = get_str(value, "middleware").unwrap_or_default(); + Ok(ChainEntry { + name, + implementation, + config: get_field(value, "config") + .map(regorus_value_to_struct) + .unwrap_or_default(), + on_error: openshell_supervisor_middleware::OnError::parse( + get_str(value, "on_error").as_deref().unwrap_or_default(), + )?, + }) +} + +fn get_field<'a>(val: &'a regorus::Value, key: &str) -> Option<&'a regorus::Value> { + let key_val = regorus::Value::String(key.into()); + match val { + regorus::Value::Object(map) => map.get(&key_val), + _ => None, + } +} + +fn regorus_value_to_struct(value: ®orus::Value) -> prost_types::Struct { + let regorus::Value::Object(map) = value else { + return prost_types::Struct::default(); + }; + prost_types::Struct { + fields: map + .iter() + .filter_map(|(key, value)| match key { + regorus::Value::String(key) => { + Some((key.to_string(), regorus_value_to_prost(value))) + } + _ => None, + }) + .collect(), + } +} + +fn regorus_value_to_prost(value: ®orus::Value) -> prost_types::Value { + use prost_types::{ListValue, Struct, Value, value::Kind}; + Value { + kind: Some(match value { + regorus::Value::Bool(value) => Kind::BoolValue(*value), + regorus::Value::Number(value) => Kind::NumberValue(value.as_f64().unwrap_or_default()), + regorus::Value::String(value) => Kind::StringValue(value.to_string()), + regorus::Value::Array(values) => Kind::ListValue(ListValue { + values: values.iter().map(regorus_value_to_prost).collect(), + }), + regorus::Value::Object(values) => Kind::StructValue(Struct { + fields: values + .iter() + .filter_map(|(key, value)| match key { + regorus::Value::String(key) => { + Some((key.to_string(), regorus_value_to_prost(value))) + } + _ => None, + }) + .collect(), + }), + regorus::Value::Null | regorus::Value::Undefined => Kind::NullValue(0), + _ => Kind::NullValue(0), + }), + } +} + fn parse_filesystem_policy(val: ®orus::Value) -> FilesystemPolicy { FilesystemPolicy { read_only: get_str_array(val, "read_only") @@ -735,6 +1010,14 @@ fn preprocess_yaml_data(yaml_str: &str) -> Result { } // Validate BEFORE expanding presets (catches user errors like rules+access) + let middleware_errors = validate_middleware_policies(&data); + if !middleware_errors.is_empty() { + return Err(miette::miette!( + "middleware policy validation failed:\n{}", + middleware_errors.join("\n") + )); + } + let (errors, warnings) = crate::l7::validate_l7_policies(&data); for w in &warnings { openshell_ocsf::ocsf_emit!( @@ -955,6 +1238,131 @@ fn normalize_l7_rule_aliases( } } +fn validate_middleware_policies(data: &serde_json::Value) -> Vec { + let mut errors = Vec::new(); + let middlewares = data + .get("network_middlewares") + .and_then(serde_json::Value::as_array) + .map_or(&[][..], Vec::as_slice); + let mut names = HashSet::new(); + for mw in middlewares { + let name = mw + .get("name") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let implementation = mw + .get("middleware") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if name.is_empty() { + errors.push("network_middlewares entry has empty name".to_string()); + } else if !names.insert(name.to_string()) { + errors.push(format!("duplicate middleware config '{name}'")); + } + if implementation.is_empty() { + errors.push(format!( + "middleware config '{name}' has empty implementation" + )); + } + if implementation.starts_with("openshell/") + && implementation != openshell_supervisor_middleware::BUILTIN_SECRETS + { + errors.push(format!( + "middleware config '{name}' references unsupported built-in '{implementation}'" + )); + } + let on_error = mw + .get("on_error") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if !matches!(on_error, "" | "fail_closed" | "fail_open") { + errors.push(format!( + "middleware config '{name}' has invalid on_error '{on_error}'" + )); + } + } + + let Some(policies) = data + .get("network_policies") + .and_then(serde_json::Value::as_object) + else { + return errors; + }; + + for (policy_name, policy) in policies { + let policy_middleware = json_string_array(policy.get("middleware")); + for name in &policy_middleware { + if !names.contains(name) { + errors.push(format!( + "network policy '{policy_name}' references unknown middleware config '{name}'" + )); + } + } + for endpoint in policy + .get("endpoints") + .and_then(serde_json::Value::as_array) + .map_or(&[][..], Vec::as_slice) + { + let endpoint_middleware = json_string_array(endpoint.get("middleware")); + for name in &endpoint_middleware { + if !names.contains(name) { + errors.push(format!( + "network policy '{policy_name}' endpoint references unknown middleware config '{name}'" + )); + } + } + let tls_skip = endpoint + .get("tls") + .and_then(serde_json::Value::as_str) + .is_some_and(|tls| tls == "skip"); + if tls_skip && (!policy_middleware.is_empty() || !endpoint_middleware.is_empty()) { + errors.push(format!( + "network policy '{policy_name}' attaches middleware to a tls: skip endpoint" + )); + } + if tls_skip && global_selector_matches_any_middleware(middlewares, endpoint) { + errors.push(format!( + "network policy '{policy_name}' tls: skip endpoint matches a global middleware selector" + )); + } + } + } + errors +} + +fn json_string_array(value: Option<&serde_json::Value>) -> Vec { + value + .and_then(serde_json::Value::as_array) + .map(|values| { + values + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect() + }) + .unwrap_or_default() +} + +fn global_selector_matches_any_middleware( + middlewares: &[serde_json::Value], + endpoint: &serde_json::Value, +) -> bool { + let host = endpoint + .get("host") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + middlewares.iter().any(|mw| { + let Some(selector) = mw.get("endpoints") else { + return false; + }; + let includes = json_string_array(selector.get("include")); + let excludes = json_string_array(selector.get("exclude")); + !includes.is_empty() + && includes.iter().any(|pattern| host_matches(pattern, host)) + && !excludes.iter().any(|pattern| host_matches(pattern, host)) + }) +} + /// Resolve a policy binary path through the container's root filesystem. /// /// On Linux, `/proc//root/` provides access to the container's mount @@ -1316,6 +1724,9 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St allow_all_known_mcp_methods.into(); } } + if !e.middleware.is_empty() { + ep["middleware"] = e.middleware.clone().into(); + } ep }) .collect(); @@ -1341,14 +1752,43 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St entries }) .collect(); - ( - key.clone(), - serde_json::json!({ - "name": rule.name, - "endpoints": endpoints, - "binaries": binaries, - }), - ) + let mut policy = serde_json::json!({ + "name": rule.name, + "endpoints": endpoints, + "binaries": binaries, + }); + if !rule.middleware.is_empty() { + policy["middleware"] = rule.middleware.clone().into(); + } + (key.clone(), policy) + }) + .collect(); + + let network_middlewares: Vec = proto + .network_middlewares + .iter() + .map(|mw| { + let mut value = serde_json::json!({ + "name": mw.name, + "middleware": mw.middleware, + }); + if let Some(config) = &mw.config { + value["config"] = prost_struct_to_json(config); + } + if !mw.on_error.is_empty() { + value["on_error"] = mw.on_error.clone().into(); + } + if let Some(selector) = &mw.endpoints { + let mut endpoints = serde_json::json!({}); + if !selector.include.is_empty() { + endpoints["include"] = selector.include.clone().into(); + } + if !selector.exclude.is_empty() { + endpoints["exclude"] = selector.exclude.clone().into(); + } + value["endpoints"] = endpoints; + } + value }) .collect(); @@ -1357,10 +1797,37 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St "landlock": landlock, "process": process, "network_policies": network_policies, + "network_middlewares": network_middlewares, }) .to_string() } +fn prost_struct_to_json(config: &prost_types::Struct) -> serde_json::Value { + serde_json::Value::Object( + config + .fields + .iter() + .map(|(key, value)| (key.clone(), prost_value_to_json(value))) + .collect(), + ) +} + +fn prost_value_to_json(value: &prost_types::Value) -> serde_json::Value { + match value.kind.as_ref() { + Some(prost_types::value::Kind::NullValue(_)) | None => serde_json::Value::Null, + Some(prost_types::value::Kind::BoolValue(value)) => serde_json::Value::Bool(*value), + Some(prost_types::value::Kind::NumberValue(value)) => serde_json::Number::from_f64(*value) + .map_or(serde_json::Value::Null, serde_json::Value::Number), + Some(prost_types::value::Kind::StringValue(value)) => { + serde_json::Value::String(value.clone()) + } + Some(prost_types::value::Kind::ListValue(value)) => { + serde_json::Value::Array(value.values.iter().map(prost_value_to_json).collect()) + } + Some(prost_types::value::Kind::StructValue(value)) => prost_struct_to_json(value), + } +} + #[cfg(test)] #[allow( clippy::needless_raw_string_hashes, @@ -1407,6 +1874,7 @@ mod tests { path: "/usr/local/bin/claude".to_string(), ..Default::default() }], + ..Default::default() }, ); network_policies.insert( @@ -1422,6 +1890,7 @@ mod tests { path: "/usr/bin/glab".to_string(), ..Default::default() }], + ..Default::default() }, ); ProtoSandboxPolicy { @@ -1439,6 +1908,7 @@ mod tests { run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], } } @@ -2763,6 +3233,7 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -2781,6 +3252,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3783,6 +4255,7 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -3800,6 +4273,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3840,6 +4314,7 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -3857,6 +4332,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3898,6 +4374,7 @@ network_policies: path: "/usr/local/bin/claude".to_string(), ..Default::default() }], + middleware: vec![], }, ); let proto = ProtoSandboxPolicy { @@ -3915,6 +4392,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3958,6 +4436,7 @@ network_policies: path: "/usr/local/bin/aws".to_string(), ..Default::default() }], + middleware: vec![], }, ); let proto = ProtoSandboxPolicy { @@ -3975,6 +4454,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4017,6 +4497,7 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -4034,6 +4515,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4966,6 +5448,7 @@ process: ..Default::default() }], binaries: vec![proposal_binary], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -4983,6 +5466,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); let input = NetworkInput { @@ -5020,6 +5504,7 @@ process: path: "/usr/bin/python".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5037,6 +5522,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); let input = NetworkInput { @@ -5090,6 +5576,7 @@ process: path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5107,6 +5594,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("Failed to create engine from proto"); @@ -5320,6 +5808,7 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5337,6 +5826,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).unwrap(); // Port 443 @@ -6023,6 +6513,7 @@ network_policies: path: "/usr/bin/python3".to_string(), ..Default::default() }], + ..Default::default() }, ); @@ -6279,6 +6770,7 @@ network_policies: path: link_path, ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -6296,6 +6788,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; // Build engine with our PID (symlink resolution will work via /proc/self/root/) @@ -6356,6 +6849,7 @@ network_policies: path: link_path, ..Default::default() }], + ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -6373,6 +6867,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; // Initial load at pid=0 — no symlink expansion @@ -6415,6 +6910,133 @@ network_policies: assert!(eval_l7(&engine, &input)); } + #[test] + fn middleware_chain_orders_global_policy_endpoint_once() { + let data = r#" +network_middlewares: + - name: global-redactor + middleware: openshell/secrets + endpoints: + include: ["api.example.com"] + - name: policy-redactor + middleware: openshell/secrets + - name: endpoint-redactor + middleware: openshell/secrets +network_policies: + api: + name: api + middleware: ["global-redactor", "policy-redactor"] + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + middleware: ["policy-redactor", "endpoint-redactor"] + rules: + - allow: { method: POST, path: "/v1/**" } + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "api.example.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (chain, _) = engine + .query_middleware_chain_with_generation(&input, "/v1/messages") + .unwrap(); + let names: Vec<_> = chain.iter().map(|entry| entry.name.as_str()).collect(); + assert_eq!( + names, + vec!["global-redactor", "policy-redactor", "endpoint-redactor"] + ); + } + + #[test] + fn middleware_policy_validation_rejects_bad_configs() { + let cases = [ + ( + "missing reference", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets +network_policies: + api: + middleware: ["missing"] + endpoints: + - { host: api.example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#, + "unknown middleware config 'missing'", + ), + ( + "invalid on_error", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets + on_error: maybe +"#, + "invalid on_error", + ), + ( + "duplicate names", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets + - name: redactor + middleware: openshell/secrets +"#, + "duplicate middleware config 'redactor'", + ), + ( + "reserved builtin", + r#" +network_middlewares: + - name: sigv4 + middleware: openshell/sigv4 +"#, + "unsupported built-in", + ), + ( + "tls skip attachment", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets +network_policies: + api: + endpoints: + - host: api.example.com + port: 443 + tls: skip + middleware: ["redactor"] + binaries: + - { path: /usr/bin/curl } +"#, + "tls: skip", + ), + ]; + + for (name, data, expected) in cases { + let err = match OpaEngine::from_strings(TEST_POLICY, data) { + Ok(_) => panic!("{name}: expected policy validation failure"), + Err(err) => err.to_string(), + }; + assert!( + err.contains(expected), + "{name}: expected {expected:?} in {err:?}" + ); + } + } + #[test] fn l7_head_denied_when_only_post_allowed() { let engine = OpaEngine::from_strings( diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index 3cbc315026..fa8029c723 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -1047,6 +1047,7 @@ fn network_rule_from_json( name: rule.name.unwrap_or_default(), endpoints, binaries, + middleware: Vec::new(), }) } @@ -1133,6 +1134,7 @@ fn network_endpoint_from_json( credential_signing: String::new(), signing_service: String::new(), signing_region: String::new(), + middleware: Vec::new(), }) } @@ -1829,6 +1831,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }), ..Default::default() }; @@ -1853,6 +1856,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() } } @@ -1916,6 +1920,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() })); }) }; diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 0d2c8c0258..af33317353 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1183,6 +1183,7 @@ async fn handle_tcp_connection( &mut tls_upstream, &ctx, &generation_guard, + Some(&opa_engine), ) .await } @@ -1288,6 +1289,7 @@ async fn handle_tcp_connection( &mut upstream, &ctx, &generation_guard, + Some(&opa_engine), ) .await { diff --git a/proto/middleware.proto b/proto/middleware.proto new file mode 100644 index 0000000000..d5d2ad48d8 --- /dev/null +++ b/proto/middleware.proto @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.middleware.v1; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +service SupervisorMiddleware { + rpc Describe(google.protobuf.Empty) returns (MiddlewareManifest); + rpc ValidateConfig(ValidateConfigRequest) returns (ValidateConfigResponse); + rpc EvaluateHttpRequest(HttpRequestEvaluation) returns (HttpRequestResult); +} + +message MiddlewareManifest { + string api_version = 1; + string name = 2; + string service_version = 3; + repeated MiddlewareBinding bindings = 4; +} + +message MiddlewareBinding { + string id = 1; + string operation = 2; + string phase = 3; +} + +message ValidateConfigRequest { + string api_version = 1; + string binding_id = 2; + google.protobuf.Struct config = 3; +} + +message ValidateConfigResponse { + bool valid = 1; + string reason = 2; +} + +message HttpRequestEvaluation { + string api_version = 1; + string binding_id = 2; + string phase = 3; + RequestContext context = 4; + google.protobuf.Struct config = 5; + HttpRequestTarget target = 6; + map headers = 7; + bytes body = 8; +} + +message RequestContext { + string request_id = 1; + string sandbox_id = 2; + Process originating_process = 3; +} + +message HttpRequestTarget { + string scheme = 1; + string host = 2; + uint32 port = 3; + string method = 4; + string path = 5; + string query = 6; +} + +message Process { + string binary = 1; + uint32 pid = 2; + repeated string ancestors = 3; +} + +enum Decision { + DECISION_UNSPECIFIED = 0; + DECISION_ALLOW = 1; + DECISION_DENY = 2; +} + +message Finding { + string type = 1; + string label = 2; + uint32 count = 3; + string confidence = 4; + string severity = 5; +} + +message HttpRequestResult { + Decision decision = 1; + string reason = 2; + bytes body = 3; + bool has_body = 4; + map add_headers = 5; + repeated Finding findings = 6; + map metadata = 7; +} diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 8a5a593334..5d2bc31a53 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -5,6 +5,8 @@ syntax = "proto3"; package openshell.sandbox.v1; +import "google/protobuf/struct.proto"; + // Sandbox-supervisor configuration and policy messages. // // Conventions: @@ -25,6 +27,8 @@ message SandboxPolicy { ProcessPolicy process = 4; // Network access policies keyed by name (e.g. "claude_code", "gitlab"). map network_policies = 5; + // Reusable supervisor middleware configs for network egress. + repeated NetworkMiddlewareConfig network_middlewares = 6; } // Filesystem access policy. @@ -59,6 +63,27 @@ message NetworkPolicyRule { repeated NetworkEndpoint endpoints = 2; // Allowed binary identities. repeated NetworkBinary binaries = 3; + // Ordered middleware configs applied to every endpoint in this policy. + repeated string middleware = 4; +} + +// A reusable middleware config referenced by network policies/endpoints. +message NetworkMiddlewareConfig { + // Policy-local config name. + string name = 1; + // Built-in or registered middleware implementation name. + string middleware = 2; + // Service-specific configuration. + google.protobuf.Struct config = 3; + // Failure behavior: "fail_closed" (default) or "fail_open". + string on_error = 4; + // Optional global endpoint selector for this config. + MiddlewareEndpointSelector endpoints = 5; +} + +message MiddlewareEndpointSelector { + repeated string include = 1; + repeated string exclude = 2; } // A network endpoint (host + port) with optional L7 inspection config. @@ -143,6 +168,8 @@ message NetworkEndpoint { uint32 json_rpc_max_body_bytes = 22; // MCP-only policy and inspection options. Only used when protocol is "mcp". McpOptions mcp = 23; + // Ordered middleware configs applied to this endpoint after policy-level middleware. + repeated string middleware = 24; } // MCP options are grouped so MCP-specific policy can grow without adding more @@ -175,8 +202,6 @@ message McpOptions { // MCP-family methods at the method layer unless a tool-name policy narrows // tools/call. When unset or false, explicit method rules are required. optional bool allow_all_known_mcp_methods = 2; -} - // Trusted GraphQL operation classification. message GraphqlOperation { // Operation type: "query", "mutation", or "subscription". From e4e6f8fefc23a2d1745b272ac730e97c179e77be Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 26 Jun 2026 13:25:39 -0700 Subject: [PATCH 02/59] fix(supervisor-middleware): harden middleware relay handling Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 + crates/openshell-cli/src/policy_update.rs | 1 + .../src/mechanistic_mapper.rs | 1 + crates/openshell-server/src/grpc/policy.rs | 45 +- .../src/builtins/mod.rs | 2 +- .../src/builtins/secrets.rs | 87 +++- .../src/lib.rs | 298 ++++++++++- .../openshell-supervisor-network/Cargo.toml | 1 + .../src/l7/relay.rs | 489 +++++++++++++++++- .../src/l7/rest.rs | 283 ++++++---- .../openshell-supervisor-network/src/opa.rs | 23 +- 11 files changed, 1059 insertions(+), 172 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f664acbad..3ed95de906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3982,6 +3982,7 @@ dependencies = [ "tokio-tungstenite 0.26.2", "tower-mcp-types", "tracing", + "tracing-subscriber", "uuid", "webpki-roots 1.0.7", ] diff --git a/crates/openshell-cli/src/policy_update.rs b/crates/openshell-cli/src/policy_update.rs index 1f1f647506..824b1dde03 100644 --- a/crates/openshell-cli/src/policy_update.rs +++ b/crates/openshell-cli/src/policy_update.rs @@ -65,6 +65,7 @@ pub fn build_policy_update_plan( ..Default::default() }) .collect(), + middleware: Vec::new(), }; merge_operations.push(PolicyMergeOperation { operation: Some(policy_merge_operation::Operation::AddRule(AddNetworkRule { diff --git a/crates/openshell-sandbox/src/mechanistic_mapper.rs b/crates/openshell-sandbox/src/mechanistic_mapper.rs index 8ee2fc37f9..bb83ddb665 100644 --- a/crates/openshell-sandbox/src/mechanistic_mapper.rs +++ b/crates/openshell-sandbox/src/mechanistic_mapper.rs @@ -162,6 +162,7 @@ pub fn generate_proposals(summaries: &[DenialSummary]) -> Vec { name: rule_name.clone(), endpoints: vec![endpoint], binaries, + middleware: Vec::new(), }; // Compute confidence. diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index cc8ff0d2e2..09e311bb2e 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5746,6 +5746,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -5959,6 +5960,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -6075,6 +6077,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6180,6 +6183,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let mechanistic_submit = handle_submit_policy_analysis( &state, @@ -6257,6 +6261,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let agent_submit = handle_submit_policy_analysis( &state, @@ -6384,6 +6389,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6484,6 +6490,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6584,6 +6591,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6677,6 +6685,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6761,6 +6770,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6849,6 +6859,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -6940,6 +6951,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -7026,6 +7038,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let response = handle_submit_policy_analysis( @@ -7201,6 +7214,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -7297,6 +7311,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -7382,6 +7397,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -7523,6 +7539,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -7648,6 +7665,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let step1 = handle_submit_policy_analysis( &state, @@ -7689,6 +7707,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let step2 = handle_submit_policy_analysis( &state, @@ -7820,6 +7839,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let submit_one = |rule_name: &str, rule: NetworkPolicyRule| { @@ -7928,6 +7948,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let submit_one = || { let state = state.clone(); @@ -8028,6 +8049,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -8159,6 +8181,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; handle_submit_policy_analysis( @@ -8357,6 +8380,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, }; @@ -8385,6 +8409,7 @@ mod tests { path: "/usr/bin/node".to_string(), ..Default::default() }], + ..Default::default() }, }; @@ -8413,6 +8438,7 @@ mod tests { path: "/usr/bin/node".to_string(), ..Default::default() }], + ..Default::default() }, }; @@ -8440,6 +8466,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-1".to_string(), @@ -8508,6 +8535,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, )) .collect(), @@ -8536,6 +8564,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-merge".to_string(), @@ -8609,6 +8638,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }, )) .collect(), @@ -8637,6 +8667,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-new".to_string(), @@ -8773,7 +8804,7 @@ mod tests { allowed_ips: vec!["127.0.0.1".to_string()], ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_err()); @@ -8794,7 +8825,7 @@ mod tests { allowed_ips: vec!["169.254.169.254".to_string()], ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_err()); @@ -8812,7 +8843,7 @@ mod tests { port: 80, ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_err()); @@ -8830,7 +8861,7 @@ mod tests { port: 8080, ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_err()); @@ -8848,7 +8879,7 @@ mod tests { port: 80, ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_err()); @@ -8896,7 +8927,7 @@ mod tests { allowed_ips: vec!["10.0.5.0/24".to_string()], ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_ok()); @@ -8913,7 +8944,7 @@ mod tests { port: 443, ..Default::default() }], - binaries: vec![], + ..Default::default() }; let result = validate_rule_not_always_blocked(&rule); assert!(result.is_ok()); diff --git a/crates/openshell-supervisor-middleware/src/builtins/mod.rs b/crates/openshell-supervisor-middleware/src/builtins/mod.rs index 60572d3e82..d91ee745e9 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/mod.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/mod.rs @@ -1,4 +1,4 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub(crate) mod secrets; +pub mod secrets; diff --git a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs index 6c94eb4396..5721025593 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::HashMap; +use std::sync::LazyLock; use miette::{Result, miette}; use openshell_core::proto::{Decision, Finding, HttpRequestEvaluation, HttpRequestResult}; @@ -9,7 +10,36 @@ use regex::Regex; use crate::BUILTIN_SECRETS; -pub(crate) fn validate_config(config: &prost_types::Struct) -> Result<()> { +/// A named secret-detection pattern. The `kind` is an audit-safe label that +/// flows into findings so operators can see *what* matched without seeing the +/// raw value. +struct SecretPattern { + kind: &'static str, + regex: Regex, +} + +impl SecretPattern { + fn new(kind: &'static str, pattern: &str) -> Self { + Self { + kind, + regex: Regex::new(pattern).expect("valid built-in secret redaction pattern"), + } + } +} + +/// Compiled once: recompiling per request would put regex construction on the +/// egress hot path. +static SECRET_PATTERNS: LazyLock<[SecretPattern; 2]> = LazyLock::new(|| { + [ + SecretPattern::new( + "keyword", + r#"(?i)(api[_-]?key|access[_-]?token|secret|password)(["']?\s*[:=]\s*["'])[^"',\s}]+(["']?)"#, + ), + SecretPattern::new("openai", r"(sk-[A-Za-z0-9_-]{16,})"), + ] +}); + +pub fn validate_config(config: &prost_types::Struct) -> Result<()> { let mode = config .fields .get("secrets") @@ -27,49 +57,54 @@ pub(crate) fn validate_config(config: &prost_types::Struct) -> Result<()> { Ok(()) } -pub(crate) fn evaluate_http_request( - evaluation: &HttpRequestEvaluation, -) -> Result { +pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { let default_config = prost_types::Struct::default(); validate_config(evaluation.config.as_ref().unwrap_or(&default_config))?; let text = String::from_utf8(evaluation.body.clone()) .map_err(|_| miette!("{} requires UTF-8 request bodies", BUILTIN_SECRETS))?; - let (body, count) = redact_common_secrets(&text)?; + let (body, matches) = redact_common_secrets(&text); + let total: u32 = matches + .iter() + .fold(0u32, |acc, (_, count)| acc.saturating_add(*count)); let mut result = HttpRequestResult { decision: Decision::Allow as i32, reason: String::new(), body: body.into_bytes(), - has_body: count > 0, + has_body: !matches.is_empty(), add_headers: HashMap::new(), findings: Vec::new(), metadata: HashMap::new(), }; - if count > 0 { - result.findings.push(Finding { - r#type: "secret.common".into(), - label: "common secret pattern".into(), - count, - confidence: "medium".into(), - severity: "medium".into(), - }); + if !matches.is_empty() { + // One finding per matched pattern kind, so audit shows what matched. + for (kind, count) in &matches { + result.findings.push(Finding { + r#type: format!("secret.{kind}"), + label: format!("{kind} secret pattern"), + count: *count, + confidence: "medium".into(), + severity: "medium".into(), + }); + } result .metadata - .insert("secrets_redacted".into(), count.to_string()); + .insert("secrets_redacted".into(), total.to_string()); } Ok(result) } -fn redact_common_secrets(input: &str) -> Result<(String, u32)> { - let patterns = [ - r#"(?i)(api[_-]?key|access[_-]?token|secret|password)(["']?\s*[:=]\s*["'])[^"',\s}]+(["']?)"#, - r#"(sk-[A-Za-z0-9_-]{16,})"#, - ]; +/// Redact every configured secret pattern, returning the transformed text and +/// the per-kind match counts (only kinds that matched are included). +fn redact_common_secrets(input: &str) -> (String, Vec<(&'static str, u32)>) { let mut output = input.to_string(); - let mut count = 0u32; - for pattern in patterns { - let regex = Regex::new(pattern).map_err(|e| miette!("{e}"))?; - count = count.saturating_add(regex.find_iter(&output).count() as u32); - output = regex + let mut matches = Vec::new(); + for pattern in SECRET_PATTERNS.iter() { + let count = u32::try_from(pattern.regex.find_iter(&output).count()).unwrap_or(u32::MAX); + if count > 0 { + matches.push((pattern.kind, count)); + } + output = pattern + .regex .replace_all(&output, |captures: ®ex::Captures<'_>| { if captures.len() >= 4 { format!("{}{}[REDACTED]{}", &captures[1], &captures[2], &captures[3]) @@ -79,5 +114,5 @@ fn redact_common_secrets(input: &str) -> Result<(String, u32)> { }) .into_owned(); } - Ok((output, count)) + (output, matches) } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 7d9161fcf3..b68d83c86e 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -100,7 +100,7 @@ pub struct ChainOutcome { pub applied: Vec, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct NamespacedFinding { pub middleware: String, pub finding: Finding, @@ -112,6 +112,48 @@ pub struct MiddlewareInvocation { pub implementation: String, pub decision: Decision, pub transformed: bool, + /// True when the middleware could not be evaluated and `on_error` was applied + /// (service error, malformed/unsafe response, etc.). The `decision` reflects + /// the `on_error` outcome, not a decision the middleware actually returned. + pub failed: bool, +} + +enum OnErrorAction { + /// `fail_open`: skip this middleware, leaving the request unchanged. + FailOpen, + /// `fail_closed`: short-circuit the chain and deny with the given reason. + FailClosed(String), +} + +/// Apply a middleware entry's `on_error` policy after a failure (service error or +/// malformed response). Records a `failed` invocation for telemetry in both cases. +fn apply_on_error( + entry: &ChainEntry, + reason: &str, + applied: &mut Vec, +) -> OnErrorAction { + match entry.on_error { + OnError::FailOpen => { + applied.push(MiddlewareInvocation { + name: entry.name.clone(), + implementation: entry.implementation.clone(), + decision: Decision::Allow, + transformed: false, + failed: true, + }); + OnErrorAction::FailOpen + } + OnError::FailClosed => { + applied.push(MiddlewareInvocation { + name: entry.name.clone(), + implementation: entry.implementation.clone(), + decision: Decision::Deny, + transformed: false, + failed: true, + }); + OnErrorAction::FailClosed(format!("middleware_failed: {reason}")) + } + } } #[derive(Clone)] @@ -150,20 +192,33 @@ impl ChainRunner { .await { Ok(result) => result.into_inner(), - Err(err) => match entry.on_error { - OnError::FailOpen => { - applied.push(MiddlewareInvocation { - name: entry.name.clone(), - implementation: entry.implementation.clone(), - decision: Decision::Allow, - transformed: false, - }); - continue; + Err(err) => { + match apply_on_error(entry, &safe_reason(&err.to_string()), &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } } - OnError::FailClosed => { + } + }; + + // A result proposing unsafe header mutations is a malformed response: + // route it through `on_error` instead of applying any of it. + if validate_header_mutations(&headers, &result.add_headers).is_err() { + match apply_on_error(entry, "unsafe_response_headers", &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { return Ok(ChainOutcome { allowed: false, - reason: format!("middleware_failed: {}", safe_reason(&err.to_string())), + reason, body, added_headers, findings, @@ -171,17 +226,15 @@ impl ChainRunner { applied, }); } - }, - }; - - validate_header_mutations(&headers, &result.add_headers)?; + } + } for (name, value) in &result.add_headers { headers.insert(name.to_ascii_lowercase(), value.clone()); added_headers.insert(name.to_ascii_lowercase(), value.clone()); } let transformed = result.has_body; if result.has_body { - body = result.body.clone(); + result.body.clone_into(&mut body); } for finding in result.findings { findings.push(NamespacedFinding { @@ -200,6 +253,7 @@ impl ChainRunner { implementation: entry.implementation.clone(), decision: Decision::try_from(result.decision).unwrap_or(Decision::Unspecified), transformed, + failed: false, }); if result.decision == Decision::Deny as i32 { return Ok(ChainOutcome { @@ -264,7 +318,7 @@ fn validate_header_mutations( mutations: &HashMap, ) -> Result<()> { let mut seen = HashSet::new(); - for name in mutations.keys() { + for (name, value) in mutations { let lower = name.to_ascii_lowercase(); if !seen.insert(lower.clone()) || existing_headers.contains_key(&lower) { return Err(miette!( @@ -274,10 +328,27 @@ fn validate_header_mutations( if !is_safe_append_header(&lower) { return Err(miette!("middleware cannot append unsafe header '{name}'")); } + // Reject CR/LF and other control characters in the value: writing them + // verbatim into the upstream header block would enable header injection + // and request smuggling past the credential boundary. + if !is_safe_header_value(value) { + return Err(miette!( + "middleware cannot append header '{name}' with an unsafe value" + )); + } } Ok(()) } +/// A header value is safe to append only if it contains no control characters. +/// Horizontal tab, printable ASCII, and obs-text (>= 0x80) are permitted; CR, LF, +/// NUL, and other control bytes are rejected. +fn is_safe_header_value(value: &str) -> bool { + value + .bytes() + .all(|b| b == b'\t' || (0x20..=0x7e).contains(&b) || b >= 0x80) +} + fn is_safe_append_header(name: &str) -> bool { if name.is_empty() || name.contains(':') @@ -312,13 +383,12 @@ mod tests { name: name.into(), implementation: BUILTIN_SECRETS.into(), config: prost_types::Struct { - fields: [( + fields: std::iter::once(( "secrets".into(), prost_types::Value { kind: Some(prost_types::value::Kind::StringValue("redact".into())), }, - )] - .into_iter() + )) .collect(), }, on_error, @@ -427,11 +497,191 @@ mod tests { fn unsafe_header_mutation_is_rejected() { let err = validate_header_mutations( &BTreeMap::new(), - &[("Authorization".into(), "Bearer nope".into())] - .into_iter() - .collect(), + &std::iter::once(("Authorization".into(), "Bearer nope".into())).collect(), ) .expect_err("unsafe header"); assert!(err.to_string().contains("unsafe header")); } + + #[test] + fn header_value_with_crlf_is_rejected() { + // A safe header *name* with a CRLF-bearing value must still be rejected, + // otherwise it would inject extra headers into the upstream request. + let err = validate_header_mutations( + &BTreeMap::new(), + &std::iter::once(( + "x-openshell-middleware-inject".into(), + "ok\r\nAuthorization: Bearer evil".into(), + )) + .collect(), + ) + .expect_err("crlf value"); + assert!(err.to_string().contains("unsafe value")); + } + + /// A mock middleware that returns a fixed, caller-supplied result for every + /// evaluation. Used to exercise chain behavior the built-in cannot produce + /// (explicit deny, metadata, findings, unsafe header mutations). + struct ScriptedService { + result: openshell_core::proto::HttpRequestResult, + } + + #[tonic::async_trait] + impl SupervisorMiddleware for ScriptedService { + async fn describe( + &self, + _request: Request<()>, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::MiddlewareManifest::default(), + )) + } + + async fn validate_config( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new(self.result.clone())) + } + } + + fn allow_result() -> openshell_core::proto::HttpRequestResult { + openshell_core::proto::HttpRequestResult { + decision: Decision::Allow as i32, + reason: String::new(), + body: Vec::new(), + has_body: false, + add_headers: HashMap::new(), + findings: Vec::new(), + metadata: HashMap::new(), + } + } + + #[tokio::test] + async fn deny_decision_short_circuits_chain() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + result: openshell_core::proto::HttpRequestResult { + decision: Decision::Deny as i32, + reason: "blocked_by_policy".into(), + ..allow_result() + }, + })); + let outcome = runner + .evaluate( + &[ + entry("first", OnError::FailClosed), + entry("second", OnError::FailClosed), + ], + input("hello"), + ) + .await + .expect("evaluate"); + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "blocked_by_policy"); + // The deny short-circuits the chain: the second middleware never runs. + assert_eq!(outcome.applied.len(), 1); + assert_eq!(outcome.applied[0].decision, Decision::Deny); + assert!(!outcome.applied[0].failed); + } + + #[tokio::test] + async fn metadata_and_findings_are_namespaced_per_config() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + result: openshell_core::proto::HttpRequestResult { + findings: vec![Finding { + r#type: "pii.email".into(), + label: "email address".into(), + count: 2, + confidence: "high".into(), + severity: "medium".into(), + }], + metadata: std::iter::once(("sensitivity".to_string(), "high".to_string())) + .collect(), + ..allow_result() + }, + })); + let outcome = runner + .evaluate( + &[ + entry("alpha", OnError::FailClosed), + entry("beta", OnError::FailClosed), + ], + input("hello"), + ) + .await + .expect("evaluate"); + assert!(outcome.allowed); + // Metadata is bucketed under each config's local name, so two configs + // emitting the same key do not collide. + assert_eq!(outcome.metadata["alpha"]["sensitivity"], "high"); + assert_eq!(outcome.metadata["beta"]["sensitivity"], "high"); + // Findings are tagged with the emitting config's name. + assert_eq!(outcome.findings.len(), 2); + assert_eq!(outcome.findings[0].middleware, "alpha"); + assert_eq!(outcome.findings[1].middleware, "beta"); + assert_eq!(outcome.findings[0].finding.r#type, "pii.email"); + assert_eq!(outcome.findings[0].finding.count, 2); + } + + fn unsafe_header_service() -> ScriptedService { + ScriptedService { + result: openshell_core::proto::HttpRequestResult { + add_headers: std::iter::once(( + "x-openshell-middleware-inject".to_string(), + "ok\r\nHost: evil".to_string(), + )) + .collect(), + ..allow_result() + }, + } + } + + #[tokio::test] + async fn malformed_response_headers_fail_closed_denies() { + let runner = ChainRunner::new(Arc::new(unsafe_header_service())); + let outcome = runner + .evaluate(&[entry("redact", OnError::FailClosed)], input("hello")) + .await + .expect("evaluate"); + assert!(!outcome.allowed); + assert!(outcome.reason.starts_with("middleware_failed:")); + assert!(outcome.applied.iter().any(|inv| inv.failed)); + // The unsafe header is never forwarded. + assert!(outcome.added_headers.is_empty()); + } + + #[tokio::test] + async fn malformed_response_headers_fail_open_continues() { + let runner = ChainRunner::new(Arc::new(unsafe_header_service())); + let outcome = runner + .evaluate(&[entry("redact", OnError::FailOpen)], input("hello")) + .await + .expect("evaluate"); + assert!(outcome.allowed); + assert_eq!(outcome.body, b"hello"); + assert!(outcome.added_headers.is_empty()); + assert_eq!(outcome.applied.len(), 1); + assert!(outcome.applied[0].failed); + } } diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index fd8fad5f75..b8cae51133 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -55,6 +55,7 @@ tempfile = "3" temp-env = "0.3" tokio-tungstenite = { workspace = true } futures = { workspace = true } +tracing-subscriber = { workspace = true } [target.'cfg(unix)'.dev-dependencies] libc = "0.2" diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 4d501d0a39..6e5c3c4e9e 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -783,9 +783,18 @@ async fn apply_middleware_chain( if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); } - let buffered = - crate::l7::rest::buffer_request_body_for_middleware(&req, client, Some(generation_guard)) - .await?; + let buffered = match crate::l7::rest::buffer_request_body_for_middleware( + &req, + client, + Some(generation_guard), + ) + .await? + { + crate::l7::rest::BufferResult::Buffered(buffered) => buffered, + crate::l7::rest::BufferResult::OverCapacity { recoverable } => { + return Ok(resolve_unbuffered_body(ctx, req, &chain, recoverable)); + } + }; let headers = safe_middleware_headers(&buffered.headers)?; let input = openshell_supervisor_middleware::HttpRequestInput { request_id: uuid::Uuid::new_v4().to_string(), @@ -819,6 +828,52 @@ async fn apply_middleware_chain( } } +/// Apply the chain's `on_error` policy when the request body cannot be buffered +/// for inspection because it exceeds the size cap. The RFC treats an unbufferable +/// body as an `on_error` event: it is denied unless every attached middleware is +/// `fail_open`, and passing it through is only safe when no bytes were consumed. +fn resolve_unbuffered_body( + ctx: &L7EvalContext, + req: crate::l7::provider::L7Request, + chain: &[openshell_supervisor_middleware::ChainEntry], + recoverable: bool, +) -> MiddlewareApplyResult { + let all_fail_open = chain + .iter() + .all(|entry| entry.on_error == openshell_supervisor_middleware::OnError::FailOpen); + if recoverable && all_fail_open { + emit_middleware_body_unavailable(ctx, false); + return MiddlewareApplyResult::Allowed(req); + } + emit_middleware_body_unavailable(ctx, true); + MiddlewareApplyResult::Denied("middleware_failed: request_body_over_capacity".into()) +} + +fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if denied { + SeverityId::High + } else { + SeverityId::Medium + }) + .finding_info(FindingInfo::new( + "openshell.middleware.body_unavailable", + "Supervisor middleware could not inspect request body", + )) + .evidence_pairs(&[ + ("policy", ctx.policy_name.as_str()), + ("host", ctx.host.as_str()), + ("disposition", if denied { "denied" } else { "fail_open" }), + ]) + .message(if denied { + "Request body exceeded middleware inspection cap; denied" + } else { + "Request body exceeded middleware inspection cap; passed through (fail_open)" + }) + .build(); + ocsf_emit!(event); +} + fn safe_middleware_headers(headers: &[u8]) -> Result> { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; @@ -885,14 +940,37 @@ fn emit_middleware_events( .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "middleware") .message(format!( - "MIDDLEWARE {} {} decision={:?} transformed={}", + "MIDDLEWARE {} {} decision={:?} transformed={} failed={}", invocation.name, invocation.implementation, invocation.decision, - invocation.transformed + invocation.transformed, + invocation.failed )) .build(); ocsf_emit!(event); + + // A middleware that failed but was bypassed under `fail_open` is an + // enforcement failure operators must be able to alert on, even though the + // request proceeded. + if invocation.failed && allowed { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Medium) + .finding_info(FindingInfo::new( + "openshell.middleware.failure", + "Supervisor middleware failed open", + )) + .evidence_pairs(&[ + ("middleware", invocation.name.as_str()), + ("implementation", invocation.implementation.as_str()), + ]) + .message(format!( + "Middleware {} failed and was bypassed (fail_open)", + invocation.name + )) + .build(); + ocsf_emit!(event); + } } if !outcome.allowed && outcome.reason.starts_with("middleware_failed:") { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) @@ -2658,6 +2736,407 @@ network_policies: .unwrap(); } + #[tokio::test] + async fn l7_rest_middleware_over_capacity_fails_closed() { + let (config, tunnel_engine, ctx) = + middleware_relay_context("openshell/secrets", "fail_closed"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + // A declared body far above the 256 KiB inspection cap must be denied + // (fail-closed) before the body is read or reaches the upstream. + let request = format!( + "POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + 300 * 1024 + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden")); + assert!(response.contains("request_body_over_capacity")); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive request bytes" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[test] + fn over_capacity_resolution_honors_on_error() { + use openshell_supervisor_middleware::{ChainEntry, OnError}; + + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "p".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let req = || crate::l7::provider::L7Request { + action: "POST".into(), + target: "/v1".into(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + let fail_open = ChainEntry { + name: "m".into(), + implementation: "openshell/secrets".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }; + let fail_closed = ChainEntry { + on_error: OnError::FailClosed, + ..fail_open.clone() + }; + + // Recoverable (Content-Length over cap, nothing consumed) + all fail-open + // -> stream through unprocessed. + assert!(matches!( + resolve_unbuffered_body(&ctx, req(), std::slice::from_ref(&fail_open), true), + MiddlewareApplyResult::Allowed(_) + )); + // Any fail-closed entry -> deny. + assert!(matches!( + resolve_unbuffered_body(&ctx, req(), &[fail_open.clone(), fail_closed], true), + MiddlewareApplyResult::Denied(_) + )); + // Not recoverable (chunked overflow already consumed bytes) -> deny even + // when every entry is fail-open. + assert!(matches!( + resolve_unbuffered_body(&ctx, req(), &[fail_open], false), + MiddlewareApplyResult::Denied(_) + )); + } + + /// Tracing layer that captures emitted `OcsfEvent`s for assertions. + struct OcsfCaptureLayer(Arc>>); + + impl tracing_subscriber::Layer for OcsfCaptureLayer { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + if event.metadata().target() == openshell_ocsf::OCSF_TARGET + && let Some(ocsf_event) = openshell_ocsf::clone_current_event() + { + self.0.lock().unwrap().push(ocsf_event); + } + } + } + + #[test] + fn middleware_ocsf_events_are_audit_safe() { + use openshell_supervisor_middleware::{ + ChainOutcome, MiddlewareInvocation, NamespacedFinding, + }; + use tracing_subscriber::layer::SubscriberExt; + + const RAW_SECRET: &str = "sk-RAWSECRETVALUE0123456789"; + + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry().with(OcsfCaptureLayer(Arc::clone(&events))); + let _guard = tracing::subscriber::set_default(subscriber); + + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "rest_api".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let req = crate::l7::provider::L7Request { + action: "POST".into(), + target: "/v1/messages".into(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + let outcome = ChainOutcome { + allowed: true, + reason: String::new(), + // The transformed body still holds the raw secret; emission must never + // serialize it. + body: format!(r#"{{"api_key":"{RAW_SECRET}"}}"#).into_bytes(), + added_headers: BTreeMap::new(), + findings: vec![NamespacedFinding { + middleware: "redact-secrets".into(), + finding: openshell_core::proto::Finding { + r#type: "secret.common".into(), + label: "common secret pattern".into(), + count: 1, + confidence: "medium".into(), + severity: "medium".into(), + }, + }], + metadata: BTreeMap::new(), + applied: vec![MiddlewareInvocation { + name: "redact-secrets".into(), + implementation: "openshell/secrets".into(), + decision: openshell_core::proto::Decision::Allow, + transformed: true, + failed: false, + }], + }; + + emit_middleware_events(&ctx, &req, &outcome); + + let captured = events.lock().unwrap(); + // Per-invocation decisions are HTTP Activity (class 4002). + assert!( + captured.iter().any(|e| e.class_uid() == 4002), + "expected an HTTP Activity event for the middleware invocation" + ); + // Findings are Detection Finding (class 2004) with the finding's severity. + let finding_event = captured + .iter() + .find(|e| e.class_uid() == 2004) + .expect("expected a Detection Finding event"); + assert_eq!(finding_event.base().severity, SeverityId::Medium); + + // No raw payload material may appear in any emitted event. + let serialized = serde_json::to_string(&*captured).expect("serialize events"); + assert!( + !serialized.contains(RAW_SECRET), + "raw secret leaked into OCSF events: {serialized}" + ); + // Safe finding metadata is still present. + assert!(serialized.contains("secret.common")); + } + + #[tokio::test] + async fn passthrough_relay_runs_middleware_redaction() { + // A no-protocol endpoint takes the credential-injection passthrough path; + // policy-level middleware must still inspect and redact its body. + let data = r#" +network_middlewares: + - name: request-middleware + middleware: openshell/secrets + on_error: fail_closed +network_policies: + passthrough_api: + name: passthrough_api + middleware: ["request-middleware"] + endpoints: + - host: api.example.test + port: 8080 + binaries: + - { path: /usr/bin/curl } +"#; + let engine = Arc::new(OpaEngine::from_strings(TEST_POLICY, data).unwrap()); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 8080, + policy_name: "passthrough_api".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let engine_task = Arc::clone(&engine); + let relay = tokio::spawn(async move { + relay_passthrough_with_credentials( + &mut relay_client, + &mut relay_upstream, + &ctx, + &generation_guard, + Some(engine_task.as_ref()), + ) + .await + }); + + let body = br#"{"api_key":"sk-1234567890abcdef"}"#; + let request = format!( + "POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut upstream_request = [0u8; 1024]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_request), + ) + .await + .expect("request should reach upstream") + .unwrap(); + let upstream_request = String::from_utf8_lossy(&upstream_request[..n]); + assert!( + upstream_request.contains(r#""api_key":"[REDACTED]""#), + "unexpected upstream request: {upstream_request:?}" + ); + assert!(!upstream_request.contains("sk-1234567890abcdef")); + + upstream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut client_response = [0u8; 512]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + app.read(&mut client_response), + ) + .await + .expect("response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("204 No Content")); + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn websocket_upgrade_request_is_inspected_and_denied() { + // The WebSocket upgrade handshake is an HTTP request the hook can inspect + // and deny: a fail-closed middleware blocks the upgrade before it is + // forwarded. + let data = r#" +network_middlewares: + - name: request-middleware + middleware: example/unavailable + on_error: fail_closed +network_policies: + ws_api: + name: ws_api + middleware: ["request-middleware"] + endpoints: + - host: gateway.example.test + port: 443 + protocol: websocket + enforcement: enforce + rules: + - allow: + method: GET + path: "/ws" + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "gateway.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&endpoint_config.unwrap()).unwrap(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "gateway.example.test".into(), + port: 443, + policy_name: "ws_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all( + b"GET /ws HTTP/1.1\r\nHost: gateway.example.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n", + ) + .await + .unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden")); + assert!(response.contains("middleware_failed")); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive the upgrade request" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[tokio::test] async fn passthrough_relay_injects_token_grant_authorization_header() { let (generation_guard, ctx, fixture) = diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 1a4036abd7..2c85cacf67 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -774,11 +774,23 @@ pub(crate) struct BufferedRequestBody { pub(crate) body: Vec, } +/// Result of attempting to buffer a request body for middleware inspection. +pub(crate) enum BufferResult { + /// The full body was buffered within the size cap. + Buffered(BufferedRequestBody), + /// The body exceeded the inspection cap. `recoverable` is true when no body + /// bytes were consumed yet (a declared `Content-Length` over the cap), so the + /// request can still be streamed through unprocessed under fail-open. It is + /// false once bytes have been consumed (chunked overflow), where denying is + /// the only safe outcome. + OverCapacity { recoverable: bool }, +} + pub(crate) async fn buffer_request_body_for_middleware( req: &L7Request, client: &mut C, generation_guard: Option<&PolicyGenerationGuard>, -) -> Result { +) -> Result { let header_end = req .raw_header .windows(4) @@ -787,17 +799,19 @@ pub(crate) async fn buffer_request_body_for_middleware( let headers = req.raw_header[..header_end].to_vec(); let already_read = &req.raw_header[header_end..]; match req.body_length { - BodyLength::None => Ok(BufferedRequestBody { + BodyLength::None => Ok(BufferResult::Buffered(BufferedRequestBody { headers, body: already_read.to_vec(), - }), + })), BodyLength::ContentLength(len) => { - let len = usize::try_from(len) - .map_err(|_| miette!("request body is too large for middleware"))?; + // The declared length is known before any further reads, so an + // over-cap body here has not consumed the stream and can be passed + // through unprocessed if every middleware is fail-open. + let Ok(len) = usize::try_from(len) else { + return Ok(BufferResult::OverCapacity { recoverable: true }); + }; if len > MAX_MIDDLEWARE_BODY_BYTES { - return Err(miette!( - "middleware buffers at most {MAX_MIDDLEWARE_BODY_BYTES} request body bytes" - )); + return Ok(BufferResult::OverCapacity { recoverable: true }); } let initial_len = already_read.len().min(len); let mut body = Vec::with_capacity(len); @@ -818,11 +832,21 @@ pub(crate) async fn buffer_request_body_for_middleware( body.extend_from_slice(&buf[..n]); remaining -= n; } - Ok(BufferedRequestBody { headers, body }) + Ok(BufferResult::Buffered(BufferedRequestBody { + headers, + body, + })) } BodyLength::Chunked => { - let body = collect_chunked_body(client, already_read, generation_guard).await?; - Ok(BufferedRequestBody { headers, body }) + // Chunked bodies are decoded incrementally into the payload bytes + // middleware expects. On overflow, we have already consumed wire + // bytes from the client stream and cannot re-enter the normal raw + // relay path without a separate splice-through buffer. + Ok(collect_chunked_body(client, already_read, generation_guard) + .await + .map_or(BufferResult::OverCapacity { recoverable: false }, |body| { + BufferResult::Buffered(BufferedRequestBody { headers, body }) + })) } } } @@ -835,7 +859,7 @@ pub(crate) fn rebuild_request_with_buffered_body( ) -> Result { let mut header_bytes = set_content_length(headers, body.len())?; header_bytes = strip_header(&header_bytes, "transfer-encoding")?; - header_bytes = append_headers(&header_bytes, add_headers)?; + header_bytes = append_headers(&header_bytes, add_headers); header_bytes.extend_from_slice(body); Ok(L7Request { action: req.action.clone(), @@ -900,15 +924,11 @@ async fn collect_and_rewrite_request_body( } BodyLength::Chunked => { let body = collect_chunked_body(client, already_read, generation_guard).await?; - if body_bytes_contain_reserved_marker(&body) { - return Err(miette!( - "request body credential rewrite does not support chunked bodies containing credential placeholders" - )); - } - Ok(PreparedRequestBody { - headers: rewritten_headers.to_vec(), - body, - }) + let (mut headers, body) = + rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; + headers = set_content_length(&headers, body.len())?; + headers = strip_header(&headers, "transfer-encoding")?; + Ok(PreparedRequestBody { headers, body }) } } } @@ -1076,37 +1096,15 @@ async fn collect_chunked_body( already_read: &[u8], generation_guard: Option<&PolicyGenerationGuard>, ) -> Result> { - let mut read_buf = [0u8; RELAY_BUF_SIZE]; - let mut parse_buf = Vec::from(already_read); - let mut pos = 0usize; + let mut buffered_pos = 0usize; + let mut body = Vec::new(); loop { - if parse_buf.len() > MAX_REWRITE_BODY_BYTES { - return Err(miette!( - "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" - )); - } - - let size_line_end = loop { - if let Some(end) = find_crlf(&parse_buf, pos) { - break end; - } - let n = client.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended before chunk-size line")); - } - if let Some(guard) = generation_guard { - guard.ensure_current()?; - } - parse_buf.extend_from_slice(&read_buf[..n]); - if parse_buf.len() > MAX_REWRITE_BODY_BYTES { - return Err(miette!( - "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" - )); - } - }; - - let size_line = std::str::from_utf8(&parse_buf[pos..size_line_end]) + let size_line = + read_chunked_line(client, already_read, &mut buffered_pos, generation_guard) + .await + .map_err(|e| miette!("Chunked body ended before chunk-size line: {e}"))?; + let size_line = std::str::from_utf8(&size_line) .into_diagnostic() .map_err(|_| miette!("Invalid UTF-8 in chunk-size line"))?; let size_token = size_line @@ -1117,64 +1115,109 @@ async fn collect_chunked_body( let chunk_size = usize::from_str_radix(size_token, 16) .into_diagnostic() .map_err(|_| miette!("Invalid chunk size token: {size_token:?}"))?; - pos = size_line_end + 2; if chunk_size == 0 { loop { - let trailer_end = loop { - if let Some(end) = find_crlf(&parse_buf, pos) { - break end; - } - let n = client.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended before trailer terminator")); - } - if let Some(guard) = generation_guard { - guard.ensure_current()?; - } - parse_buf.extend_from_slice(&read_buf[..n]); - if parse_buf.len() > MAX_REWRITE_BODY_BYTES { - return Err(miette!( - "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" - )); - } - }; - let trailer_line = &parse_buf[pos..trailer_end]; - pos = trailer_end + 2; + let trailer_line = + read_chunked_line(client, already_read, &mut buffered_pos, generation_guard) + .await + .map_err(|e| { + miette!("Chunked body ended before trailer terminator: {e}") + })?; if trailer_line.is_empty() { - return Ok(parse_buf); + return Ok(body); } } } - let chunk_end = pos - .checked_add(chunk_size) - .ok_or_else(|| miette!("Chunk size overflow"))?; - let chunk_with_crlf_end = chunk_end - .checked_add(2) - .ok_or_else(|| miette!("Chunk size overflow"))?; - while parse_buf.len() < chunk_with_crlf_end { - let n = client.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended mid-chunk")); - } - if let Some(guard) = generation_guard { - guard.ensure_current()?; - } - parse_buf.extend_from_slice(&read_buf[..n]); - if parse_buf.len() > MAX_REWRITE_BODY_BYTES { - return Err(miette!( - "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" - )); - } + if body.len().saturating_add(chunk_size) > MAX_REWRITE_BODY_BYTES { + return Err(miette!( + "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" + )); } - if &parse_buf[chunk_end..chunk_with_crlf_end] != b"\r\n" { + read_buffered_exact( + client, + already_read, + &mut buffered_pos, + chunk_size, + &mut body, + generation_guard, + ) + .await + .map_err(|e| miette!("Chunked body ended mid-chunk: {e}"))?; + + let mut chunk_crlf = Vec::with_capacity(2); + read_buffered_exact( + client, + already_read, + &mut buffered_pos, + 2, + &mut chunk_crlf, + generation_guard, + ) + .await + .map_err(|e| miette!("Chunked body ended before chunk terminator: {e}"))?; + if chunk_crlf.as_slice() != b"\r\n" { return Err(miette!("Chunk missing terminating CRLF")); } - pos = chunk_with_crlf_end; } } +async fn read_chunked_line( + client: &mut C, + already_read: &[u8], + buffered_pos: &mut usize, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result> { + let mut line = Vec::new(); + loop { + let byte = read_buffered_byte(client, already_read, buffered_pos, generation_guard).await?; + line.push(byte); + if line.len() > MAX_REWRITE_BODY_BYTES { + return Err(miette!( + "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" + )); + } + if line.ends_with(b"\r\n") { + line.truncate(line.len() - 2); + return Ok(line); + } + } +} + +async fn read_buffered_exact( + client: &mut C, + already_read: &[u8], + buffered_pos: &mut usize, + len: usize, + out: &mut Vec, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result<()> { + for _ in 0..len { + let byte = read_buffered_byte(client, already_read, buffered_pos, generation_guard).await?; + out.push(byte); + } + Ok(()) +} + +async fn read_buffered_byte( + client: &mut C, + already_read: &[u8], + buffered_pos: &mut usize, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result { + if *buffered_pos < already_read.len() { + let byte = already_read[*buffered_pos]; + *buffered_pos += 1; + return Ok(byte); + } + let byte = client.read_u8().await.into_diagnostic()?; + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + Ok(byte) +} + fn content_type(headers: &str) -> Option { headers.lines().skip(1).find_map(|line| { let (name, value) = line.split_once(':')?; @@ -1262,9 +1305,9 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { fn append_headers( headers: &[u8], add_headers: &std::collections::BTreeMap, -) -> Result> { +) -> Vec { if add_headers.is_empty() { - return Ok(headers.to_vec()); + return headers.to_vec(); } let split = headers .windows(4) @@ -1279,7 +1322,7 @@ fn append_headers( out.extend_from_slice(value.as_bytes()); } out.extend_from_slice(b"\r\n\r\n"); - Ok(out) + out } pub(crate) fn request_is_websocket_upgrade(raw_header: &[u8]) -> bool { @@ -3151,6 +3194,20 @@ mod tests { } } + #[tokio::test] + async fn collect_chunked_body_decodes_payload_bytes() { + let mut client = tokio::io::empty(); + let body = collect_chunked_body( + &mut client, + b"5\r\nhello\r\n6;ext=value\r\n world\r\n0\r\nx-checksum: abc\r\n\r\n", + None, + ) + .await + .expect("chunked body should decode"); + + assert_eq!(body, b"hello world"); + } + /// SEC-009: Bare LF in headers enables header injection. #[tokio::test] async fn reject_bare_lf_in_headers() { @@ -5257,6 +5314,38 @@ mod tests { assert!(!forwarded.contains("OPENSHELL-RESOLVE-ENV")); } + #[tokio::test] + async fn relay_request_body_rewrite_normalizes_chunked_payload() { + let (_, resolver) = SecretResolver::from_provider_env( + [("API_TOKEN".to_string(), "provider-real-token".to_string())] + .into_iter() + .collect(), + ); + let resolver = resolver.expect("resolver"); + let alias = "provider.v1-OPENSHELL-RESOLVE-ENV-API_TOKEN"; + let raw = format!( + "POST /api/messages HTTP/1.1\r\n\ + Host: api.example.com\r\n\ + Authorization: Bearer {alias}\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 5\r\nhello\r\n0\r\n\r\n", + ); + + let forwarded = relay_and_capture_with_options( + raw.into_bytes(), + BodyLength::Chunked, + Some(&resolver), + true, + ) + .await + .expect("relay should succeed"); + + assert!(forwarded.contains("Authorization: Bearer provider-real-token\r\n")); + assert!(forwarded.contains("Content-Length: 5\r\n")); + assert!(!forwarded.contains("Transfer-Encoding: chunked\r\n")); + assert!(forwarded.ends_with("hello")); + } + #[tokio::test] async fn relay_request_body_rewrites_percent_encoded_canonical_urlencoded_token() { let (_, resolver) = SecretResolver::from_provider_env( diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 451c57e597..a584b414bc 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -777,11 +777,7 @@ fn query_middleware_chain_locked( .map_err(|e| miette::miette!("{e}"))?; let contexts = parse_middleware_contexts(&contexts_val); let Some(context) = select_middleware_context(&contexts, request_path) else { - return Ok(global_middleware_entries( - &configs, - &input.host, - &HashSet::new(), - )?); + return global_middleware_entries(&configs, &input.host, &HashSet::new()); }; let mut explicit = Vec::new(); @@ -876,12 +872,16 @@ fn middleware_selector_matches(config: ®orus::Value, host: &str) -> bool { let Some(selector) = get_field(config, "endpoints") else { return false; }; - let includes = get_str_array(selector, "include"); - let excludes = get_str_array(selector, "exclude"); - let included = - !includes.is_empty() && includes.iter().any(|pattern| host_matches(pattern, host)); - let excluded = excludes.iter().any(|pattern| host_matches(pattern, host)); - included && !excluded + let include_patterns = get_str_array(selector, "include"); + let exclude_patterns = get_str_array(selector, "exclude"); + let matches_include = !include_patterns.is_empty() + && include_patterns + .iter() + .any(|pattern| host_matches(pattern, host)); + let matches_exclude = exclude_patterns + .iter() + .any(|pattern| host_matches(pattern, host)); + matches_include && !matches_exclude } fn host_matches(pattern: &str, host: &str) -> bool { @@ -956,7 +956,6 @@ fn regorus_value_to_prost(value: ®orus::Value) -> prost_types::Value { }) .collect(), }), - regorus::Value::Null | regorus::Value::Undefined => Kind::NullValue(0), _ => Kind::NullValue(0), }), } From 27650555f7b122bf381317638cf5e79184901e95 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 26 Jun 2026 16:20:15 -0700 Subject: [PATCH 03/59] fix(supervisor-middleware): default stored policy rule fields Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/grpc/policy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 09e311bb2e..ad4fdf5ba2 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -7100,6 +7100,7 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], + ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-provider-prefix".to_string(), From e487f2c5de06e90db76c1d12449adda53abb982b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 26 Jun 2026 16:58:26 -0700 Subject: [PATCH 04/59] fix(supervisor-middleware): resolve rebase policy conflicts Signed-off-by: Piotr Mlocek --- crates/openshell-policy/src/lib.rs | 4 +- .../data/sandbox-policy.rego | 35 ++-- .../openshell-supervisor-network/src/opa.rs | 183 +++++++++++++++--- proto/sandbox.proto | 2 + 4 files changed, 177 insertions(+), 47 deletions(-) diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 0aa43c30d7..46054a91d1 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -19,8 +19,8 @@ use std::path::Path; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ FilesystemPolicy, GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, - LandlockPolicy, MiddlewareEndpointSelector, NetworkBinary, NetworkEndpoint, - NetworkMiddlewareConfig, NetworkPolicyRule, ProcessPolicy, SandboxPolicy, McpOptions, + LandlockPolicy, McpOptions, MiddlewareEndpointSelector, NetworkBinary, NetworkEndpoint, + NetworkMiddlewareConfig, NetworkPolicyRule, ProcessPolicy, SandboxPolicy, }; use serde::{Deserialize, Serialize}; diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index afa4f69478..9228416e1e 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -842,14 +842,29 @@ _policy_endpoint_configs(policy) := [ep | endpoint_has_extended_config(ep) ] -# Collect matching endpoint configs across all policies. Iterates over -# _matching_policy_names (a set, safe from regorus variable collisions) -# then collects per-policy configs via the helper function. +# Collect matching endpoint identities across all policies. Iterates over +# _matching_policy_names (a set, safe from regorus variable collisions) then +# returns the selected policy name plus endpoint index/path. Rust uses that +# identity to look up middleware attachment from policy data. +_matching_endpoint_contexts := [ctx | + some pname + _matching_policy_names[pname] + policy := data.network_policies[pname] + ep := policy.endpoints[i] + endpoint_matches_request(ep, input.network) + ctx := { + "policy": pname, + "endpoint_index": i, + "endpoint_path": object.get(ep, "path", ""), + } +] + _matching_endpoint_configs := [cfg | some pname _matching_policy_names[pname] cfgs := _policy_endpoint_configs(data.network_policies[pname]) cfg := cfgs[_] + endpoint_has_extended_config(cfg) ] matched_endpoint_config := _matching_endpoint_configs[0] if { @@ -858,20 +873,6 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { network_middlewares := object.get(data, "network_middlewares", []) -_matching_middleware_contexts := [ctx | - some pname - _matching_policy_names[pname] - policy := data.network_policies[pname] - some ep - ep := policy.endpoints[_] - endpoint_matches_request(ep, input.network) - ctx := { - "policy": pname, - "policy_middleware": object.get(policy, "middleware", []), - "endpoint": ep, - } -] - _policy_has_exact_declared_endpoint(policy) if { some ep ep := policy.endpoints[_] diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index a584b414bc..3d0f75bf78 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -750,9 +750,9 @@ fn network_input_json(input: &NetworkInput) -> serde_json::Value { } #[derive(Debug, Clone)] -struct MiddlewareContext { - policy_middleware: Vec, - endpoint_middleware: Vec, +struct MatchedEndpointContext { + policy_name: String, + endpoint_index: usize, endpoint_path: String, } @@ -773,19 +773,20 @@ fn query_middleware_chain_locked( return Ok(Vec::new()); } let contexts_val = engine - .eval_rule("data.openshell.sandbox._matching_middleware_contexts".into()) + .eval_rule("data.openshell.sandbox._matching_endpoint_contexts".into()) .map_err(|e| miette::miette!("{e}"))?; - let contexts = parse_middleware_contexts(&contexts_val); - let Some(context) = select_middleware_context(&contexts, request_path) else { + let contexts = parse_endpoint_contexts(&contexts_val); + let Some(context) = select_endpoint_context(&contexts, request_path)? else { return global_middleware_entries(&configs, &input.host, &HashSet::new()); }; + let policies_val = engine + .eval_rule("data.network_policies".into()) + .map_err(|e| miette::miette!("{e}"))?; + let (policy_middleware, endpoint_middleware) = + middleware_for_endpoint_identity(&policies_val, context)?; let mut explicit = Vec::new(); - for name in context - .policy_middleware - .iter() - .chain(context.endpoint_middleware.iter()) - { + for name in policy_middleware.iter().chain(endpoint_middleware.iter()) { if !explicit.contains(name) { explicit.push(name.clone()); } @@ -814,7 +815,7 @@ fn parse_middleware_configs(value: ®orus::Value) -> Result Vec { +fn parse_endpoint_contexts(value: ®orus::Value) -> Vec { let regorus::Value::Array(values) = value else { return Vec::new(); }; @@ -824,30 +825,87 @@ fn parse_middleware_contexts(value: ®orus::Value) -> Vec { let regorus::Value::Object(_) = value else { return None; }; - let endpoint = get_field(value, "endpoint")?; - Some(MiddlewareContext { - policy_middleware: get_str_array(value, "policy_middleware"), - endpoint_middleware: get_str_array(endpoint, "middleware"), - endpoint_path: get_str(endpoint, "path").unwrap_or_default(), + Some(MatchedEndpointContext { + policy_name: get_str(value, "policy").unwrap_or_default(), + endpoint_index: get_usize(value, "endpoint_index").unwrap_or_default(), + endpoint_path: get_str(value, "endpoint_path").unwrap_or_default(), }) }) .collect() } -fn select_middleware_context<'a>( - contexts: &'a [MiddlewareContext], +fn middleware_for_endpoint_identity( + policies: ®orus::Value, + context: &MatchedEndpointContext, +) -> Result<(Vec, Vec)> { + let policy = get_field(policies, &context.policy_name).ok_or_else(|| { + miette::miette!( + "matched endpoint policy '{}' was not found in OPA data", + context.policy_name + ) + })?; + let endpoint = get_array(policy, "endpoints") + .and_then(|endpoints| endpoints.get(context.endpoint_index)) + .ok_or_else(|| { + miette::miette!( + "matched endpoint {}[{}] was not found in OPA data", + context.policy_name, + context.endpoint_index + ) + })?; + Ok(( + get_str_array(policy, "middleware"), + get_str_array(endpoint, "middleware"), + )) +} + +fn select_endpoint_context<'a>( + contexts: &'a [MatchedEndpointContext], request_path: &str, -) -> Option<&'a MiddlewareContext> { - contexts +) -> Result> { + let matching: Vec<_> = contexts .iter() .filter(|context| crate::l7::endpoint_path_matches(&context.endpoint_path, request_path)) - .max_by_key(|context| { - if context.endpoint_path.is_empty() { - 0 - } else { - context.endpoint_path.chars().filter(|c| *c != '*').count() - } - }) + .map(|context| (endpoint_path_specificity(&context.endpoint_path), context)) + .collect(); + let Some(max_specificity) = matching.iter().map(|(specificity, _)| *specificity).max() else { + return Ok(None); + }; + let best: Vec<_> = matching + .into_iter() + .filter(|(specificity, _)| *specificity == max_specificity) + .map(|(_, context)| context) + .collect(); + if best.len() > 1 { + let matches = best + .iter() + .map(|context| { + format!( + "{}[{}] path={}", + context.policy_name, + context.endpoint_index, + if context.endpoint_path.is_empty() { + "" + } else { + context.endpoint_path.as_str() + } + ) + }) + .collect::>() + .join(", "); + return Err(miette::miette!( + "ambiguous middleware endpoint match for request path '{request_path}': {matches}" + )); + } + Ok(best.into_iter().next()) +} + +fn endpoint_path_specificity(path: &str) -> usize { + if path.is_empty() { + 0 + } else { + path.chars().filter(|c| *c != '*').count() + } } fn global_middleware_entries( @@ -918,6 +976,25 @@ fn get_field<'a>(val: &'a regorus::Value, key: &str) -> Option<&'a regorus::Valu } } +fn get_array<'a>(val: &'a regorus::Value, key: &str) -> Option<&'a [regorus::Value]> { + let regorus::Value::Array(values) = get_field(val, key)? else { + return None; + }; + Some(values) +} + +fn get_usize(val: ®orus::Value, key: &str) -> Option { + let value = get_field(val, key)?; + let regorus::Value::Number(number) = value else { + return None; + }; + let value = number.as_f64()?; + if !value.is_finite() || value.fract() != 0.0 || value < 0.0 { + return None; + } + format!("{value:.0}").parse::().ok() +} + fn regorus_value_to_struct(value: ®orus::Value) -> prost_types::Struct { let regorus::Value::Object(map) = value else { return prost_types::Struct::default(); @@ -3305,6 +3382,7 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], + middleware: vec![], }, ); @@ -3323,6 +3401,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3377,6 +3456,7 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], + middleware: vec![], }, ); @@ -3395,6 +3475,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, + network_middlewares: vec![], }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -6955,6 +7036,52 @@ network_policies: ); } + #[test] + fn middleware_chain_rejects_ambiguous_duplicate_endpoint_identity() { + let data = r#" +network_middlewares: + - name: first-redactor + middleware: openshell/secrets + - name: second-redactor + middleware: openshell/secrets +network_policies: + api: + name: api + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + middleware: ["first-redactor"] + access: full + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + middleware: ["second-redactor"] + access: full + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "api.example.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let err = engine + .query_middleware_chain_with_generation(&input, "/v1/messages") + .expect_err("equivalent endpoint identities should be ambiguous"); + assert!( + err.to_string() + .contains("ambiguous middleware endpoint match"), + "{err:?}" + ); + } + #[test] fn middleware_policy_validation_rejects_bad_configs() { let cases = [ diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 5d2bc31a53..a73d762e5e 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -202,6 +202,8 @@ message McpOptions { // MCP-family methods at the method layer unless a tool-name policy narrows // tools/call. When unset or false, explicit method rules are required. optional bool allow_all_known_mcp_methods = 2; +} + // Trusted GraphQL operation classification. message GraphqlOperation { // Operation type: "query", "mutation", or "subscription". From 9f892ef6cf32eec74c52019e7cf1b3fd03c37b4f Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 29 Jun 2026 10:15:03 -0700 Subject: [PATCH 05/59] feat(supervisor-middleware): implement phase one runtime Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 + crates/openshell-policy/Cargo.toml | 1 + crates/openshell-policy/src/lib.rs | 47 +++++ .../src/lib.rs | 88 +++++++-- .../src/service.rs | 9 +- .../data/sandbox-policy.rego | 2 + .../src/l7/relay.rs | 164 ++++++++++++++++- .../src/l7/rest.rs | 63 +++++++ .../openshell-supervisor-network/src/opa.rs | 174 ++++++++++++++---- .../openshell-supervisor-network/src/proxy.rs | 67 +++++++ 10 files changed, 551 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ed95de906..c083f7a8df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3771,6 +3771,7 @@ version = "0.0.0" dependencies = [ "miette", "openshell-core", + "openshell-supervisor-middleware", "prost-types", "serde", "serde_json", diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 50bea5b32e..7ccd5d9678 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } prost-types = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 46054a91d1..46646f7557 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1253,6 +1253,8 @@ pub enum PolicyViolation { }, /// `credential_signing` and `request_body_credential_rewrite` are both set. CredentialSigningWithBodyRewrite { policy_name: String, host: String }, + /// A built-in middleware configuration is invalid. + InvalidBuiltinMiddlewareConfig { name: String, reason: String }, } impl fmt::Display for PolicyViolation { @@ -1317,6 +1319,9 @@ impl fmt::Display for PolicyViolation { and request_body_credential_rewrite set; these options are mutually exclusive" ) } + Self::InvalidBuiltinMiddlewareConfig { name, reason } => { + write!(f, "middleware config '{name}' is invalid: {reason}") + } } } } @@ -1449,6 +1454,21 @@ pub fn validate_sandbox_policy( } } + for middleware in &policy.network_middlewares { + if middleware.middleware.starts_with("openshell/") { + let config = middleware.config.as_ref().cloned().unwrap_or_default(); + if let Err(error) = openshell_supervisor_middleware::validate_builtin_config( + &middleware.middleware, + &config, + ) { + violations.push(PolicyViolation::InvalidBuiltinMiddlewareConfig { + name: middleware.name.clone(), + reason: error.to_string(), + }); + } + } + } + if violations.is_empty() { Ok(()) } else { @@ -1884,6 +1904,33 @@ network_policies: assert_eq!(violations.len(), 2); } + #[test] + fn validate_rejects_invalid_builtin_middleware_config() { + let mut policy = restrictive_default_policy(); + policy.network_middlewares.push(NetworkMiddlewareConfig { + name: "redact-secrets".into(), + middleware: "openshell/secrets".into(), + config: Some(prost_types::Struct { + fields: std::iter::once(( + "secrets".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("allow".into())), + }, + )) + .collect(), + }), + on_error: String::new(), + endpoints: None, + }); + + let violations = validate_sandbox_policy(&policy).expect_err("invalid config"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::InvalidBuiltinMiddlewareConfig { name, .. } + if name == "redact-secrets" + ))); + } + #[test] fn validate_rejects_non_sandbox_user() { let mut policy = restrictive_default_policy(); diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index b68d83c86e..4ec7e27825 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -14,7 +14,7 @@ pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, NetworkMiddlewareConfig, Process, + Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, NetworkMiddlewareConfig, RequestContext, }; use tonic::Request; @@ -24,6 +24,19 @@ pub const HTTP_REQUEST_OPERATION: &str = "HttpRequest"; pub const PRE_CREDENTIALS_PHASE: &str = "pre_credentials"; pub const BUILTIN_SECRETS: &str = "openshell/secrets"; +/// Validate the configuration for an in-process middleware implementation. +/// +/// Policy admission uses this same implementation-specific validation before a +/// configuration can reach the request path. +pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { + match implementation { + BUILTIN_SECRETS => builtins::secrets::validate_config(config), + other => Err(miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OnError { FailClosed, @@ -76,9 +89,6 @@ impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { pub struct HttpRequestInput { pub request_id: String, pub sandbox_id: String, - pub binary: String, - pub pid: u32, - pub ancestors: Vec, pub scheme: String, pub host: String, pub port: u16, @@ -210,6 +220,26 @@ impl ChainRunner { } }; + let decision = match Decision::try_from(result.decision) { + Ok(decision @ (Decision::Allow | Decision::Deny)) => decision, + Ok(Decision::Unspecified) | Err(_) => { + match apply_on_error(entry, "invalid_response_decision", &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } + } + } + }; + // A result proposing unsafe header mutations is a malformed response: // route it through `on_error` instead of applying any of it. if validate_header_mutations(&headers, &result.add_headers).is_err() { @@ -251,11 +281,11 @@ impl ChainRunner { applied.push(MiddlewareInvocation { name: entry.name.clone(), implementation: entry.implementation.clone(), - decision: Decision::try_from(result.decision).unwrap_or(Decision::Unspecified), + decision, transformed, failed: false, }); - if result.decision == Decision::Deny as i32 { + if decision == Decision::Deny { return Ok(ChainOutcome { allowed: false, reason: safe_reason(&result.reason), @@ -293,11 +323,7 @@ fn build_evaluation( context: Some(RequestContext { request_id: input.request_id.clone(), sandbox_id: input.sandbox_id.clone(), - originating_process: Some(Process { - binary: input.binary.clone(), - pid: input.pid, - ancestors: input.ancestors.clone(), - }), + originating_process: None, }), config: Some(entry.config.clone()), target: Some(HttpRequestTarget { @@ -399,9 +425,6 @@ mod tests { HttpRequestInput { request_id: "req".into(), sandbox_id: "sbx".into(), - binary: "/usr/bin/curl".into(), - pid: 42, - ancestors: vec![], scheme: "https".into(), host: "api.example.com".into(), port: 443, @@ -413,6 +436,21 @@ mod tests { } } + #[test] + fn phase_one_evaluation_omits_originating_process() { + let entry = entry("redact", OnError::FailClosed); + let input = input("payload"); + let evaluation = build_evaluation(&entry, &input, &BTreeMap::new(), b"payload"); + + assert!( + evaluation + .context + .expect("request context") + .originating_process + .is_none() + ); + } + #[tokio::test] async fn redacts_common_secret_patterns() { let outcome = ChainRunner::default() @@ -684,4 +722,26 @@ mod tests { assert_eq!(outcome.applied.len(), 1); assert!(outcome.applied[0].failed); } + + #[tokio::test] + async fn unspecified_decision_uses_fail_closed() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + result: openshell_core::proto::HttpRequestResult { + decision: Decision::Unspecified as i32, + ..allow_result() + }, + })); + + let outcome = runner + .evaluate(&[entry("redact", OnError::FailClosed)], input("hello")) + .await + .expect("evaluate"); + + assert!(!outcome.allowed); + assert_eq!( + outcome.reason, + "middleware_failed: invalid_response_decision" + ); + assert!(outcome.applied[0].failed); + } } diff --git a/crates/openshell-supervisor-middleware/src/service.rs b/crates/openshell-supervisor-middleware/src/service.rs index 31cca5694b..cbd9231cdf 100644 --- a/crates/openshell-supervisor-middleware/src/service.rs +++ b/crates/openshell-supervisor-middleware/src/service.rs @@ -10,7 +10,7 @@ use tonic::{Request, Response, Status}; use crate::{ API_VERSION, BUILTIN_SECRETS, HTTP_REQUEST_OPERATION, PRE_CREDENTIALS_PHASE, builtins, - safe_reason, + safe_reason, validate_builtin_config, }; #[derive(Debug, Default)] @@ -40,12 +40,7 @@ impl SupervisorMiddleware for InProcessMiddlewareService { ) -> Result, Status> { let request = request.into_inner(); let config = request.config.unwrap_or_default(); - let validation = match request.binding_id.as_str() { - BUILTIN_SECRETS => builtins::secrets::validate_config(&config), - other => Err(miette::miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - }; + let validation = validate_builtin_config(&request.binding_id, &config); Ok(Response::new(match validation { Ok(()) => ValidateConfigResponse { valid: true, diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index 9228416e1e..52f6f10461 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -871,6 +871,8 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { count(_matching_endpoint_configs) > 0 } +network_policies := object.get(data, "network_policies", {}) + network_middlewares := object.get(data, "network_middlewares", []) _policy_has_exact_declared_endpoint(policy) if { diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 6e5c3c4e9e..c773fdcf48 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -768,12 +768,12 @@ fn jsonrpc_engine_type(protocol: L7Protocol) -> &'static str { } } -enum MiddlewareApplyResult { +pub(crate) enum MiddlewareApplyResult { Allowed(crate::l7::provider::L7Request), Denied(String), } -async fn apply_middleware_chain( +pub(crate) async fn apply_middleware_chain( req: crate::l7::provider::L7Request, client: &mut C, ctx: &L7EvalContext, @@ -796,18 +796,16 @@ async fn apply_middleware_chain( } }; let headers = safe_middleware_headers(&buffered.headers)?; + let query = raw_query_from_request_headers(&buffered.headers)?; let input = openshell_supervisor_middleware::HttpRequestInput { request_id: uuid::Uuid::new_v4().to_string(), - sandbox_id: String::new(), - binary: ctx.binary_path.clone(), - pid: 0, - ancestors: ctx.ancestors.clone(), + sandbox_id: openshell_ocsf::ctx::ctx().sandbox_id.clone(), scheme: "https".into(), host: ctx.host.clone(), port: ctx.port, method: req.action.clone(), path: req.target.clone(), - query: String::new(), + query, headers, body: buffered.body, }; @@ -828,6 +826,19 @@ async fn apply_middleware_chain( } } +fn raw_query_from_request_headers(headers: &[u8]) -> Result { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let target = header_str + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .ok_or_else(|| miette!("HTTP request line is missing a target"))?; + Ok(target + .split_once('?') + .map_or_else(String::new, |(_, query)| query.to_string())) +} + /// Apply the chain's `on_error` policy when the request body cannot be buffered /// for inspection because it exceeds the size cap. The RFC treats an unbufferable /// body as an `on_error` event: it is denied unless every attached middleware is @@ -1446,6 +1457,37 @@ where } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { + let chain = + engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let req = + match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; // Future MCP response/SSE introspection or rewrite would hook here // before returning upstream bytes. The current policy schema has no // trusted-annotations or version-profile field, so MCP responses and @@ -2736,6 +2778,104 @@ network_policies: .unwrap(); } + #[tokio::test] + async fn jsonrpc_middleware_fail_closed_does_not_reach_upstream() { + let data = r#" +network_middlewares: + - name: request-middleware + middleware: example/unavailable + on_error: fail_closed +network_policies: + jsonrpc_api: + name: jsonrpc_api + middleware: ["request-middleware"] + endpoints: + - host: api.example.test + port: 443 + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: reports.list + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "api.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let config = crate::l7::parse_l7_config(&endpoint_config.expect("json-rpc config")) + .expect("parse JSON-RPC config"); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_jsonrpc( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.list"}"#; + let request = format!( + "POST /rpc HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden")); + assert!(response.contains("middleware_failed")); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive request bytes" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[tokio::test] async fn l7_rest_middleware_over_capacity_fails_closed() { let (config, tunnel_engine, ctx) = @@ -2842,6 +2982,16 @@ network_policies: )); } + #[test] + fn middleware_keeps_the_raw_request_query() { + let query = raw_query_from_request_headers( + b"POST /v1/messages?token=a%2Bb&scope=private HTTP/1.1\r\nHost: api.example.test\r\n\r\n", + ) + .expect("query from request headers"); + + assert_eq!(query, "token=a%2Bb&scope=private"); + } + /// Tracing layer that captures emitted `OcsfEvent`s for assertions. struct OcsfCaptureLayer(Arc>>); diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 2c85cacf67..19f73e2ad3 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -246,6 +246,36 @@ async fn parse_http_request( })) } +/// Build an L7 request from a request already buffered by another proxy path. +/// +/// The forward proxy needs this after it has consumed the incoming HTTP/1 +/// headers itself. Keep the framing and query parsing here so it matches the +/// stream-based REST parser rather than growing another local parser. +pub(crate) fn request_from_buffered_http( + action: impl Into, + target: impl Into, + query_target: &str, + raw_header: Vec, +) -> Result { + let header_end = raw_header + .windows(4) + .position(|window| window == b"\r\n\r\n") + .ok_or_else(|| miette!("HTTP request headers are missing the CRLF terminator"))? + + 4; + let header_str = std::str::from_utf8(&raw_header[..header_end]) + .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let body_length = parse_body_length(header_str)?; + let (_, query_params) = parse_target_query(query_target)?; + + Ok(L7Request { + action: action.into(), + target: target.into(), + query_params, + raw_header, + body_length, + }) +} + /// Rebuild the request line in a raw HTTP header block with a canonicalized /// target. Called when the canonical path differs from what the client sent, /// so the upstream dispatches on the exact bytes the policy engine evaluated. @@ -3015,6 +3045,39 @@ mod tests { } } + #[test] + fn buffered_request_parser_uses_shared_framing_and_query_parsing() { + let request = request_from_buffered_http( + "POST", + "/v1/items", + "/v1/items?tag=first&tag=second", + b"POST /v1/items?tag=first&tag=second HTTP/1.1\r\nHost: api.example.com\r\nContent-Length: 3\r\n\r\nabc" + .to_vec(), + ) + .expect("parse buffered request"); + + assert_eq!(request.action, "POST"); + assert_eq!(request.target, "/v1/items"); + assert_eq!( + request.query_params.get("tag"), + Some(&vec!["first".to_string(), "second".to_string()]) + ); + assert!(matches!(request.body_length, BodyLength::ContentLength(3))); + } + + #[test] + fn buffered_request_parser_rejects_missing_header_terminator() { + let err = request_from_buffered_http( + "GET", + "/v1/items", + "/v1/items", + b"GET /v1/items HTTP/1.1\r\nHost: api.example.com\r\n".to_vec(), + ) + .expect_err("unterminated headers must be rejected"); + + assert!(err.to_string().contains("missing the CRLF terminator")); + } + #[test] fn parse_chunked() { let headers = diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 3d0f75bf78..3efec02127 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -776,12 +776,12 @@ fn query_middleware_chain_locked( .eval_rule("data.openshell.sandbox._matching_endpoint_contexts".into()) .map_err(|e| miette::miette!("{e}"))?; let contexts = parse_endpoint_contexts(&contexts_val); - let Some(context) = select_endpoint_context(&contexts, request_path)? else { - return global_middleware_entries(&configs, &input.host, &HashSet::new()); - }; let policies_val = engine - .eval_rule("data.network_policies".into()) + .eval_rule("data.openshell.sandbox.network_policies".into()) .map_err(|e| miette::miette!("{e}"))?; + let Some(context) = select_endpoint_context(&contexts, request_path, &policies_val)? else { + return global_middleware_entries(&configs, &input.host, &HashSet::new()); + }; let (policy_middleware, endpoint_middleware) = middleware_for_endpoint_identity(&policies_val, context)?; @@ -862,6 +862,7 @@ fn middleware_for_endpoint_identity( fn select_endpoint_context<'a>( contexts: &'a [MatchedEndpointContext], request_path: &str, + policies: ®orus::Value, ) -> Result> { let matching: Vec<_> = contexts .iter() @@ -876,30 +877,56 @@ fn select_endpoint_context<'a>( .filter(|(specificity, _)| *specificity == max_specificity) .map(|(_, context)| context) .collect(); - if best.len() > 1 { - let matches = best - .iter() - .map(|context| { - format!( - "{}[{}] path={}", - context.policy_name, - context.endpoint_index, - if context.endpoint_path.is_empty() { - "" - } else { - context.endpoint_path.as_str() - } - ) - }) - .collect::>() - .join(", "); - return Err(miette::miette!( - "ambiguous middleware endpoint match for request path '{request_path}': {matches}" - )); + if let Some((first, rest)) = best.split_first() { + let first_middleware = explicit_middleware_for_endpoint_identity(policies, first)?; + for context in rest { + if explicit_middleware_for_endpoint_identity(policies, context)? != first_middleware { + let matches = best + .iter() + .map(|context| { + format!( + "{}[{}] path={}", + context.policy_name, + context.endpoint_index, + if context.endpoint_path.is_empty() { + "" + } else { + context.endpoint_path.as_str() + } + ) + }) + .collect::>() + .join(", "); + return Err(miette::miette!( + "ambiguous middleware endpoint match for request path '{request_path}': {matches}" + )); + } + } } Ok(best.into_iter().next()) } +fn explicit_middleware_for_endpoint_identity( + policies: ®orus::Value, + context: &MatchedEndpointContext, +) -> Result> { + let (policy_middleware, endpoint_middleware) = + middleware_for_endpoint_identity(policies, context)?; + Ok(dedup_middleware_names( + policy_middleware.iter().chain(endpoint_middleware.iter()), + )) +} + +fn dedup_middleware_names<'a>(names: impl IntoIterator) -> Vec { + let mut deduped = Vec::new(); + for name in names { + if !deduped.contains(name) { + deduped.push(name.clone()); + } + } + deduped +} + fn endpoint_path_specificity(path: &str) -> usize { if path.is_empty() { 0 @@ -1402,10 +1429,91 @@ fn validate_middleware_policies(data: &serde_json::Value) -> Vec { )); } } + validate_ambiguous_middleware_endpoints( + policy_name, + policy, + &policy_middleware, + &mut errors, + ); } errors } +fn validate_ambiguous_middleware_endpoints( + policy_name: &str, + policy: &serde_json::Value, + policy_middleware: &[String], + errors: &mut Vec, +) { + let endpoints = policy + .get("endpoints") + .and_then(serde_json::Value::as_array) + .map_or(&[][..], Vec::as_slice); + let mut seen: Vec<(usize, MiddlewareEndpointKey, Vec)> = Vec::new(); + for (index, endpoint) in endpoints.iter().enumerate() { + let key = middleware_endpoint_key(endpoint); + let endpoint_middleware = json_string_array(endpoint.get("middleware")); + let chain = + dedup_middleware_names(policy_middleware.iter().chain(endpoint_middleware.iter())); + for (previous_index, previous_key, previous_chain) in &seen { + if previous_key == &key && previous_chain != &chain { + errors.push(format!( + "network policy '{policy_name}' endpoints[{previous_index}] and endpoints[{index}] have equivalent middleware selection keys ({key}) but different middleware chains" + )); + } + } + seen.push((index, key, chain)); + } +} + +#[derive(Debug, PartialEq, Eq)] +struct MiddlewareEndpointKey { + host: String, + ports: Vec, + path: String, +} + +impl std::fmt::Display for MiddlewareEndpointKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "host={} ports={:?} path={}", + if self.host.is_empty() { + "" + } else { + self.host.as_str() + }, + self.ports, + if self.path.is_empty() { + "" + } else { + self.path.as_str() + } + ) + } +} + +fn middleware_endpoint_key(endpoint: &serde_json::Value) -> MiddlewareEndpointKey { + let host = endpoint + .get("host") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_ascii_lowercase(); + let mut ports: Vec = endpoint + .get("ports") + .and_then(serde_json::Value::as_array) + .map(|ports| ports.iter().filter_map(serde_json::Value::as_u64).collect()) + .unwrap_or_default(); + ports.sort_unstable(); + ports.dedup(); + let path = endpoint + .get("path") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(); + MiddlewareEndpointKey { host, ports, path } +} + fn json_string_array(value: Option<&serde_json::Value>) -> Vec { value .and_then(serde_json::Value::as_array) @@ -7037,7 +7145,7 @@ network_policies: } #[test] - fn middleware_chain_rejects_ambiguous_duplicate_endpoint_identity() { + fn middleware_validation_rejects_ambiguous_duplicate_endpoint_middleware() { let data = r#" network_middlewares: - name: first-redactor @@ -7063,21 +7171,13 @@ network_policies: binaries: - { path: /usr/bin/curl } "#; - let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); - let input = NetworkInput { - host: "api.example.com".into(), - port: 443, - binary_path: PathBuf::from("/usr/bin/curl"), - binary_sha256: "unused".into(), - ancestors: vec![], - cmdline_paths: vec![], + let err = match OpaEngine::from_strings(TEST_POLICY, data) { + Ok(_) => panic!("equivalent endpoints with different middleware should be invalid"), + Err(err) => err, }; - let err = engine - .query_middleware_chain_with_generation(&input, "/v1/messages") - .expect_err("equivalent endpoint identities should be ambiguous"); assert!( err.to_string() - .contains("ambiguous middleware endpoint match"), + .contains("equivalent middleware selection keys"), "{err:?}" ); } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index af33317353..f8310fbdc3 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4184,6 +4184,73 @@ async fn handle_forward_proxy( } emit_forward_success_activity(activity_tx, l7_activity_pending); + let middleware_path = path.split_once('?').map_or(path.as_str(), |(path, _)| path); + let middleware_input = crate::opa::NetworkInput { + host: host_lc.clone(), + port, + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), + }; + let (chain, generation) = + opa_engine.query_middleware_chain_with_generation(&middleware_input, middleware_path)?; + if generation != forward_generation_guard.captured_generation() { + emit_l7_tunnel_close_after_policy_change( + &host_lc, + port, + miette::miette!( + "policy changed before forward middleware evaluation [expected_generation:{} current_generation:{}]", + forward_generation_guard.captured_generation(), + generation, + ), + ); + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "policy_denied", + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), + ), + ) + .await?; + return Ok(()); + } + if !chain.is_empty() { + let request = crate::l7::rest::request_from_buffered_http( + method, + middleware_path, + &upstream_target, + forward_request_bytes, + )?; + forward_request_bytes = match crate::l7::relay::apply_middleware_chain( + request, + client, + &l7_ctx, + chain, + &forward_generation_guard, + ) + .await? + { + crate::l7::relay::MiddlewareApplyResult::Allowed(request) => request.raw_header, + crate::l7::relay::MiddlewareApplyResult::Denied(reason) => { + emit_activity_simple(activity_tx, true, "middleware"); + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "middleware_denied", + &format!("{method} {host_lc}:{port}{path} denied by middleware: {reason}"), + ), + ) + .await?; + return Ok(()); + } + }; + } + forward_request_bytes = match inject_token_grant_for_forward_request( method, &upstream_target, From da486b2f3e11621190dbf718fe66ba383b853279 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 29 Jun 2026 14:10:40 -0700 Subject: [PATCH 06/59] fix(supervisor-middleware): harden selection and buffering Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 + crates/openshell-cli/src/policy_update.rs | 1 - crates/openshell-policy/Cargo.toml | 1 + crates/openshell-policy/src/compose.rs | 1 - crates/openshell-policy/src/lib.rs | 373 ++++++++++-- crates/openshell-policy/src/merge.rs | 24 - crates/openshell-providers/src/profiles.rs | 2 - .../src/mechanistic_mapper.rs | 1 - crates/openshell-server/src/grpc/policy.rs | 32 -- .../openshell-server/src/grpc/validation.rs | 22 + .../Cargo.toml | 4 +- .../src/builtins/mod.rs | 25 + .../src/builtins/secrets.rs | 22 +- .../src/lib.rs | 383 +++++++++++-- .../src/service.rs | 24 +- .../data/sandbox-policy.rego | 19 - .../src/l7/relay.rs | 151 +++-- .../src/l7/rest.rs | 138 +++-- .../openshell-supervisor-network/src/opa.rs | 534 ++++-------------- .../src/policy_local.rs | 5 - .../openshell-supervisor-network/src/proxy.rs | 5 +- proto/middleware.proto | 2 + proto/sandbox.proto | 8 +- 23 files changed, 1073 insertions(+), 705 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c083f7a8df..0634d12e9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3769,6 +3769,7 @@ dependencies = [ name = "openshell-policy" version = "0.0.0" dependencies = [ + "glob", "miette", "openshell-core", "openshell-supervisor-middleware", diff --git a/crates/openshell-cli/src/policy_update.rs b/crates/openshell-cli/src/policy_update.rs index 824b1dde03..1f1f647506 100644 --- a/crates/openshell-cli/src/policy_update.rs +++ b/crates/openshell-cli/src/policy_update.rs @@ -65,7 +65,6 @@ pub fn build_policy_update_plan( ..Default::default() }) .collect(), - middleware: Vec::new(), }; merge_operations.push(PolicyMergeOperation { operation: Some(policy_merge_operation::Operation::AddRule(AddNetworkRule { diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 7ccd5d9678..073728db16 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -11,6 +11,7 @@ license.workspace = true repository.workspace = true [dependencies] +glob = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } prost-types = { workspace = true } diff --git a/crates/openshell-policy/src/compose.rs b/crates/openshell-policy/src/compose.rs index 1ad0d46171..7ca8584d9d 100644 --- a/crates/openshell-policy/src/compose.rs +++ b/crates/openshell-policy/src/compose.rs @@ -115,7 +115,6 @@ mod tests { ..Default::default() }], binaries: Vec::new(), - middleware: Vec::new(), } } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 46646f7557..29585a6233 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -12,7 +12,7 @@ mod compose; mod merge; -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt; use std::path::Path; @@ -89,8 +89,6 @@ struct NetworkPolicyRuleDef { endpoints: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] binaries: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - middleware: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -174,8 +172,6 @@ struct NetworkEndpointDef { json_rpc: Option, #[serde(default, skip_serializing_if = "Option::is_none")] mcp: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - middleware: Vec, } // Signature dictated by serde's `skip_serializing_if`, which requires `&T`. @@ -788,7 +784,6 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { signing_region: e.signing_region, json_rpc_max_body_bytes: json_rpc_max_body_bytes(&e.json_rpc, &e.mcp), mcp: mcp_options(&e.mcp), - middleware: e.middleware, } }) .collect(), @@ -800,7 +795,6 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { ..Default::default() }) .collect(), - middleware: rule.middleware, }; (key, proto_rule) }) @@ -938,7 +932,6 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { signing_region: e.signing_region.clone(), json_rpc, mcp, - middleware: e.middleware.clone(), } }) .collect(), @@ -950,7 +943,6 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { harness: false, }) .collect(), - middleware: rule.middleware.clone(), }; (key.clone(), yaml_rule) }) @@ -1255,6 +1247,16 @@ pub enum PolicyViolation { CredentialSigningWithBodyRewrite { policy_name: String, host: String }, /// A built-in middleware configuration is invalid. InvalidBuiltinMiddlewareConfig { name: String, reason: String }, + /// A middleware configuration is structurally invalid. + InvalidMiddlewareConfig { name: String, reason: String }, + /// Middleware configuration names must be unique. + DuplicateMiddlewareConfigName { name: String }, + /// A middleware selector conflicts with an endpoint that skips TLS inspection. + MiddlewareTlsSkipConflict { + middleware_name: String, + policy_name: String, + host: String, + }, } impl fmt::Display for PolicyViolation { @@ -1319,13 +1321,67 @@ impl fmt::Display for PolicyViolation { and request_body_credential_rewrite set; these options are mutually exclusive" ) } - Self::InvalidBuiltinMiddlewareConfig { name, reason } => { + Self::InvalidBuiltinMiddlewareConfig { name, reason } + | Self::InvalidMiddlewareConfig { name, reason } => { write!(f, "middleware config '{name}' is invalid: {reason}") } + Self::DuplicateMiddlewareConfigName { name } => { + write!(f, "duplicate middleware config '{name}'") + } + Self::MiddlewareTlsSkipConflict { + middleware_name, + policy_name, + host, + } => { + write!( + f, + "middleware config '{middleware_name}' selects network policy \ + '{policy_name}' tls: skip endpoint '{host}'" + ) + } } } } +/// Match a middleware host selector pattern using the runtime's glob semantics. +/// +/// Invalid or empty patterns return an error instead of silently becoming a +/// non-match. +pub fn middleware_host_matches(pattern: &str, host: &str) -> std::result::Result { + if pattern.is_empty() { + return Err("host pattern must not be empty".to_string()); + } + if pattern.chars().any(char::is_whitespace) { + return Err("host pattern must not contain whitespace".to_string()); + } + + let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) + .map_err(|error| format!("invalid host pattern: {error}"))?; + Ok(pattern.matches(&host.to_ascii_lowercase())) +} + +fn middleware_selector_matches_host( + middleware: &NetworkMiddlewareConfig, + host: &str, +) -> std::result::Result { + let Some(selector) = &middleware.endpoints else { + return Ok(false); + }; + let matches_include = selector + .include + .iter() + .try_fold(false, |matched, pattern| { + middleware_host_matches(pattern, host).map(|matches| matched || matches) + })?; + let matches_exclude = selector + .exclude + .iter() + .try_fold(false, |matched, pattern| { + middleware_host_matches(pattern, host).map(|matches| matched || matches) + })?; + Ok(matches_include && !matches_exclude) +} + /// Validate that a sandbox policy does not contain unsafe content. /// /// Returns `Ok(())` if the policy is safe, or `Err(violations)` listing all @@ -1340,6 +1396,9 @@ impl fmt::Display for PolicyViolation { /// - Individual path lengths must not exceed [`MAX_PATH_LENGTH`] /// - Total path count must not exceed [`MAX_FILESYSTEM_PATHS`] /// - Network endpoint hosts must not use TLD wildcards (e.g. `*.com`) +/// - Middleware names, implementations, failure modes, selectors, and built-in +/// configurations must be valid +/// - Middleware selectors must not match endpoints that skip TLS inspection pub fn validate_sandbox_policy( policy: &SandboxPolicy, ) -> std::result::Result<(), Vec> { @@ -1454,9 +1513,67 @@ pub fn validate_sandbox_policy( } } + let mut middleware_names = HashSet::new(); for middleware in &policy.network_middlewares { - if middleware.middleware.starts_with("openshell/") { - let config = middleware.config.as_ref().cloned().unwrap_or_default(); + if middleware.name.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "name must not be empty".to_string(), + }); + } else if !middleware_names.insert(middleware.name.clone()) { + violations.push(PolicyViolation::DuplicateMiddlewareConfigName { + name: middleware.name.clone(), + }); + } + + if middleware.middleware.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "implementation must not be empty".to_string(), + }); + } else if middleware.middleware.starts_with("openshell/") + && middleware.middleware != openshell_supervisor_middleware::BUILTIN_SECRETS + { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("unsupported built-in '{}'", middleware.middleware), + }); + } + + if !matches!( + middleware.on_error.as_str(), + "" | "fail_closed" | "fail_open" + ) { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("invalid on_error '{}'", middleware.on_error), + }); + } + + let Some(selector) = &middleware.endpoints else { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "endpoint selector is required".to_string(), + }); + continue; + }; + if selector.include.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "endpoint selector must include at least one host pattern".to_string(), + }); + } + for pattern in selector.include.iter().chain(&selector.exclude) { + if let Err(reason) = middleware_host_matches(pattern, "validation.invalid") { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("endpoint selector pattern '{pattern}' is invalid: {reason}"), + }); + } + } + + if middleware.middleware == openshell_supervisor_middleware::BUILTIN_SECRETS { + let config = middleware.config.clone().unwrap_or_default(); if let Err(error) = openshell_supervisor_middleware::validate_builtin_config( &middleware.middleware, &config, @@ -1467,6 +1584,25 @@ pub fn validate_sandbox_policy( }); } } + + for (key, rule) in &policy.network_policies { + let policy_name = if rule.name.is_empty() { + key + } else { + &rule.name + }; + for endpoint in &rule.endpoints { + if endpoint.tls == "skip" + && middleware_selector_matches_host(middleware, &endpoint.host).unwrap_or(false) + { + violations.push(PolicyViolation::MiddlewareTlsSkipConflict { + middleware_name: middleware.name.clone(), + policy_name: policy_name.clone(), + host: endpoint.host.clone(), + }); + } + } + } } if violations.is_empty() { @@ -1608,17 +1744,17 @@ network_middlewares: service: mode: redact max_matches: 2 - - name: endpoint-redactor + - name: secondary-redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] network_policies: api: name: api - middleware: ["global-redactor"] endpoints: - host: api.example.com port: 443 protocol: rest - middleware: ["endpoint-redactor"] binaries: - path: /usr/bin/curl "#; @@ -1651,26 +1787,9 @@ network_policies: .fields .contains_key("service") ); - assert_eq!( - proto.network_policies["api"].middleware, - vec!["global-redactor"] - ); - assert_eq!( - proto.network_policies["api"].endpoints[0].middleware, - vec!["endpoint-redactor"] - ); - let yaml_out = serialize_sandbox_policy(&proto).expect("serialize failed"); let reparsed = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); assert_eq!(reparsed.network_middlewares, proto.network_middlewares); - assert_eq!( - reparsed.network_policies["api"].middleware, - vec!["global-redactor"] - ); - assert_eq!( - reparsed.network_policies["api"].endpoints[0].middleware, - vec!["endpoint-redactor"] - ); } #[test] @@ -1803,6 +1922,31 @@ network_policies: assert!(parse_sandbox_policy(yaml).is_err()); } + #[test] + fn parse_rejects_middleware_attachments_on_network_policies_and_endpoints() { + let policy_attachment = r" +version: 1 +network_policies: + api: + middleware: [redact] + endpoints: + - host: api.example.com + port: 443 +"; + assert!(parse_sandbox_policy(policy_attachment).is_err()); + + let endpoint_attachment = r" +version: 1 +network_policies: + api: + endpoints: + - host: api.example.com + port: 443 + middleware: [redact] +"; + assert!(parse_sandbox_policy(endpoint_attachment).is_err()); + } + #[test] fn l7_config_stanza_runtime_fields_use_canonical_schema() { let fields = l7_config_alias_runtime_fields( @@ -1876,6 +2020,19 @@ network_policies: // ---- Policy validation tests ---- + fn middleware_config(name: &str, implementation: &str) -> NetworkMiddlewareConfig { + NetworkMiddlewareConfig { + name: name.into(), + middleware: implementation.into(), + config: None, + on_error: String::new(), + endpoints: Some(MiddlewareEndpointSelector { + include: vec!["api.example.com".into()], + exclude: Vec::new(), + }), + } + } + #[test] fn validate_rejects_root_run_as_user() { let mut policy = restrictive_default_policy(); @@ -1907,21 +2064,17 @@ network_policies: #[test] fn validate_rejects_invalid_builtin_middleware_config() { let mut policy = restrictive_default_policy(); - policy.network_middlewares.push(NetworkMiddlewareConfig { - name: "redact-secrets".into(), - middleware: "openshell/secrets".into(), - config: Some(prost_types::Struct { - fields: std::iter::once(( - "secrets".into(), - prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("allow".into())), - }, - )) - .collect(), - }), - on_error: String::new(), - endpoints: None, + let mut middleware = middleware_config("redact-secrets", "openshell/secrets"); + middleware.config = Some(prost_types::Struct { + fields: std::iter::once(( + "secrets".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("allow".into())), + }, + )) + .collect(), }); + policy.network_middlewares.push(middleware); let violations = validate_sandbox_policy(&policy).expect_err("invalid config"); assert!(violations.iter().any(|violation| matches!( @@ -1931,6 +2084,134 @@ network_policies: ))); } + #[test] + fn validate_rejects_invalid_middleware_control_fields() { + let cases = [ + ( + middleware_config("", "openshell/secrets"), + "name must not be empty", + ), + ( + middleware_config("redactor", ""), + "implementation must not be empty", + ), + ( + middleware_config("redactor", "openshell/unknown"), + "unsupported built-in", + ), + ( + { + let mut middleware = middleware_config("redactor", "openshell/secrets"); + middleware.on_error = "maybe".into(); + middleware + }, + "invalid on_error", + ), + ( + { + let mut middleware = middleware_config("redactor", "openshell/secrets"); + middleware.endpoints = None; + middleware + }, + "endpoint selector is required", + ), + ( + { + let mut middleware = middleware_config("redactor", "openshell/secrets"); + middleware.endpoints.as_mut().unwrap().include.clear(); + middleware + }, + "must include at least one host pattern", + ), + ]; + + for (middleware, expected) in cases { + let mut policy = restrictive_default_policy(); + policy.network_middlewares.push(middleware); + let errors = validate_sandbox_policy(&policy) + .expect_err("invalid middleware must be rejected") + .into_iter() + .map(|violation| violation.to_string()) + .collect::>() + .join("; "); + assert!( + errors.contains(expected), + "expected {expected:?} in {errors:?}" + ); + } + } + + #[test] + fn validate_rejects_duplicate_middleware_config_names() { + let mut policy = restrictive_default_policy(); + policy + .network_middlewares + .push(middleware_config("redactor", "openshell/secrets")); + policy + .network_middlewares + .push(middleware_config("redactor", "openshell/secrets")); + + let violations = validate_sandbox_policy(&policy).expect_err("duplicate name"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::DuplicateMiddlewareConfigName { name } if name == "redactor" + ))); + } + + #[test] + fn validate_rejects_malformed_middleware_selector_patterns() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/secrets"); + middleware.endpoints.as_mut().unwrap().include = vec!["api[.example.com".into()]; + policy.network_middlewares.push(middleware); + + let errors = validate_sandbox_policy(&policy) + .expect_err("malformed selector") + .into_iter() + .map(|violation| violation.to_string()) + .collect::>() + .join("; "); + assert!(errors.contains("invalid host pattern"), "{errors}"); + } + + #[test] + fn middleware_host_selector_matching_is_case_insensitive() { + assert!(middleware_host_matches("*.Example.COM", "API.example.com").unwrap()); + assert!(!middleware_host_matches("*.example.com", "example.com").unwrap()); + assert!(middleware_host_matches("*", "deep.api.example.com").unwrap()); + } + + #[test] + fn validate_rejects_middleware_selector_matching_tls_skip_endpoint() { + let mut policy = restrictive_default_policy(); + policy + .network_middlewares + .push(middleware_config("redactor", "openshell/secrets")); + policy.network_policies.insert( + "api".into(), + NetworkPolicyRule { + name: "api".into(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".into(), + port: 443, + tls: "skip".into(), + ..Default::default() + }], + binaries: Vec::new(), + }, + ); + + let violations = validate_sandbox_policy(&policy).expect_err("tls skip conflict"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::MiddlewareTlsSkipConflict { + middleware_name, + policy_name, + host, + } if middleware_name == "redactor" && policy_name == "api" && host == "api.example.com" + ))); + } + #[test] fn validate_rejects_non_sandbox_user() { let mut policy = restrictive_default_policy(); diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs index 1c63e6ebc4..04f390198d 100644 --- a/crates/openshell-policy/src/merge.rs +++ b/crates/openshell-policy/src/merge.rs @@ -989,7 +989,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1008,7 +1007,6 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], - ..Default::default() }; let result = merge_policy( @@ -1037,7 +1035,6 @@ mod tests { name: "existing".to_string(), endpoints: vec![endpoint("api.github.com", 443)], binaries: vec![advisor_binary("/usr/bin/curl")], - ..Default::default() }, ); @@ -1048,7 +1045,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let result = merge_policy( @@ -1080,7 +1076,6 @@ mod tests { ..Default::default() }, ], - ..Default::default() }; let result = merge_policy( @@ -1112,7 +1107,6 @@ mod tests { path: "/usr/bin/python".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1126,7 +1120,6 @@ mod tests { ..Default::default() }], binaries: vec![advisor_binary("/usr/bin/python")], - ..Default::default() }; let result = merge_policy( @@ -1454,7 +1447,6 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1479,7 +1471,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let merged = merge_policy( @@ -1503,7 +1494,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; // Merge an *unrelated* rule for a different host. The proposed rule @@ -1534,7 +1524,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1547,7 +1536,6 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1579,7 +1567,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; // Endpoint exists in the policy but with a *different* binary. The @@ -1595,7 +1582,6 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1632,7 +1618,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1652,7 +1637,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1680,7 +1664,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1703,7 +1686,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1727,7 +1709,6 @@ mod tests { path: "/usr/bin/git".to_string(), ..Default::default() }], - ..Default::default() }; let merged = merge_policy( @@ -1752,7 +1733,6 @@ mod tests { name: "any_binary_rule".to_string(), endpoints: vec![endpoint("api.github.com", 443)], binaries: vec![], - ..Default::default() }; let mut policy = restrictive_default_policy(); @@ -1765,7 +1745,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -1823,7 +1802,6 @@ mod tests { path: "/usr/bin/gh".to_string(), ..Default::default() }], - ..Default::default() }; let composed = compose_effective_policy( &SandboxPolicy::default(), @@ -1855,7 +1833,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let result = merge_policy( composed, @@ -1924,7 +1901,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let result = merge_policy( policy, diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index 1eb1b54d23..ddfbcaf7dc 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -450,7 +450,6 @@ impl ProviderTypeProfile { NetworkPolicyRule { name: rule_name.to_string(), endpoints: self.endpoints.iter().map(endpoint_to_proto).collect(), - middleware: Vec::new(), binaries: self.binaries.iter().map(binary_to_proto).collect(), } } @@ -788,7 +787,6 @@ fn endpoint_to_proto(endpoint: &EndpointProfile) -> NetworkEndpoint { request_body_credential_rewrite: endpoint.request_body_credential_rewrite, advisor_proposed: false, persisted_queries: endpoint.persisted_queries.clone(), - middleware: Vec::new(), graphql_persisted_queries: endpoint .graphql_persisted_queries .iter() diff --git a/crates/openshell-sandbox/src/mechanistic_mapper.rs b/crates/openshell-sandbox/src/mechanistic_mapper.rs index bb83ddb665..8ee2fc37f9 100644 --- a/crates/openshell-sandbox/src/mechanistic_mapper.rs +++ b/crates/openshell-sandbox/src/mechanistic_mapper.rs @@ -162,7 +162,6 @@ pub fn generate_proposals(summaries: &[DenialSummary]) -> Vec { name: rule_name.clone(), endpoints: vec![endpoint], binaries, - middleware: Vec::new(), }; // Compute confidence. diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index ad4fdf5ba2..d3bc213ba0 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5746,7 +5746,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -5960,7 +5959,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -6077,7 +6075,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6183,7 +6180,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let mechanistic_submit = handle_submit_policy_analysis( &state, @@ -6261,7 +6257,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let agent_submit = handle_submit_policy_analysis( &state, @@ -6389,7 +6384,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6490,7 +6484,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6591,7 +6584,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6685,7 +6677,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6770,7 +6761,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6859,7 +6849,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -6951,7 +6940,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -7038,7 +7026,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let response = handle_submit_policy_analysis( @@ -7100,7 +7087,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-provider-prefix".to_string(), @@ -7215,7 +7201,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -7312,7 +7297,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -7398,7 +7382,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -7540,7 +7523,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -7666,7 +7648,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let step1 = handle_submit_policy_analysis( &state, @@ -7708,7 +7689,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let step2 = handle_submit_policy_analysis( &state, @@ -7840,7 +7820,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let submit_one = |rule_name: &str, rule: NetworkPolicyRule| { @@ -7949,7 +7928,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let submit_one = || { let state = state.clone(); @@ -8050,7 +8028,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let submit = handle_submit_policy_analysis( @@ -8182,7 +8159,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; handle_submit_policy_analysis( @@ -8381,7 +8357,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, }; @@ -8410,7 +8385,6 @@ mod tests { path: "/usr/bin/node".to_string(), ..Default::default() }], - ..Default::default() }, }; @@ -8439,7 +8413,6 @@ mod tests { path: "/usr/bin/node".to_string(), ..Default::default() }], - ..Default::default() }, }; @@ -8467,7 +8440,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-1".to_string(), @@ -8536,7 +8508,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, )) .collect(), @@ -8565,7 +8536,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-merge".to_string(), @@ -8639,7 +8609,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, )) .collect(), @@ -8668,7 +8637,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }; let chunk = DraftChunkRecord { id: "chunk-new".to_string(), diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 0b3548b062..c98ad2d625 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -1613,6 +1613,28 @@ mod tests { assert!(err.message().contains("TLD wildcard")); } + #[test] + fn validate_policy_safety_rejects_invalid_middleware_before_acceptance() { + use openshell_core::proto::{MiddlewareEndpointSelector, NetworkMiddlewareConfig}; + + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_middlewares.push(NetworkMiddlewareConfig { + name: "redactor".into(), + middleware: "openshell/secrets".into(), + on_error: "maybe".into(), + endpoints: Some(MiddlewareEndpointSelector { + include: vec!["api[.example.com".into()], + exclude: Vec::new(), + }), + ..Default::default() + }); + + let err = validate_policy_safety(&policy).unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("invalid on_error")); + assert!(err.message().contains("invalid host pattern")); + } + #[test] fn validate_no_reserved_provider_policy_keys_rejects_reserved_key() { use openshell_core::proto::NetworkPolicyRule; diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index fdaeb2e824..4ae355894d 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -16,10 +16,8 @@ openshell-core = { path = "../openshell-core" } miette = { workspace = true } prost-types = { workspace = true } regex = { workspace = true } -tonic = { workspace = true } - -[dev-dependencies] tokio = { workspace = true } +tonic = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-supervisor-middleware/src/builtins/mod.rs b/crates/openshell-supervisor-middleware/src/builtins/mod.rs index d91ee745e9..1db620220a 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/mod.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/mod.rs @@ -2,3 +2,28 @@ // SPDX-License-Identifier: Apache-2.0 pub mod secrets; + +use miette::{Result, miette}; +use openshell_core::proto::{HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding}; + +pub fn describe() -> Vec { + vec![secrets::describe()] +} + +pub fn validate_config(binding_id: &str, config: &prost_types::Struct) -> Result<()> { + match binding_id { + secrets::BINDING_ID => secrets::validate_config(config), + other => Err(miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + } +} + +pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { + match evaluation.binding_id.as_str() { + secrets::BINDING_ID => secrets::evaluate_http_request(evaluation), + other => Err(miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + } +} diff --git a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs index 5721025593..d88ac080d8 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs @@ -5,10 +5,24 @@ use std::collections::HashMap; use std::sync::LazyLock; use miette::{Result, miette}; -use openshell_core::proto::{Decision, Finding, HttpRequestEvaluation, HttpRequestResult}; +use openshell_core::proto::{ + Decision, Finding, HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding, +}; use regex::Regex; -use crate::BUILTIN_SECRETS; +pub const BINDING_ID: &str = "openshell/secrets"; +const OPERATION: &str = "HttpRequest"; +const PHASE: &str = "pre_credentials"; +const MAX_BODY_BYTES: u64 = 256 * 1024; + +pub fn describe() -> MiddlewareBinding { + MiddlewareBinding { + id: BINDING_ID.into(), + operation: OPERATION.into(), + phase: PHASE.into(), + max_body_bytes: MAX_BODY_BYTES, + } +} /// A named secret-detection pattern. The `kind` is an audit-safe label that /// flows into findings so operators can see *what* matched without seeing the @@ -51,7 +65,7 @@ pub fn validate_config(config: &prost_types::Struct) -> Result<()> { if mode != "redact" { return Err(miette!( "{} only supports config.secrets: redact in phase 1", - BUILTIN_SECRETS + BINDING_ID )); } Ok(()) @@ -61,7 +75,7 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result Result<()> { - match implementation { - BUILTIN_SECRETS => builtins::secrets::validate_config(config), - other => Err(miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - } + builtins::validate_config(implementation, config) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -85,6 +79,26 @@ impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { } } +/// A policy-selected middleware config joined with metadata reported by its +/// service's `Describe` call. A missing binding is retained so `on_error` can +/// decide whether the request fails open or closed. +#[derive(Debug, Clone)] +pub struct DescribedChainEntry { + entry: ChainEntry, + binding: Option, + max_body_bytes: usize, +} + +impl DescribedChainEntry { + pub fn max_body_bytes(&self) -> usize { + self.max_body_bytes + } + + pub fn on_error(&self) -> OnError { + self.entry.on_error + } +} + #[derive(Debug, Clone)] pub struct HttpRequestInput { pub request_id: String, @@ -138,15 +152,15 @@ enum OnErrorAction { /// Apply a middleware entry's `on_error` policy after a failure (service error or /// malformed response). Records a `failed` invocation for telemetry in both cases. fn apply_on_error( - entry: &ChainEntry, + entry: &DescribedChainEntry, reason: &str, applied: &mut Vec, ) -> OnErrorAction { - match entry.on_error { + match entry.entry.on_error { OnError::FailOpen => { applied.push(MiddlewareInvocation { - name: entry.name.clone(), - implementation: entry.implementation.clone(), + name: entry.entry.name.clone(), + implementation: entry.entry.implementation.clone(), decision: Decision::Allow, transformed: false, failed: true, @@ -155,8 +169,8 @@ fn apply_on_error( } OnError::FailClosed => { applied.push(MiddlewareInvocation { - name: entry.name.clone(), - implementation: entry.implementation.clone(), + name: entry.entry.name.clone(), + implementation: entry.entry.implementation.clone(), decision: Decision::Deny, transformed: false, failed: true, @@ -168,24 +182,102 @@ fn apply_on_error( #[derive(Clone)] pub struct ChainRunner { + state: Arc, +} + +struct MiddlewareServiceState { service: Arc, + manifest: OnceCell, } +static IN_PROCESS_SERVICE: LazyLock> = LazyLock::new(|| { + Arc::new(MiddlewareServiceState { + service: Arc::new(InProcessMiddlewareService), + manifest: OnceCell::new(), + }) +}); + impl Default for ChainRunner { fn default() -> Self { - Self::new(Arc::new(InProcessMiddlewareService)) + Self { + state: Arc::clone(&IN_PROCESS_SERVICE), + } } } impl ChainRunner { pub fn new(service: Arc) -> Self { - Self { service } + Self { + state: Arc::new(MiddlewareServiceState { + service, + manifest: OnceCell::new(), + }), + } + } + + async fn manifest(&self) -> Result<&MiddlewareManifest> { + self.state + .manifest + .get_or_try_init(|| async { + self.state + .service + .describe(Request::new(())) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware Describe failed: {}", + safe_reason(&error.to_string()) + ) + }) + }) + .await + } + + pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result> { + let manifest = self.manifest().await?; + entries + .iter() + .map(|entry| { + let binding = manifest + .bindings + .iter() + .find(|binding| binding.id == entry.implementation) + .cloned(); + let max_body_bytes = binding + .as_ref() + .map(|binding| { + usize::try_from(binding.max_body_bytes).map_err(|_| { + miette!( + "middleware binding '{}' reports a body limit too large for this platform", + binding.id + ) + }) + }) + .transpose()? + .unwrap_or(0); + Ok(DescribedChainEntry { + entry: entry.clone(), + binding, + max_body_bytes, + }) + }) + .collect() } pub async fn evaluate( &self, entries: &[ChainEntry], input: HttpRequestInput, + ) -> Result { + let entries = self.describe_chain(entries).await?; + self.evaluate_described(&entries, input).await + } + + pub async fn evaluate_described( + &self, + entries: &[DescribedChainEntry], + input: HttpRequestInput, ) -> Result { let mut headers = input.headers.clone(); let mut body = input.body.clone(); @@ -195,8 +287,41 @@ impl ChainRunner { let mut applied = Vec::new(); for entry in entries { - let evaluation = build_evaluation(entry, &input, &headers, &body); + let Some(binding) = entry.binding.as_ref() else { + match apply_on_error(entry, "binding_not_described", &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } + } + }; + if body.len() > entry.max_body_bytes { + match apply_on_error(entry, "request_body_over_capacity", &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } + } + } + let evaluation = build_evaluation(entry, binding, &input, &headers, &body); let result = match self + .state .service .evaluate_http_request(Request::new(evaluation)) .await @@ -240,6 +365,23 @@ impl ChainRunner { } }; + if result.has_body && result.body.len() > entry.max_body_bytes { + match apply_on_error(entry, "response_body_over_capacity", &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } + } + } + // A result proposing unsafe header mutations is a malformed response: // route it through `on_error` instead of applying any of it. if validate_header_mutations(&headers, &result.add_headers).is_err() { @@ -268,19 +410,19 @@ impl ChainRunner { } for finding in result.findings { findings.push(NamespacedFinding { - middleware: entry.name.clone(), + middleware: entry.entry.name.clone(), finding, }); } if !result.metadata.is_empty() { metadata.insert( - entry.name.clone(), + entry.entry.name.clone(), result.metadata.clone().into_iter().collect(), ); } applied.push(MiddlewareInvocation { - name: entry.name.clone(), - implementation: entry.implementation.clone(), + name: entry.entry.name.clone(), + implementation: entry.entry.implementation.clone(), decision, transformed, failed: false, @@ -311,21 +453,22 @@ impl ChainRunner { } fn build_evaluation( - entry: &ChainEntry, + entry: &DescribedChainEntry, + binding: &MiddlewareBinding, input: &HttpRequestInput, headers: &BTreeMap, body: &[u8], ) -> HttpRequestEvaluation { HttpRequestEvaluation { api_version: API_VERSION.into(), - binding_id: entry.implementation.clone(), - phase: PRE_CREDENTIALS_PHASE.into(), + binding_id: binding.id.clone(), + phase: binding.phase.clone(), context: Some(RequestContext { request_id: input.request_id.clone(), sandbox_id: input.sandbox_id.clone(), originating_process: None, }), - config: Some(entry.config.clone()), + config: Some(entry.entry.config.clone()), target: Some(HttpRequestTarget { scheme: input.scheme.clone(), host: input.host.clone(), @@ -436,11 +579,16 @@ mod tests { } } - #[test] - fn phase_one_evaluation_omits_originating_process() { - let entry = entry("redact", OnError::FailClosed); + #[tokio::test] + async fn phase_one_evaluation_omits_originating_process() { + let entries = ChainRunner::default() + .describe_chain(&[entry("redact", OnError::FailClosed)]) + .await + .expect("describe chain"); + let entry = &entries[0]; + let binding = entry.binding.as_ref().expect("described binding"); let input = input("payload"); - let evaluation = build_evaluation(&entry, &input, &BTreeMap::new(), b"payload"); + let evaluation = build_evaluation(entry, binding, &input, &BTreeMap::new(), b"payload"); assert!( evaluation @@ -527,8 +675,9 @@ mod tests { .into_inner(); assert_eq!(manifest.api_version, API_VERSION); assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); - assert_eq!(manifest.bindings[0].operation, HTTP_REQUEST_OPERATION); - assert_eq!(manifest.bindings[0].phase, PRE_CREDENTIALS_PHASE); + assert_eq!(manifest.bindings[0].operation, "HttpRequest"); + assert_eq!(manifest.bindings[0].phase, "pre_credentials"); + assert_eq!(manifest.bindings[0].max_body_bytes, 256 * 1024); } #[test] @@ -561,6 +710,8 @@ mod tests { /// evaluation. Used to exercise chain behavior the built-in cannot produce /// (explicit deny, metadata, findings, unsafe header mutations). struct ScriptedService { + binding_id: String, + max_body_bytes: u64, result: openshell_core::proto::HttpRequestResult, } @@ -569,13 +720,18 @@ mod tests { async fn describe( &self, _request: Request<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - Ok(tonic::Response::new( - openshell_core::proto::MiddlewareManifest::default(), - )) + ) -> std::result::Result, tonic::Status> { + Ok(tonic::Response::new(MiddlewareManifest { + api_version: API_VERSION.into(), + name: "test/middleware".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: self.binding_id.clone(), + operation: "HttpRequest".into(), + phase: "pre_credentials".into(), + max_body_bytes: self.max_body_bytes, + }], + })) } async fn validate_config( @@ -604,6 +760,14 @@ mod tests { } } + fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService { + ScriptedService { + binding_id: BUILTIN_SECRETS.into(), + max_body_bytes: 256 * 1024, + result, + } + } + fn allow_result() -> openshell_core::proto::HttpRequestResult { openshell_core::proto::HttpRequestResult { decision: Decision::Allow as i32, @@ -617,14 +781,49 @@ mod tests { } #[tokio::test] - async fn deny_decision_short_circuits_chain() { + async fn descriptors_are_resolved_from_any_middleware_service() { let runner = ChainRunner::new(Arc::new(ScriptedService { - result: openshell_core::proto::HttpRequestResult { + binding_id: "example/redactor".into(), + max_body_bytes: 4096, + result: allow_result(), + })); + let entry = ChainEntry { + name: "external".into(), + implementation: "example/redactor".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + + let described = runner + .describe_chain(std::slice::from_ref(&entry)) + .await + .expect("describe external middleware"); + assert_eq!(described[0].max_body_bytes(), 4096); + assert_eq!( + described[0] + .binding + .as_ref() + .expect("described binding") + .phase, + "pre_credentials" + ); + + let outcome = runner + .evaluate_described(&described, input("hello")) + .await + .expect("evaluate external middleware"); + assert!(outcome.allowed); + } + + #[tokio::test] + async fn deny_decision_short_circuits_chain() { + let runner = ChainRunner::new(Arc::new(scripted_service( + openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, reason: "blocked_by_policy".into(), ..allow_result() }, - })); + ))); let outcome = runner .evaluate( &[ @@ -645,8 +844,8 @@ mod tests { #[tokio::test] async fn metadata_and_findings_are_namespaced_per_config() { - let runner = ChainRunner::new(Arc::new(ScriptedService { - result: openshell_core::proto::HttpRequestResult { + let runner = ChainRunner::new(Arc::new(scripted_service( + openshell_core::proto::HttpRequestResult { findings: vec![Finding { r#type: "pii.email".into(), label: "email address".into(), @@ -658,7 +857,7 @@ mod tests { .collect(), ..allow_result() }, - })); + ))); let outcome = runner .evaluate( &[ @@ -683,16 +882,14 @@ mod tests { } fn unsafe_header_service() -> ScriptedService { - ScriptedService { - result: openshell_core::proto::HttpRequestResult { - add_headers: std::iter::once(( - "x-openshell-middleware-inject".to_string(), - "ok\r\nHost: evil".to_string(), - )) - .collect(), - ..allow_result() - }, - } + scripted_service(openshell_core::proto::HttpRequestResult { + add_headers: std::iter::once(( + "x-openshell-middleware-inject".to_string(), + "ok\r\nHost: evil".to_string(), + )) + .collect(), + ..allow_result() + }) } #[tokio::test] @@ -724,13 +921,79 @@ mod tests { } #[tokio::test] - async fn unspecified_decision_uses_fail_closed() { + async fn oversized_replacement_body_honors_on_error() { let runner = ChainRunner::new(Arc::new(ScriptedService { + binding_id: BUILTIN_SECRETS.into(), + max_body_bytes: 4, result: openshell_core::proto::HttpRequestResult { - decision: Decision::Unspecified as i32, + body: b"too large".to_vec(), + has_body: true, ..allow_result() }, })); + let fail_open = entry("small", OnError::FailOpen); + let mut fail_closed = fail_open.clone(); + fail_closed.on_error = OnError::FailClosed; + + let open_outcome = runner + .evaluate(&[fail_open], input("safe")) + .await + .expect("fail-open evaluation"); + assert!(open_outcome.allowed); + assert_eq!(open_outcome.body, b"safe"); + assert!(open_outcome.applied[0].failed); + + let closed_outcome = runner + .evaluate(&[fail_closed], input("safe")) + .await + .expect("fail-closed evaluation"); + assert!(!closed_outcome.allowed); + assert_eq!( + closed_outcome.reason, + "middleware_failed: response_body_over_capacity" + ); + assert!(closed_outcome.applied[0].failed); + } + + #[tokio::test] + async fn oversized_request_body_honors_on_error() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + binding_id: BUILTIN_SECRETS.into(), + max_body_bytes: 4, + result: allow_result(), + })); + let fail_open = entry("small", OnError::FailOpen); + let mut fail_closed = fail_open.clone(); + fail_closed.on_error = OnError::FailClosed; + + let open_outcome = runner + .evaluate(&[fail_open], input("hello")) + .await + .expect("fail-open evaluation"); + assert!(open_outcome.allowed); + assert_eq!(open_outcome.body, b"hello"); + assert!(open_outcome.applied[0].failed); + + let closed_outcome = runner + .evaluate(&[fail_closed], input("hello")) + .await + .expect("fail-closed evaluation"); + assert!(!closed_outcome.allowed); + assert_eq!( + closed_outcome.reason, + "middleware_failed: request_body_over_capacity" + ); + assert!(closed_outcome.applied[0].failed); + } + + #[tokio::test] + async fn unspecified_decision_uses_fail_closed() { + let runner = ChainRunner::new(Arc::new(scripted_service( + openshell_core::proto::HttpRequestResult { + decision: Decision::Unspecified as i32, + ..allow_result() + }, + ))); let outcome = runner .evaluate(&[entry("redact", OnError::FailClosed)], input("hello")) diff --git a/crates/openshell-supervisor-middleware/src/service.rs b/crates/openshell-supervisor-middleware/src/service.rs index cbd9231cdf..51df8d0700 100644 --- a/crates/openshell-supervisor-middleware/src/service.rs +++ b/crates/openshell-supervisor-middleware/src/service.rs @@ -3,15 +3,12 @@ use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding, MiddlewareManifest, - ValidateConfigRequest, ValidateConfigResponse, + HttpRequestEvaluation, HttpRequestResult, MiddlewareManifest, ValidateConfigRequest, + ValidateConfigResponse, }; use tonic::{Request, Response, Status}; -use crate::{ - API_VERSION, BUILTIN_SECRETS, HTTP_REQUEST_OPERATION, PRE_CREDENTIALS_PHASE, builtins, - safe_reason, validate_builtin_config, -}; +use crate::{API_VERSION, builtins, safe_reason, validate_builtin_config}; #[derive(Debug, Default)] pub struct InProcessMiddlewareService; @@ -26,11 +23,7 @@ impl SupervisorMiddleware for InProcessMiddlewareService { api_version: API_VERSION.into(), name: "openshell/in-process".into(), service_version: env!("CARGO_PKG_VERSION").into(), - bindings: vec![MiddlewareBinding { - id: BUILTIN_SECRETS.into(), - operation: HTTP_REQUEST_OPERATION.into(), - phase: PRE_CREDENTIALS_PHASE.into(), - }], + bindings: builtins::describe(), })) } @@ -58,13 +51,8 @@ impl SupervisorMiddleware for InProcessMiddlewareService { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let result = match request.binding_id.as_str() { - BUILTIN_SECRETS => builtins::secrets::evaluate_http_request(&request), - other => Err(miette::miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - } - .map_err(|err| Status::invalid_argument(safe_reason(&err.to_string())))?; + let result = builtins::evaluate_http_request(&request) + .map_err(|err| Status::invalid_argument(safe_reason(&err.to_string())))?; Ok(Response::new(result)) } } diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index 52f6f10461..fcc5838e1f 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -842,23 +842,6 @@ _policy_endpoint_configs(policy) := [ep | endpoint_has_extended_config(ep) ] -# Collect matching endpoint identities across all policies. Iterates over -# _matching_policy_names (a set, safe from regorus variable collisions) then -# returns the selected policy name plus endpoint index/path. Rust uses that -# identity to look up middleware attachment from policy data. -_matching_endpoint_contexts := [ctx | - some pname - _matching_policy_names[pname] - policy := data.network_policies[pname] - ep := policy.endpoints[i] - endpoint_matches_request(ep, input.network) - ctx := { - "policy": pname, - "endpoint_index": i, - "endpoint_path": object.get(ep, "path", ""), - } -] - _matching_endpoint_configs := [cfg | some pname _matching_policy_names[pname] @@ -871,8 +854,6 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { count(_matching_endpoint_configs) > 0 } -network_policies := object.get(data, "network_policies", {}) - network_middlewares := object.get(data, "network_middlewares", []) _policy_has_exact_declared_endpoint(policy) if { diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index c773fdcf48..8383b6bb2b 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -453,8 +453,7 @@ where let _ = &eval_target; if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = - engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let req = match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) .await? @@ -773,20 +772,45 @@ pub(crate) enum MiddlewareApplyResult { Denied(String), } +fn middleware_chain_body_limit( + chain: &[openshell_supervisor_middleware::DescribedChainEntry], +) -> Option { + chain + .iter() + .map(openshell_supervisor_middleware::DescribedChainEntry::max_body_bytes) + .min() +} + pub(crate) async fn apply_middleware_chain( req: crate::l7::provider::L7Request, client: &mut C, ctx: &L7EvalContext, chain: Vec, generation_guard: &PolicyGenerationGuard, +) -> Result { + apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, generation_guard).await +} + +pub(crate) async fn apply_middleware_chain_for_scheme( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + generation_guard: &PolicyGenerationGuard, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); } + let runner = openshell_supervisor_middleware::ChainRunner::default(); + let chain = runner.describe_chain(&chain).await?; + let max_body_bytes = + middleware_chain_body_limit(&chain).expect("non-empty middleware chain has a body limit"); let buffered = match crate::l7::rest::buffer_request_body_for_middleware( &req, client, Some(generation_guard), + max_body_bytes, ) .await? { @@ -797,21 +821,8 @@ pub(crate) async fn apply_middleware_chain, + query: String, + body: Vec, +) -> openshell_supervisor_middleware::HttpRequestInput { + openshell_supervisor_middleware::HttpRequestInput { + request_id: uuid::Uuid::new_v4().to_string(), + sandbox_id: openshell_ocsf::ctx::ctx().sandbox_id.clone(), + scheme: scheme.into(), + host: ctx.host.clone(), + port: ctx.port, + method: req.action.clone(), + path: req.target.clone(), + query, + headers, + body, + } +} + fn raw_query_from_request_headers(headers: &[u8]) -> Result { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; @@ -846,12 +879,12 @@ fn raw_query_from_request_headers(headers: &[u8]) -> Result { fn resolve_unbuffered_body( ctx: &L7EvalContext, req: crate::l7::provider::L7Request, - chain: &[openshell_supervisor_middleware::ChainEntry], + chain: &[openshell_supervisor_middleware::DescribedChainEntry], recoverable: bool, ) -> MiddlewareApplyResult { let all_fail_open = chain .iter() - .all(|entry| entry.on_error == openshell_supervisor_middleware::OnError::FailOpen); + .all(|entry| entry.on_error() == openshell_supervisor_middleware::OnError::FailOpen); if recoverable && all_fail_open { emit_middleware_body_unavailable(ctx, false); return MiddlewareApplyResult::Allowed(req); @@ -1187,8 +1220,7 @@ where let _ = &eval_target; if allowed || config.enforcement == EnforcementMode::Audit { - let chain = - engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let req = match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) .await? @@ -1457,8 +1489,7 @@ where } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = - engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let req = match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) .await? @@ -1682,8 +1713,7 @@ where let _ = &eval_target; if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = - engine.query_middleware_chain(&middleware_network_input(ctx), &req.target)?; + let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let req = match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) .await? @@ -2136,8 +2166,7 @@ where let req = if let Some(engine) = middleware_engine { let input = middleware_network_input(ctx); - let (chain, generation) = - engine.query_middleware_chain_with_generation(&input, &req.target)?; + let (chain, generation) = engine.query_middleware_chain_with_generation(&input)?; if generation != generation_guard.captured_generation() { return Ok(()); } @@ -2326,10 +2355,11 @@ network_middlewares: - name: request-middleware middleware: {middleware_impl} on_error: {on_error} + endpoints: + include: ["api.example.test"] network_policies: rest_api: name: rest_api - middleware: ["request-middleware"] endpoints: - host: api.example.test port: 8080 @@ -2785,10 +2815,11 @@ network_middlewares: - name: request-middleware middleware: example/unavailable on_error: fail_closed + endpoints: + include: ["api.example.test"] network_policies: jsonrpc_api: name: jsonrpc_api - middleware: ["request-middleware"] endpoints: - host: api.example.test port: 443 @@ -2929,8 +2960,8 @@ network_policies: .unwrap(); } - #[test] - fn over_capacity_resolution_honors_on_error() { + #[tokio::test] + async fn over_capacity_resolution_honors_on_error() { use openshell_supervisor_middleware::{ChainEntry, OnError}; let ctx = L7EvalContext { @@ -2963,21 +2994,31 @@ network_policies: ..fail_open.clone() }; + let runner = openshell_supervisor_middleware::ChainRunner::default(); + let open_chain = runner + .describe_chain(std::slice::from_ref(&fail_open)) + .await + .expect("describe fail-open chain"); + let mixed_chain = runner + .describe_chain(&[fail_open.clone(), fail_closed]) + .await + .expect("describe mixed chain"); + // Recoverable (Content-Length over cap, nothing consumed) + all fail-open // -> stream through unprocessed. assert!(matches!( - resolve_unbuffered_body(&ctx, req(), std::slice::from_ref(&fail_open), true), + resolve_unbuffered_body(&ctx, req(), &open_chain, true), MiddlewareApplyResult::Allowed(_) )); // Any fail-closed entry -> deny. assert!(matches!( - resolve_unbuffered_body(&ctx, req(), &[fail_open.clone(), fail_closed], true), + resolve_unbuffered_body(&ctx, req(), &mixed_chain, true), MiddlewareApplyResult::Denied(_) )); // Not recoverable (chunked overflow already consumed bytes) -> deny even // when every entry is fail-open. assert!(matches!( - resolve_unbuffered_body(&ctx, req(), &[fail_open], false), + resolve_unbuffered_body(&ctx, req(), &open_chain, false), MiddlewareApplyResult::Denied(_) )); } @@ -2992,6 +3033,40 @@ network_policies: assert_eq!(query, "token=a%2Bb&scope=private"); } + #[test] + fn middleware_request_input_preserves_plain_http_scheme() { + let req = crate::l7::provider::L7Request { + action: "POST".into(), + target: "/v1/messages".into(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 80, + policy_name: "api".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + + let input = middleware_request_input( + "http", + &req, + &ctx, + BTreeMap::new(), + String::new(), + Vec::new(), + ); + + assert_eq!(input.scheme, "http"); + } + /// Tracing layer that captures emitted `OcsfEvent`s for assertions. struct OcsfCaptureLayer(Arc>>); @@ -3096,16 +3171,17 @@ network_policies: #[tokio::test] async fn passthrough_relay_runs_middleware_redaction() { // A no-protocol endpoint takes the credential-injection passthrough path; - // policy-level middleware must still inspect and redact its body. + // host-selected middleware must still inspect and redact its body. let data = r#" network_middlewares: - name: request-middleware middleware: openshell/secrets on_error: fail_closed + endpoints: + include: ["api.example.test"] network_policies: passthrough_api: name: passthrough_api - middleware: ["request-middleware"] endpoints: - host: api.example.test port: 8080 @@ -3197,10 +3273,11 @@ network_middlewares: - name: request-middleware middleware: example/unavailable on_error: fail_closed + endpoints: + include: ["gateway.example.test"] network_policies: ws_api: name: ws_api - middleware: ["request-middleware"] endpoints: - host: gateway.example.test port: 443 diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 19f73e2ad3..15825d1b23 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,7 +27,19 @@ const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; /// Maximum body bytes for `SigV4` body-signing mode. Larger than the credential /// rewrite limit because Bedrock payloads can be several megabytes. const MAX_SIGV4_BODY_BYTES: usize = 10 * 1024 * 1024; -pub(crate) const MAX_MIDDLEWARE_BODY_BYTES: usize = MAX_REWRITE_BODY_BYTES; +#[cfg(test)] +async fn max_middleware_body_bytes() -> usize { + let chain = openshell_supervisor_middleware::ChainRunner::default() + .describe_chain(&[openshell_supervisor_middleware::ChainEntry { + name: "test".into(), + implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]) + .await + .expect("describe built-in middleware"); + chain[0].max_body_bytes() +} const RELAY_BUF_SIZE: usize = 8192; const HTTP_METHOD_PREFIXES: &[&[u8]] = &[ b"GET ", @@ -820,6 +832,7 @@ pub(crate) async fn buffer_request_body_for_middleware( req: &L7Request, client: &mut C, generation_guard: Option<&PolicyGenerationGuard>, + max_body_bytes: usize, ) -> Result { let header_end = req .raw_header @@ -840,7 +853,7 @@ pub(crate) async fn buffer_request_body_for_middleware( let Ok(len) = usize::try_from(len) else { return Ok(BufferResult::OverCapacity { recoverable: true }); }; - if len > MAX_MIDDLEWARE_BODY_BYTES { + if len > max_body_bytes { return Ok(BufferResult::OverCapacity { recoverable: true }); } let initial_len = already_read.len().min(len); @@ -869,14 +882,18 @@ pub(crate) async fn buffer_request_body_for_middleware( } BodyLength::Chunked => { // Chunked bodies are decoded incrementally into the payload bytes - // middleware expects. On overflow, we have already consumed wire - // bytes from the client stream and cannot re-enter the normal raw - // relay path without a separate splice-through buffer. - Ok(collect_chunked_body(client, already_read, generation_guard) - .await - .map_or(BufferResult::OverCapacity { recoverable: false }, |body| { - BufferResult::Buffered(BufferedRequestBody { headers, body }) - })) + // middleware expects, but the middleware cap counts the complete + // wire representation, including framing and trailers. On overflow, + // we have already consumed wire bytes from the client stream and + // cannot re-enter the normal raw relay path without a separate + // splice-through buffer. + Ok( + collect_chunked_body(client, already_read, generation_guard, Some(max_body_bytes)) + .await + .map_or(BufferResult::OverCapacity { recoverable: false }, |body| { + BufferResult::Buffered(BufferedRequestBody { headers, body }) + }), + ) } } } @@ -953,7 +970,7 @@ async fn collect_and_rewrite_request_body( Ok(PreparedRequestBody { headers, body }) } BodyLength::Chunked => { - let body = collect_chunked_body(client, already_read, generation_guard).await?; + let body = collect_chunked_body(client, already_read, generation_guard, None).await?; let (mut headers, body) = rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; headers = set_content_length(&headers, body.len())?; @@ -1125,15 +1142,19 @@ async fn collect_chunked_body( client: &mut C, already_read: &[u8], generation_guard: Option<&PolicyGenerationGuard>, + max_wire_bytes: Option, ) -> Result> { - let mut buffered_pos = 0usize; + let mut read_state = ChunkedReadState { + buffered_pos: 0, + wire_bytes: 0, + max_wire_bytes, + }; let mut body = Vec::new(); loop { - let size_line = - read_chunked_line(client, already_read, &mut buffered_pos, generation_guard) - .await - .map_err(|e| miette!("Chunked body ended before chunk-size line: {e}"))?; + let size_line = read_chunked_line(client, already_read, &mut read_state, generation_guard) + .await + .map_err(|e| miette!("Chunked body ended before chunk-size line: {e}"))?; let size_line = std::str::from_utf8(&size_line) .into_diagnostic() .map_err(|_| miette!("Invalid UTF-8 in chunk-size line"))?; @@ -1149,7 +1170,7 @@ async fn collect_chunked_body( if chunk_size == 0 { loop { let trailer_line = - read_chunked_line(client, already_read, &mut buffered_pos, generation_guard) + read_chunked_line(client, already_read, &mut read_state, generation_guard) .await .map_err(|e| { miette!("Chunked body ended before trailer terminator: {e}") @@ -1168,7 +1189,7 @@ async fn collect_chunked_body( read_buffered_exact( client, already_read, - &mut buffered_pos, + &mut read_state, chunk_size, &mut body, generation_guard, @@ -1180,7 +1201,7 @@ async fn collect_chunked_body( read_buffered_exact( client, already_read, - &mut buffered_pos, + &mut read_state, 2, &mut chunk_crlf, generation_guard, @@ -1193,15 +1214,21 @@ async fn collect_chunked_body( } } +struct ChunkedReadState { + buffered_pos: usize, + wire_bytes: usize, + max_wire_bytes: Option, +} + async fn read_chunked_line( client: &mut C, already_read: &[u8], - buffered_pos: &mut usize, + state: &mut ChunkedReadState, generation_guard: Option<&PolicyGenerationGuard>, ) -> Result> { let mut line = Vec::new(); loop { - let byte = read_buffered_byte(client, already_read, buffered_pos, generation_guard).await?; + let byte = read_buffered_byte(client, already_read, state, generation_guard).await?; line.push(byte); if line.len() > MAX_REWRITE_BODY_BYTES { return Err(miette!( @@ -1218,13 +1245,13 @@ async fn read_chunked_line( async fn read_buffered_exact( client: &mut C, already_read: &[u8], - buffered_pos: &mut usize, + state: &mut ChunkedReadState, len: usize, out: &mut Vec, generation_guard: Option<&PolicyGenerationGuard>, ) -> Result<()> { for _ in 0..len { - let byte = read_buffered_byte(client, already_read, buffered_pos, generation_guard).await?; + let byte = read_buffered_byte(client, already_read, state, generation_guard).await?; out.push(byte); } Ok(()) @@ -1233,18 +1260,30 @@ async fn read_buffered_exact( async fn read_buffered_byte( client: &mut C, already_read: &[u8], - buffered_pos: &mut usize, + state: &mut ChunkedReadState, generation_guard: Option<&PolicyGenerationGuard>, ) -> Result { - if *buffered_pos < already_read.len() { - let byte = already_read[*buffered_pos]; - *buffered_pos += 1; - return Ok(byte); - } - let byte = client.read_u8().await.into_diagnostic()?; - if let Some(guard) = generation_guard { - guard.ensure_current()?; + if state + .max_wire_bytes + .is_some_and(|max| state.wire_bytes >= max) + { + return Err(miette!( + "chunked body wire representation exceeds middleware buffer limit" + )); } + + let byte = if state.buffered_pos < already_read.len() { + let byte = already_read[state.buffered_pos]; + state.buffered_pos += 1; + byte + } else { + let byte = client.read_u8().await.into_diagnostic()?; + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + byte + }; + state.wire_bytes += 1; Ok(byte) } @@ -3264,6 +3303,7 @@ mod tests { &mut client, b"5\r\nhello\r\n6;ext=value\r\n world\r\n0\r\nx-checksum: abc\r\n\r\n", None, + None, ) .await .expect("chunked body should decode"); @@ -3271,6 +3311,40 @@ mod tests { assert_eq!(body, b"hello world"); } + #[tokio::test] + async fn middleware_chunked_wire_body_at_cap_is_allowed() { + let max_body_bytes = max_middleware_body_bytes().await; + let payload_len = max_body_bytes - 14; + let mut wire = format!("{payload_len:x}\r\n").into_bytes(); + wire.extend(std::iter::repeat_n(b'x', payload_len)); + wire.extend_from_slice(b"\r\n0\r\n\r\n"); + assert_eq!(wire.len(), max_body_bytes); + + let body = collect_chunked_body(&mut tokio::io::empty(), &wire, None, Some(max_body_bytes)) + .await + .expect("wire representation at the cap should be allowed"); + + assert_eq!(body.len(), payload_len); + } + + #[tokio::test] + async fn middleware_chunked_wire_body_over_cap_is_rejected() { + let max_body_bytes = max_middleware_body_bytes().await; + let payload_len = max_body_bytes - 13; + let mut wire = format!("{payload_len:x}\r\n").into_bytes(); + wire.extend(std::iter::repeat_n(b'x', payload_len)); + wire.extend_from_slice(b"\r\n0\r\n\r\n"); + assert_eq!(wire.len(), max_body_bytes + 1); + assert!(payload_len < max_body_bytes); + + let error = + collect_chunked_body(&mut tokio::io::empty(), &wire, None, Some(max_body_bytes)) + .await + .expect_err("wire framing over the cap must be rejected"); + + assert!(error.to_string().contains("wire representation")); + } + /// SEC-009: Bare LF in headers enables header injection. #[tokio::test] async fn reject_bare_lf_in_headers() { diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 3efec02127..c4e7739963 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -135,17 +135,13 @@ impl TunnelPolicyEngine { &self.engine } - /// Query the ordered middleware chain for a request path within this tunnel. - pub fn query_middleware_chain( - &self, - input: &NetworkInput, - request_path: &str, - ) -> Result> { + /// Query the ordered middleware chain for a destination within this tunnel. + pub fn query_middleware_chain(&self, input: &NetworkInput) -> Result> { let mut engine = self .engine .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; - query_middleware_chain_locked(&mut engine, input, request_path) + query_middleware_chain_locked(&mut engine, input) } } @@ -208,21 +204,21 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { + if let Err(violations) = openshell_policy::validate_sandbox_policy(proto) { + let errors = violations + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"); + return Err(miette::miette!("policy validation failed:\n{errors}")); + } + let data_json_str = proto_to_opa_data_json(proto, entrypoint_pid); // Parse back to Value for preprocessing, then re-serialize let mut data: serde_json::Value = serde_json::from_str(&data_json_str) .map_err(|e| miette::miette!("internal: failed to parse proto JSON: {e}"))?; - // Validate BEFORE expanding presets - let middleware_errors = validate_middleware_policies(&data); - if !middleware_errors.is_empty() { - return Err(miette::miette!( - "middleware policy validation failed:\n{}", - middleware_errors.join("\n") - )); - } - let (errors, warnings) = crate::l7::validate_l7_policies(&data); for w in &warnings { openshell_ocsf::ocsf_emit!( @@ -571,18 +567,17 @@ impl OpaEngine { } } - /// Query the ordered middleware chain for a parsed HTTP request path. + /// Query the ordered middleware chain for an admitted destination. pub fn query_middleware_chain_with_generation( &self, input: &NetworkInput, - request_path: &str, ) -> Result<(Vec, u64)> { let mut engine = self .engine .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; let generation = self.current_generation(); - let chain = query_middleware_chain_locked(&mut engine, input, request_path)?; + let chain = query_middleware_chain_locked(&mut engine, input)?; Ok((chain, generation)) } @@ -749,17 +744,9 @@ fn network_input_json(input: &NetworkInput) -> serde_json::Value { }) } -#[derive(Debug, Clone)] -struct MatchedEndpointContext { - policy_name: String, - endpoint_index: usize, - endpoint_path: String, -} - fn query_middleware_chain_locked( engine: &mut regorus::Engine, input: &NetworkInput, - request_path: &str, ) -> Result> { engine .set_input_json(&network_input_json(input).to_string()) @@ -772,37 +759,7 @@ fn query_middleware_chain_locked( if configs.is_empty() { return Ok(Vec::new()); } - let contexts_val = engine - .eval_rule("data.openshell.sandbox._matching_endpoint_contexts".into()) - .map_err(|e| miette::miette!("{e}"))?; - let contexts = parse_endpoint_contexts(&contexts_val); - let policies_val = engine - .eval_rule("data.openshell.sandbox.network_policies".into()) - .map_err(|e| miette::miette!("{e}"))?; - let Some(context) = select_endpoint_context(&contexts, request_path, &policies_val)? else { - return global_middleware_entries(&configs, &input.host, &HashSet::new()); - }; - let (policy_middleware, endpoint_middleware) = - middleware_for_endpoint_identity(&policies_val, context)?; - - let mut explicit = Vec::new(); - for name in policy_middleware.iter().chain(endpoint_middleware.iter()) { - if !explicit.contains(name) { - explicit.push(name.clone()); - } - } - let explicit_set: HashSet = explicit.iter().cloned().collect(); - let mut ordered = global_middleware_entries(&configs, &input.host, &explicit_set)?; - for name in explicit { - if !ordered.iter().any(|entry| entry.name == name) { - let config = configs - .iter() - .find(|config| get_str(config, "name").as_deref() == Some(name.as_str())) - .ok_or_else(|| miette::miette!("unknown middleware config '{name}'"))?; - ordered.push(chain_entry_from_value(config)?); - } - } - Ok(ordered) + global_middleware_entries(&configs, &input.host) } fn parse_middleware_configs(value: ®orus::Value) -> Result> { @@ -815,169 +772,37 @@ fn parse_middleware_configs(value: ®orus::Value) -> Result Vec { - let regorus::Value::Array(values) = value else { - return Vec::new(); - }; - values - .iter() - .filter_map(|value| { - let regorus::Value::Object(_) = value else { - return None; - }; - Some(MatchedEndpointContext { - policy_name: get_str(value, "policy").unwrap_or_default(), - endpoint_index: get_usize(value, "endpoint_index").unwrap_or_default(), - endpoint_path: get_str(value, "endpoint_path").unwrap_or_default(), - }) - }) - .collect() -} - -fn middleware_for_endpoint_identity( - policies: ®orus::Value, - context: &MatchedEndpointContext, -) -> Result<(Vec, Vec)> { - let policy = get_field(policies, &context.policy_name).ok_or_else(|| { - miette::miette!( - "matched endpoint policy '{}' was not found in OPA data", - context.policy_name - ) - })?; - let endpoint = get_array(policy, "endpoints") - .and_then(|endpoints| endpoints.get(context.endpoint_index)) - .ok_or_else(|| { - miette::miette!( - "matched endpoint {}[{}] was not found in OPA data", - context.policy_name, - context.endpoint_index - ) - })?; - Ok(( - get_str_array(policy, "middleware"), - get_str_array(endpoint, "middleware"), - )) -} - -fn select_endpoint_context<'a>( - contexts: &'a [MatchedEndpointContext], - request_path: &str, - policies: ®orus::Value, -) -> Result> { - let matching: Vec<_> = contexts - .iter() - .filter(|context| crate::l7::endpoint_path_matches(&context.endpoint_path, request_path)) - .map(|context| (endpoint_path_specificity(&context.endpoint_path), context)) - .collect(); - let Some(max_specificity) = matching.iter().map(|(specificity, _)| *specificity).max() else { - return Ok(None); - }; - let best: Vec<_> = matching - .into_iter() - .filter(|(specificity, _)| *specificity == max_specificity) - .map(|(_, context)| context) - .collect(); - if let Some((first, rest)) = best.split_first() { - let first_middleware = explicit_middleware_for_endpoint_identity(policies, first)?; - for context in rest { - if explicit_middleware_for_endpoint_identity(policies, context)? != first_middleware { - let matches = best - .iter() - .map(|context| { - format!( - "{}[{}] path={}", - context.policy_name, - context.endpoint_index, - if context.endpoint_path.is_empty() { - "" - } else { - context.endpoint_path.as_str() - } - ) - }) - .collect::>() - .join(", "); - return Err(miette::miette!( - "ambiguous middleware endpoint match for request path '{request_path}': {matches}" - )); - } - } - } - Ok(best.into_iter().next()) -} - -fn explicit_middleware_for_endpoint_identity( - policies: ®orus::Value, - context: &MatchedEndpointContext, -) -> Result> { - let (policy_middleware, endpoint_middleware) = - middleware_for_endpoint_identity(policies, context)?; - Ok(dedup_middleware_names( - policy_middleware.iter().chain(endpoint_middleware.iter()), - )) -} - -fn dedup_middleware_names<'a>(names: impl IntoIterator) -> Vec { - let mut deduped = Vec::new(); - for name in names { - if !deduped.contains(name) { - deduped.push(name.clone()); - } - } - deduped -} - -fn endpoint_path_specificity(path: &str) -> usize { - if path.is_empty() { - 0 - } else { - path.chars().filter(|c| *c != '*').count() - } -} - -fn global_middleware_entries( - configs: &[regorus::Value], - host: &str, - explicit: &HashSet, -) -> Result> { +fn global_middleware_entries(configs: &[regorus::Value], host: &str) -> Result> { let mut entries = Vec::new(); for config in configs { - let name = get_str(config, "name").unwrap_or_default(); - if explicit.contains(&name) { - continue; - } - if middleware_selector_matches(config, host) { + if middleware_selector_matches(config, host)? { entries.push(chain_entry_from_value(config)?); } } Ok(entries) } -fn middleware_selector_matches(config: ®orus::Value, host: &str) -> bool { +fn middleware_selector_matches(config: ®orus::Value, host: &str) -> Result { let Some(selector) = get_field(config, "endpoints") else { - return false; + return Ok(false); }; let include_patterns = get_str_array(selector, "include"); let exclude_patterns = get_str_array(selector, "exclude"); - let matches_include = !include_patterns.is_empty() - && include_patterns - .iter() - .any(|pattern| host_matches(pattern, host)); + let matches_include = include_patterns + .iter() + .try_fold(false, |matched, pattern| { + openshell_policy::middleware_host_matches(pattern, host) + .map(|matches| matched || matches) + .map_err(|error| miette::miette!(error)) + })?; let matches_exclude = exclude_patterns .iter() - .any(|pattern| host_matches(pattern, host)); - matches_include && !matches_exclude -} - -fn host_matches(pattern: &str, host: &str) -> bool { - if pattern == "*" || pattern == "**" { - return true; - } - if !pattern.contains('*') { - return pattern.eq_ignore_ascii_case(host); - } - glob::Pattern::new(&pattern.to_ascii_lowercase()) - .is_ok_and(|pattern| pattern.matches(&host.to_ascii_lowercase())) + .try_fold(false, |matched, pattern| { + openshell_policy::middleware_host_matches(pattern, host) + .map(|matches| matched || matches) + .map_err(|error| miette::miette!(error)) + })?; + Ok(matches_include && !matches_exclude) } fn chain_entry_from_value(value: ®orus::Value) -> Result { @@ -1003,25 +828,6 @@ fn get_field<'a>(val: &'a regorus::Value, key: &str) -> Option<&'a regorus::Valu } } -fn get_array<'a>(val: &'a regorus::Value, key: &str) -> Option<&'a [regorus::Value]> { - let regorus::Value::Array(values) = get_field(val, key)? else { - return None; - }; - Some(values) -} - -fn get_usize(val: ®orus::Value, key: &str) -> Option { - let value = get_field(val, key)?; - let regorus::Value::Number(number) = value else { - return None; - }; - let value = number.as_f64()?; - if !value.is_finite() || value.fract() != 0.0 || value < 0.0 { - return None; - } - format!("{value:.0}").parse::().ok() -} - fn regorus_value_to_struct(value: ®orus::Value) -> prost_types::Struct { let regorus::Value::Object(map) = value else { return prost_types::Struct::default(); @@ -1383,6 +1189,29 @@ fn validate_middleware_policies(data: &serde_json::Value) -> Vec { "middleware config '{name}' has invalid on_error '{on_error}'" )); } + + let Some(selector) = mw.get("endpoints") else { + errors.push(format!( + "middleware config '{name}' requires an endpoint selector" + )); + continue; + }; + let includes = json_string_array(selector.get("include")); + let excludes = json_string_array(selector.get("exclude")); + if includes.is_empty() { + errors.push(format!( + "middleware config '{name}' endpoint selector must include at least one host pattern" + )); + } + for pattern in includes.iter().chain(&excludes) { + if let Err(error) = + openshell_policy::middleware_host_matches(pattern, "validation.invalid") + { + errors.push(format!( + "middleware config '{name}' has invalid endpoint selector pattern '{pattern}': {error}" + )); + } + } } let Some(policies) = data @@ -1393,127 +1222,25 @@ fn validate_middleware_policies(data: &serde_json::Value) -> Vec { }; for (policy_name, policy) in policies { - let policy_middleware = json_string_array(policy.get("middleware")); - for name in &policy_middleware { - if !names.contains(name) { - errors.push(format!( - "network policy '{policy_name}' references unknown middleware config '{name}'" - )); - } - } for endpoint in policy .get("endpoints") .and_then(serde_json::Value::as_array) .map_or(&[][..], Vec::as_slice) { - let endpoint_middleware = json_string_array(endpoint.get("middleware")); - for name in &endpoint_middleware { - if !names.contains(name) { - errors.push(format!( - "network policy '{policy_name}' endpoint references unknown middleware config '{name}'" - )); - } - } let tls_skip = endpoint .get("tls") .and_then(serde_json::Value::as_str) .is_some_and(|tls| tls == "skip"); - if tls_skip && (!policy_middleware.is_empty() || !endpoint_middleware.is_empty()) { - errors.push(format!( - "network policy '{policy_name}' attaches middleware to a tls: skip endpoint" - )); - } if tls_skip && global_selector_matches_any_middleware(middlewares, endpoint) { errors.push(format!( "network policy '{policy_name}' tls: skip endpoint matches a global middleware selector" )); } } - validate_ambiguous_middleware_endpoints( - policy_name, - policy, - &policy_middleware, - &mut errors, - ); } errors } -fn validate_ambiguous_middleware_endpoints( - policy_name: &str, - policy: &serde_json::Value, - policy_middleware: &[String], - errors: &mut Vec, -) { - let endpoints = policy - .get("endpoints") - .and_then(serde_json::Value::as_array) - .map_or(&[][..], Vec::as_slice); - let mut seen: Vec<(usize, MiddlewareEndpointKey, Vec)> = Vec::new(); - for (index, endpoint) in endpoints.iter().enumerate() { - let key = middleware_endpoint_key(endpoint); - let endpoint_middleware = json_string_array(endpoint.get("middleware")); - let chain = - dedup_middleware_names(policy_middleware.iter().chain(endpoint_middleware.iter())); - for (previous_index, previous_key, previous_chain) in &seen { - if previous_key == &key && previous_chain != &chain { - errors.push(format!( - "network policy '{policy_name}' endpoints[{previous_index}] and endpoints[{index}] have equivalent middleware selection keys ({key}) but different middleware chains" - )); - } - } - seen.push((index, key, chain)); - } -} - -#[derive(Debug, PartialEq, Eq)] -struct MiddlewareEndpointKey { - host: String, - ports: Vec, - path: String, -} - -impl std::fmt::Display for MiddlewareEndpointKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "host={} ports={:?} path={}", - if self.host.is_empty() { - "" - } else { - self.host.as_str() - }, - self.ports, - if self.path.is_empty() { - "" - } else { - self.path.as_str() - } - ) - } -} - -fn middleware_endpoint_key(endpoint: &serde_json::Value) -> MiddlewareEndpointKey { - let host = endpoint - .get("host") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_ascii_lowercase(); - let mut ports: Vec = endpoint - .get("ports") - .and_then(serde_json::Value::as_array) - .map(|ports| ports.iter().filter_map(serde_json::Value::as_u64).collect()) - .unwrap_or_default(); - ports.sort_unstable(); - ports.dedup(); - let path = endpoint - .get("path") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_string(); - MiddlewareEndpointKey { host, ports, path } -} - fn json_string_array(value: Option<&serde_json::Value>) -> Vec { value .and_then(serde_json::Value::as_array) @@ -1542,8 +1269,12 @@ fn global_selector_matches_any_middleware( let includes = json_string_array(selector.get("include")); let excludes = json_string_array(selector.get("exclude")); !includes.is_empty() - && includes.iter().any(|pattern| host_matches(pattern, host)) - && !excludes.iter().any(|pattern| host_matches(pattern, host)) + && includes.iter().any(|pattern| { + openshell_policy::middleware_host_matches(pattern, host).unwrap_or(false) + }) + && !excludes.iter().any(|pattern| { + openshell_policy::middleware_host_matches(pattern, host).unwrap_or(false) + }) }) } @@ -1908,9 +1639,6 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St allow_all_known_mcp_methods.into(); } } - if !e.middleware.is_empty() { - ep["middleware"] = e.middleware.clone().into(); - } ep }) .collect(); @@ -1936,14 +1664,11 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St entries }) .collect(); - let mut policy = serde_json::json!({ + let policy = serde_json::json!({ "name": rule.name, "endpoints": endpoints, "binaries": binaries, }); - if !rule.middleware.is_empty() { - policy["middleware"] = rule.middleware.clone().into(); - } (key.clone(), policy) }) .collect(); @@ -2058,7 +1783,6 @@ mod tests { path: "/usr/local/bin/claude".to_string(), ..Default::default() }], - ..Default::default() }, ); network_policies.insert( @@ -2074,7 +1798,6 @@ mod tests { path: "/usr/bin/glab".to_string(), ..Default::default() }], - ..Default::default() }, ); ProtoSandboxPolicy { @@ -3417,7 +3140,6 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -3490,7 +3212,6 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], - middleware: vec![], }, ); @@ -3564,7 +3285,6 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], - middleware: vec![], }, ); @@ -4443,7 +4163,6 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -4502,7 +4221,6 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -4562,7 +4280,6 @@ network_policies: path: "/usr/local/bin/claude".to_string(), ..Default::default() }], - middleware: vec![], }, ); let proto = ProtoSandboxPolicy { @@ -4624,7 +4341,6 @@ network_policies: path: "/usr/local/bin/aws".to_string(), ..Default::default() }], - middleware: vec![], }, ); let proto = ProtoSandboxPolicy { @@ -4685,7 +4401,6 @@ network_policies: path: "/usr/bin/node".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5636,7 +5351,6 @@ process: ..Default::default() }], binaries: vec![proposal_binary], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5692,7 +5406,6 @@ process: path: "/usr/bin/python".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5764,7 +5477,6 @@ process: path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -5996,7 +5708,6 @@ network_policies: path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -6701,7 +6412,6 @@ network_policies: path: "/usr/bin/python3".to_string(), ..Default::default() }], - ..Default::default() }, ); @@ -7099,7 +6809,7 @@ network_policies: } #[test] - fn middleware_chain_orders_global_policy_endpoint_once() { + fn middleware_chain_uses_matching_selector_declaration_order() { let data = r#" network_middlewares: - name: global-redactor @@ -7108,18 +6818,20 @@ network_middlewares: include: ["api.example.com"] - name: policy-redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] - name: endpoint-redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] network_policies: api: name: api - middleware: ["global-redactor", "policy-redactor"] endpoints: - host: api.example.com port: 443 protocol: rest enforcement: enforce - middleware: ["policy-redactor", "endpoint-redactor"] rules: - allow: { method: POST, path: "/v1/**" } binaries: @@ -7135,7 +6847,7 @@ network_policies: cmdline_paths: vec![], }; let (chain, _) = engine - .query_middleware_chain_with_generation(&input, "/v1/messages") + .query_middleware_chain_with_generation(&input) .unwrap(); let names: Vec<_> = chain.iter().map(|entry| entry.name.as_str()).collect(); assert_eq!( @@ -7144,63 +6856,9 @@ network_policies: ); } - #[test] - fn middleware_validation_rejects_ambiguous_duplicate_endpoint_middleware() { - let data = r#" -network_middlewares: - - name: first-redactor - middleware: openshell/secrets - - name: second-redactor - middleware: openshell/secrets -network_policies: - api: - name: api - endpoints: - - host: api.example.com - port: 443 - protocol: rest - enforcement: enforce - middleware: ["first-redactor"] - access: full - - host: api.example.com - port: 443 - protocol: rest - enforcement: enforce - middleware: ["second-redactor"] - access: full - binaries: - - { path: /usr/bin/curl } -"#; - let err = match OpaEngine::from_strings(TEST_POLICY, data) { - Ok(_) => panic!("equivalent endpoints with different middleware should be invalid"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("equivalent middleware selection keys"), - "{err:?}" - ); - } - #[test] fn middleware_policy_validation_rejects_bad_configs() { let cases = [ - ( - "missing reference", - r#" -network_middlewares: - - name: redactor - middleware: openshell/secrets -network_policies: - api: - middleware: ["missing"] - endpoints: - - { host: api.example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - "unknown middleware config 'missing'", - ), ( "invalid on_error", r#" @@ -7208,6 +6866,8 @@ network_middlewares: - name: redactor middleware: openshell/secrets on_error: maybe + endpoints: + include: ["api.example.com"] "#, "invalid on_error", ), @@ -7217,8 +6877,12 @@ network_middlewares: network_middlewares: - name: redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] - name: redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] "#, "duplicate middleware config 'redactor'", ), @@ -7228,22 +6892,45 @@ network_middlewares: network_middlewares: - name: sigv4 middleware: openshell/sigv4 + endpoints: + include: ["api.example.com"] "#, "unsupported built-in", ), ( - "tls skip attachment", + "missing selector", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets +"#, + "requires an endpoint selector", + ), + ( + "malformed selector", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets + endpoints: + include: ["api[.example.com"] +"#, + "invalid host pattern", + ), + ( + "tls skip selector", r#" network_middlewares: - name: redactor middleware: openshell/secrets + endpoints: + include: ["api.example.com"] network_policies: api: endpoints: - host: api.example.com port: 443 tls: skip - middleware: ["redactor"] binaries: - { path: /usr/bin/curl } "#, @@ -7263,6 +6950,29 @@ network_policies: } } + #[test] + fn from_proto_revalidates_middleware_policy() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy + .network_middlewares + .push(openshell_core::proto::NetworkMiddlewareConfig { + name: "redactor".into(), + middleware: "openshell/secrets".into(), + endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { + include: vec!["api[.example.com".into()], + exclude: Vec::new(), + }), + ..Default::default() + }); + + let error = OpaEngine::from_proto(&policy) + .err() + .expect("supervisor must reject invalid effective middleware policy") + .to_string(); + assert!(error.contains("policy validation failed"), "{error}"); + assert!(error.contains("invalid host pattern"), "{error}"); + } + #[test] fn l7_head_denied_when_only_post_allowed() { let engine = OpaEngine::from_strings( diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index fa8029c723..3cbc315026 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -1047,7 +1047,6 @@ fn network_rule_from_json( name: rule.name.unwrap_or_default(), endpoints, binaries, - middleware: Vec::new(), }) } @@ -1134,7 +1133,6 @@ fn network_endpoint_from_json( credential_signing: String::new(), signing_service: String::new(), signing_region: String::new(), - middleware: Vec::new(), }) } @@ -1831,7 +1829,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() }), ..Default::default() }; @@ -1856,7 +1853,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() } } @@ -1920,7 +1916,6 @@ mod tests { path: "/usr/bin/curl".to_string(), ..Default::default() }], - ..Default::default() })); }) }; diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index f8310fbdc3..afec98666d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4194,7 +4194,7 @@ async fn handle_forward_proxy( cmdline_paths: decision.cmdline_paths.clone(), }; let (chain, generation) = - opa_engine.query_middleware_chain_with_generation(&middleware_input, middleware_path)?; + opa_engine.query_middleware_chain_with_generation(&middleware_input)?; if generation != forward_generation_guard.captured_generation() { emit_l7_tunnel_close_after_policy_change( &host_lc, @@ -4224,10 +4224,11 @@ async fn handle_forward_proxy( &upstream_target, forward_request_bytes, )?; - forward_request_bytes = match crate::l7::relay::apply_middleware_chain( + forward_request_bytes = match crate::l7::relay::apply_middleware_chain_for_scheme( request, client, &l7_ctx, + &scheme, chain, &forward_generation_guard, ) diff --git a/proto/middleware.proto b/proto/middleware.proto index d5d2ad48d8..2944227d8a 100644 --- a/proto/middleware.proto +++ b/proto/middleware.proto @@ -25,6 +25,8 @@ message MiddlewareBinding { string id = 1; string operation = 2; string phase = 3; + // Maximum request or replacement body this binding can process. + uint64 max_body_bytes = 4; } message ValidateConfigRequest { diff --git a/proto/sandbox.proto b/proto/sandbox.proto index a73d762e5e..04cbd67767 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -63,11 +63,9 @@ message NetworkPolicyRule { repeated NetworkEndpoint endpoints = 2; // Allowed binary identities. repeated NetworkBinary binaries = 3; - // Ordered middleware configs applied to every endpoint in this policy. - repeated string middleware = 4; } -// A reusable middleware config referenced by network policies/endpoints. +// A reusable middleware config selected for admitted egress by host. message NetworkMiddlewareConfig { // Policy-local config name. string name = 1; @@ -77,7 +75,7 @@ message NetworkMiddlewareConfig { google.protobuf.Struct config = 3; // Failure behavior: "fail_closed" (default) or "fail_open". string on_error = 4; - // Optional global endpoint selector for this config. + // Host selector controlling which admitted destinations use this config. MiddlewareEndpointSelector endpoints = 5; } @@ -168,8 +166,6 @@ message NetworkEndpoint { uint32 json_rpc_max_body_bytes = 22; // MCP-only policy and inspection options. Only used when protocol is "mcp". McpOptions mcp = 23; - // Ordered middleware configs applied to this endpoint after policy-level middleware. - repeated string middleware = 24; } // MCP options are grouped so MCP-specific policy can grow without adding more From bb08a337952881a2d7a379ad87ef35a9ae90437a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 29 Jun 2026 15:28:14 -0700 Subject: [PATCH 07/59] feat(supervisor-middleware): support external services Signed-off-by: Piotr Mlocek --- Cargo.lock | 3 + architecture/gateway.md | 8 + architecture/sandbox.md | 10 + crates/openshell-core/src/grpc_client.rs | 41 +- crates/openshell-sandbox/Cargo.toml | 1 + crates/openshell-sandbox/src/lib.rs | 68 +- crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/config_file.rs | 55 ++ crates/openshell-server/src/grpc/policy.rs | 71 ++- crates/openshell-server/src/grpc/sandbox.rs | 1 + crates/openshell-server/src/lib.rs | 25 + crates/openshell-server/src/middleware.rs | 43 ++ .../Cargo.toml | 5 +- .../src/lib.rs | 601 ++++++++++++++++-- .../src/remote.rs | 91 +++ .../src/l7/relay.rs | 266 ++++---- .../openshell-supervisor-network/src/opa.rs | 34 +- .../openshell-supervisor-network/src/proxy.rs | 2 + docs/reference/gateway-config.mdx | 28 + docs/reference/policy-schema.mdx | 32 +- docs/sandboxes/policies.mdx | 36 +- proto/sandbox.proto | 16 + 22 files changed, 1246 insertions(+), 192 deletions(-) create mode 100644 crates/openshell-server/src/middleware.rs create mode 100644 crates/openshell-supervisor-middleware/src/remote.rs diff --git a/Cargo.lock b/Cargo.lock index 0634d12e9a..6819aab32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3833,6 +3833,7 @@ dependencies = [ "openshell-core", "openshell-ocsf", "openshell-policy", + "openshell-supervisor-middleware", "openshell-supervisor-network", "openshell-supervisor-process", "rustls", @@ -3887,6 +3888,7 @@ dependencies = [ "openshell-providers", "openshell-router", "openshell-server-macros", + "openshell-supervisor-middleware", "petname", "pin-project-lite", "prost", @@ -3938,6 +3940,7 @@ dependencies = [ "prost-types", "regex", "tokio", + "tokio-stream", "tonic", ] diff --git a/architecture/gateway.md b/architecture/gateway.md index d873b2a105..ba8437b7f3 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -271,6 +271,14 @@ config path. A gateway-global policy can override sandbox-scoped policy. The sandbox supervisor polls for config revisions and hot-reloads dynamic policy when the policy engine accepts the update. +External supervisor middleware registration is operator-owned gateway +configuration. At startup the gateway connects to each service, validates its +described bindings and operator body limit, and rejects duplicate binding IDs. +Before persisting a policy, the gateway asks each selected implementation to +validate its config. The effective sandbox config contains only the registered +services required by that policy; supervisors invoke those services directly on +the request path. + Provider credential expiry is enforced during gateway-to-sandbox credential resolution and again by the sandbox placeholder resolver. This keeps expired credentials from resolving even when a running sandbox still has retained diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 580d8f96d8..deec00f320 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -66,6 +66,14 @@ matchers; generic JSON-RPC rules match only the method. JSON-RPC responses and server-to-client MCP messages on response or SSE streams are relayed but are not currently parsed for policy enforcement. +For admitted HTTP requests, the proxy can run an ordered supervisor middleware +chain before credential injection. Host selectors choose the chain independently +of the network rule that admitted the request. Built-ins run in-process; +operator-registered external services are called directly from the supervisor +over the common middleware gRPC contract. The gateway validates external +service capabilities and policy-owned config before delivery. Supervisors keep +the last-known-good service registry when a live config reload fails. + `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: @@ -176,6 +184,8 @@ quickly. - If gateway config polling fails, the sandbox keeps its last-known-good policy. - If a live policy update is invalid, the supervisor rejects it and keeps the current policy. +- If an external middleware call fails, the selected config's `on_error` + behavior decides whether to deny the request or continue without that stage. - Existing raw byte streams are connection scoped. Dynamic policy changes apply to new connections or the next parsed HTTP request where the proxy can safely re-evaluate. diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 96158a1d10..57f72dca61 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -24,11 +24,11 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::proto::{ DenialSummary, GetDraftPolicyRequest, GetInferenceBundleRequest, GetInferenceBundleResponse, - GetSandboxConfigRequest, GetSandboxProviderEnvironmentRequest, IssueSandboxTokenRequest, - NetworkActivitySummary, PolicyChunk, PolicySource, PolicyStatus, RefreshSandboxTokenRequest, - ReportPolicyStatusRequest, SandboxPolicy as ProtoSandboxPolicy, SubmitPolicyAnalysisRequest, - SubmitPolicyAnalysisResponse, UpdateConfigRequest, inference_client::InferenceClient, - open_shell_client::OpenShellClient, + GetSandboxConfigRequest, GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + IssueSandboxTokenRequest, NetworkActivitySummary, PolicyChunk, PolicySource, PolicyStatus, + RefreshSandboxTokenRequest, ReportPolicyStatusRequest, SandboxPolicy as ProtoSandboxPolicy, + SubmitPolicyAnalysisRequest, SubmitPolicyAnalysisResponse, UpdateConfigRequest, + inference_client::InferenceClient, open_shell_client::OpenShellClient, }; use crate::sandbox_env; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -573,19 +573,36 @@ pub async fn fetch_policy(endpoint: &str, sandbox_id: &str) -> Result Result { + debug!(endpoint = %endpoint, sandbox_id = %sandbox_id, "Connecting to OpenShell server"); + let mut client = connect(endpoint).await?; + fetch_sandbox_config_with_client(&mut client, sandbox_id).await +} + +async fn fetch_sandbox_config_with_client( client: &mut OpenShellClient, sandbox_id: &str, -) -> Result> { - let response = client +) -> Result { + client .get_sandbox_config(GetSandboxConfigRequest { sandbox_id: sandbox_id.to_string(), }) .await - .into_diagnostic()?; + .map(tonic::Response::into_inner) + .into_diagnostic() +} - let inner = response.into_inner(); +/// Fetch sandbox policy using an existing client connection. +async fn fetch_policy_with_client( + client: &mut OpenShellClient, + sandbox_id: &str, +) -> Result> { + let inner = fetch_sandbox_config_with_client(client, sandbox_id).await?; // version 0 with no policy means the sandbox was created without one. if inner.version == 0 && inner.policy.is_none() { @@ -711,6 +728,7 @@ pub struct SettingsPollResult { /// When `policy_source` is `Global`, the version of the global policy revision. pub global_policy_version: u32, pub provider_env_revision: u64, + pub external_middleware: Vec, } pub struct ProviderEnvironmentResult { @@ -755,6 +773,7 @@ impl CachedOpenShellClient { settings: inner.settings, global_policy_version: inner.global_policy_version, provider_env_revision: inner.provider_env_revision, + external_middleware: inner.external_middleware, }) } diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 086dbe02c3..d3c3e7108a 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -19,6 +19,7 @@ openshell-core = { path = "../openshell-core", default-features = false } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network" } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } openshell-supervisor-process = { path = "../openshell-supervisor-process" } # Async runtime diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 53b1eba582..30da1c292b 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1409,12 +1409,12 @@ async fn load_policy( endpoint = %endpoint, "Fetching sandbox policy via gRPC" ); - let proto_policy = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_policy(endpoint, id) + let mut sandbox_config = grpc_retry("Policy fetch", || { + openshell_core::grpc_client::fetch_sandbox_config(endpoint, id) }) .await?; - let mut proto_policy = if let Some(p) = proto_policy { + let mut proto_policy = if let Some(p) = sandbox_config.policy.take() { p } else { // No policy configured on the server. Discover from disk or @@ -1442,7 +1442,7 @@ async fn load_policy( // Sync and re-fetch over a single connection to avoid extra // TLS handshakes. - grpc_retry("Policy discovery sync", || { + let synced = grpc_retry("Policy discovery sync", || { openshell_core::grpc_client::discover_and_sync_policy( endpoint, id, @@ -1450,7 +1450,12 @@ async fn load_policy( &discovered, ) }) - .await? + .await?; + sandbox_config = grpc_retry("Policy refetch after discovery", || { + openshell_core::grpc_client::fetch_sandbox_config(endpoint, id) + }) + .await?; + sandbox_config.policy.take().unwrap_or(synced) }; // Ensure baseline filesystem paths are present for proxy-mode @@ -1476,7 +1481,14 @@ async fn load_policy( // container hasn't started yet. After the entrypoint spawns, the // engine is rebuilt with the real PID for symlink resolution. info!("Creating OPA engine from proto policy data"); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto_policy)?)); + let engine = OpaEngine::from_proto(&proto_policy)?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_external( + sandbox_config.external_middleware, + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; + let opa_engine = Some(Arc::new(engine)); let policy = SandboxPolicy::try_from(proto_policy.clone())?; return Ok((policy, opa_engine, Some(proto_policy))); @@ -1626,6 +1638,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let mut current_config_revision: u64 = 0; let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; let mut current_policy_hash = String::new(); + let mut current_external_middleware = Vec::new(); let mut current_settings: std::collections::HashMap< String, openshell_core::proto::EffectiveSetting, @@ -1637,6 +1650,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); current_config_revision = result.config_revision; current_policy_hash = result.policy_hash.clone(); + current_external_middleware = result.external_middleware; current_settings = result.settings; debug!( config_revision = current_config_revision, @@ -1666,6 +1680,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } let policy_changed = result.policy_hash != current_policy_hash; + let middleware_changed = result.external_middleware != current_external_middleware; // Log which settings changed. log_setting_changes(¤t_settings, &result.settings); @@ -1724,6 +1739,47 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } } + if middleware_changed { + match openshell_supervisor_middleware::MiddlewareRegistry::connect_external( + result.external_middleware.clone(), + ) + .await + .and_then(|registry| ctx.opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + current_external_middleware = result.external_middleware.clone(); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "external_middleware_count", + serde_json::json!(current_external_middleware.len()) + ) + .message(format!( + "External middleware registry reloaded [service_count:{}]", + current_external_middleware.len() + )) + .build() + ); + } + Err(error) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "External middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + continue; + } + } + } + // Only reload OPA when the policy payload actually changed. if policy_changed { let Some(policy) = result.policy.as_ref() else { diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index b5c9b34d71..fafc72ba76 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -26,6 +26,7 @@ openshell-prover = { path = "../openshell-prover" } openshell-providers = { path = "../openshell-providers" } openshell-router = { path = "../openshell-router" } openshell-server-macros = { path = "../openshell-server-macros" } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } # Kubernetes client (used by the `generate-certs` subcommand) kube = { workspace = true } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index b65b5f3b09..13c7e9ebb2 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -25,6 +25,7 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use openshell_core::config::ComputeDriverKind; +use openshell_core::proto::ExternalMiddlewareService; use openshell_core::{GatewayAuthConfig, GatewayJwtConfig, MtlsAuthConfig, OidcConfig, TlsConfig}; use serde::{Deserialize, Serialize}; @@ -151,6 +152,12 @@ pub struct GatewayFileSection { #[serde(default)] pub gateway_jwt: Option, + // ── Supervisor middleware ───────────────────────────────────────────── + /// Statically registered external middleware services. Registration is + /// operator-owned and changes require a gateway restart. + #[serde(default)] + pub middleware: Vec, + // ── Disallowed-in-file fields ──────────────────────────────────────── // // Captured so we can produce a friendly "set this via env/CLI instead" @@ -160,6 +167,32 @@ pub struct GatewayFileSection { pub database_url: Option, } +/// One `[[openshell.gateway.middleware]]` external middleware registration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MiddlewareServiceFileConfig { + /// Operator-facing name used for diagnostics. + pub name: String, + /// Plaintext gRPC endpoint reachable by the gateway and supervisors. + pub endpoint: String, + /// Required explicit opt-in to the local-development-only insecure mode. + #[serde(default)] + pub allow_insecure: bool, + /// Operator-owned body limit for every binding exposed by this service. + pub max_body_bytes: u64, +} + +impl From<&MiddlewareServiceFileConfig> for ExternalMiddlewareService { + fn from(config: &MiddlewareServiceFileConfig) -> Self { + Self { + name: config.name.clone(), + endpoint: config.endpoint.clone(), + allow_insecure: config.allow_insecure, + max_body_bytes: config.max_body_bytes, + } + } +} + #[derive(Debug, thiserror::Error)] pub enum ConfigFileError { #[error("failed to read gateway config file '{}': {source}", path.display())] @@ -401,6 +434,28 @@ allow_unauthenticated_users = true assert!(auth.allow_unauthenticated_users); } + #[test] + fn parses_external_middleware_registration() { + let toml = r#" +[[openshell.gateway.middleware]] +name = "local-guard" +endpoint = "http://127.0.0.1:50051" +allow_insecure = true +max_body_bytes = 262144 +"#; + let tmp = write_tmp(toml); + let file = load(tmp.path()).expect("valid middleware registration parses"); + assert_eq!( + file.openshell.gateway.middleware, + vec![MiddlewareServiceFileConfig { + name: "local-guard".into(), + endpoint: "http://127.0.0.1:50051".into(), + allow_insecure: true, + max_body_bytes: 262_144, + }] + ); + } + #[test] fn rejects_database_url_in_file() { let toml = r#" diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index d3bc213ba0..c29de71ffd 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1218,8 +1218,27 @@ pub(super) async fn handle_get_sandbox_config( } } + if let Some(policy) = policy.as_ref() { + state + .middleware_registry + .ensure_policy_bindings_registered(policy) + .map_err(|error| { + Status::failed_precondition(format!( + "effective policy middleware registration is invalid: {error}" + )) + })?; + } + let settings = merge_effective_settings(&global_settings, &sandbox_settings)?; - let config_revision = compute_config_revision(policy.as_ref(), &settings, policy_source); + let external_middleware = state + .middleware_registry + .required_external_services(policy.as_ref()); + let config_revision = compute_config_revision( + policy.as_ref(), + &settings, + policy_source, + &external_middleware, + ); let provider_env_revision = compute_provider_env_revision(state.store.as_ref(), &sandbox_provider_names).await?; @@ -1232,6 +1251,7 @@ pub(super) async fn handle_get_sandbox_config( policy_source: policy_source.into(), global_policy_version, provider_env_revision, + external_middleware, })) } @@ -1510,6 +1530,8 @@ async fn handle_update_config_inner( openshell_policy::ensure_sandbox_process_identity(&mut new_policy); validate_no_reserved_provider_policy_keys(&new_policy)?; validate_policy_safety(&new_policy)?; + crate::middleware::validate_policy(state.middleware_registry.as_ref(), &new_policy) + .await?; let payload = new_policy.encode_to_vec(); let hash = deterministic_policy_hash(&new_policy); @@ -1827,9 +1849,11 @@ async fn handle_update_config_inner( validate_no_reserved_provider_policy_keys(&new_policy)?; } + validate_policy_safety(&new_policy)?; + crate::middleware::validate_policy(state.middleware_registry.as_ref(), &new_policy).await?; + if let Some(baseline_policy) = spec.policy.as_ref() { validate_static_fields_unchanged(baseline_policy, &new_policy)?; - validate_policy_safety(&new_policy)?; } else { // Backfill spec.policy using CAS (first-time policy discovery) let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; @@ -3120,6 +3144,7 @@ fn compute_config_revision( policy: Option<&ProtoSandboxPolicy>, settings: &HashMap, policy_source: PolicySource, + external_middleware: &[openshell_core::proto::ExternalMiddlewareService], ) -> u64 { let mut hasher = Sha256::new(); hasher.update((policy_source as i32).to_le_bytes()); @@ -3152,6 +3177,11 @@ fn compute_config_revision( } } } + let mut middleware = external_middleware.iter().collect::>(); + middleware.sort_by(|left, right| left.name.cmp(&right.name)); + for service in middleware { + hasher.update(service.encode_to_vec()); + } let digest = hasher.finalize(); let mut bytes = [0_u8; 8]; @@ -8989,7 +9019,7 @@ mod tests { }, ); - let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox); + let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); settings.insert( "mode".to_string(), EffectiveSetting { @@ -8999,7 +9029,7 @@ mod tests { scope: SettingScope::Sandbox.into(), }, ); - let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox); + let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); assert_ne!(rev_a, rev_b); } @@ -9264,8 +9294,8 @@ mod tests { }, ); - let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox); - let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox); + let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); + let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); assert_eq!(rev_a, rev_b); } @@ -9281,8 +9311,8 @@ mod tests { }; let settings = HashMap::new(); - let rev_a = compute_config_revision(Some(&policy_a), &settings, PolicySource::Sandbox); - let rev_b = compute_config_revision(Some(&policy_b), &settings, PolicySource::Sandbox); + let rev_a = compute_config_revision(Some(&policy_a), &settings, PolicySource::Sandbox, &[]); + let rev_b = compute_config_revision(Some(&policy_b), &settings, PolicySource::Sandbox, &[]); assert_ne!(rev_a, rev_b); } @@ -9291,11 +9321,28 @@ mod tests { let policy = ProtoSandboxPolicy::default(); let settings = HashMap::new(); - let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox); - let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Global); + let rev_a = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); + let rev_b = compute_config_revision(Some(&policy), &settings, PolicySource::Global, &[]); assert_ne!(rev_a, rev_b); } + #[test] + fn config_revision_changes_when_external_middleware_changes() { + let policy = ProtoSandboxPolicy::default(); + let settings = HashMap::new(); + let service = openshell_core::proto::ExternalMiddlewareService { + name: "local-guard".into(), + endpoint: "http://127.0.0.1:50051".into(), + allow_insecure: true, + max_body_bytes: 1024, + }; + + let without = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); + let with = + compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[service]); + assert_ne!(without, with); + } + #[test] fn config_revision_without_policy_still_hashes_settings() { let mut settings = HashMap::new(); @@ -9309,7 +9356,7 @@ mod tests { }, ); - let rev_a = compute_config_revision(None, &settings, PolicySource::Sandbox); + let rev_a = compute_config_revision(None, &settings, PolicySource::Sandbox, &[]); settings.insert( "log_level".to_string(), @@ -9321,7 +9368,7 @@ mod tests { }, ); - let rev_b = compute_config_revision(None, &settings, PolicySource::Sandbox); + let rev_b = compute_config_revision(None, &settings, PolicySource::Sandbox, &[]); assert_ne!(rev_a, rev_b); } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 04d5a4ed52..203cd7dbe0 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -164,6 +164,7 @@ async fn handle_create_sandbox_inner( openshell_policy::ensure_sandbox_process_identity(policy); validate_no_reserved_provider_policy_keys(policy)?; validate_policy_safety(policy)?; + crate::middleware::validate_policy(state.middleware_registry.as_ref(), policy).await?; } let id = uuid::Uuid::new_v4().to_string(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6462ccbbf3..bca8abe2e7 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -32,6 +32,7 @@ mod defaults; mod grpc; mod http; mod inference; +mod middleware; mod multiplex; mod persistence; pub(crate) mod policy_store; @@ -53,6 +54,8 @@ mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use openshell_supervisor_middleware::MiddlewareRegistry; +use serde::Deserialize; use std::collections::HashMap; use std::io::ErrorKind; use std::net::SocketAddr; @@ -126,6 +129,9 @@ pub struct ServerState { /// query session state to surface supervisor readiness. pub supervisor_sessions: Arc, + /// Validated built-in and operator-registered supervisor middleware. + pub middleware_registry: Arc, + /// OIDC JWKS cache for JWT validation. `None` when OIDC is not configured. pub oidc_cache: Option>, @@ -192,6 +198,7 @@ impl ServerState { ssh_connections_by_sandbox: Mutex::new(HashMap::new()), settings_mutex: tokio::sync::Mutex::new(()), supervisor_sessions, + middleware_registry: Arc::new(MiddlewareRegistry::default()), oidc_cache, sandbox_jwt_issuer: None, sandbox_jwt_authenticator: None, @@ -223,6 +230,23 @@ pub(crate) async fn run_server( return Err(Error::config("database_url is required")); } + let middleware_registrations = config_file + .as_ref() + .map(|file| { + file.openshell + .gateway + .middleware + .iter() + .map(Into::into) + .collect() + }) + .unwrap_or_default(); + let middleware_registry = Arc::new( + MiddlewareRegistry::connect_external(middleware_registrations) + .await + .map_err(|error| Error::config(format!("middleware registration failed: {error}")))?, + ); + let store = Arc::new(Store::connect(database_url).await?); let oidc_cache = if let Some(ref oidc) = config.oidc { @@ -273,6 +297,7 @@ pub(crate) async fn run_server( supervisor_sessions, oidc_cache, ); + state.middleware_registry = middleware_registry; // Load the gateway-minted sandbox JWT signing key when configured. // Optional so single-driver dev deployments without certgen continue diff --git a/crates/openshell-server/src/middleware.rs b/crates/openshell-server/src/middleware.rs new file mode 100644 index 0000000000..4c94f021a3 --- /dev/null +++ b/crates/openshell-server/src/middleware.rs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_core::proto::SandboxPolicy; +use openshell_supervisor_middleware::MiddlewareRegistry; +use tonic::Status; + +/// Validate implementation-owned middleware config before accepting a policy. +pub async fn validate_policy( + registry: &MiddlewareRegistry, + policy: &SandboxPolicy, +) -> Result<(), Status> { + registry + .validate_policy_configs(policy) + .await + .map_err(|error| { + Status::invalid_argument(format!("policy middleware validation failed: {error}")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::NetworkMiddlewareConfig; + + #[tokio::test] + async fn unregistered_external_binding_is_rejected_before_admission() { + let policy = SandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "guard".into(), + middleware: "example/content-guard".into(), + ..Default::default() + }], + ..Default::default() + }; + + let error = validate_policy(&MiddlewareRegistry::default(), &policy) + .await + .expect_err("unregistered binding must fail"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + assert!(error.message().contains("not registered")); + } +} diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index 4ae355894d..e5e53618df 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -17,7 +17,10 @@ miette = { workspace = true } prost-types = { workspace = true } regex = { workspace = true } tokio = { workspace = true } -tonic = { workspace = true } +tonic = { workspace = true, features = ["channel", "server"] } + +[dev-dependencies] +tokio-stream = { workspace = true, features = ["net"] } [lints] workspace = true diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index a9cb524344..828179d186 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! In-process supervisor middleware chain execution. +//! Supervisor middleware registration and chain execution. mod builtins; +mod remote; mod service; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -14,14 +15,17 @@ pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, - MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, + Decision, ExternalMiddlewareService, Finding, HttpRequestEvaluation, HttpRequestTarget, + MiddlewareBinding, MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, + ValidateConfigRequest, }; use tokio::sync::OnceCell; use tonic::Request; pub const API_VERSION: &str = "openshell.middleware.v1"; pub const BUILTIN_SECRETS: &str = builtins::secrets::BINDING_ID; +const HTTP_REQUEST_OPERATION: &str = "HttpRequest"; +const PRE_CREDENTIALS_PHASE: &str = "pre_credentials"; /// Validate the configuration for an in-process middleware implementation. /// @@ -82,9 +86,10 @@ impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { /// A policy-selected middleware config joined with metadata reported by its /// service's `Describe` call. A missing binding is retained so `on_error` can /// decide whether the request fails open or closed. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct DescribedChainEntry { entry: ChainEntry, + service: Option>, binding: Option, max_body_bytes: usize, } @@ -182,82 +187,380 @@ fn apply_on_error( #[derive(Clone)] pub struct ChainRunner { - state: Arc, + registry: Arc, } struct MiddlewareServiceState { service: Arc, manifest: OnceCell, + operator_max_body_bytes: Option, } static IN_PROCESS_SERVICE: LazyLock> = LazyLock::new(|| { Arc::new(MiddlewareServiceState { service: Arc::new(InProcessMiddlewareService), manifest: OnceCell::new(), + operator_max_body_bytes: None, }) }); -impl Default for ChainRunner { +/// Validated middleware services available to a gateway or one supervisor. +/// +/// The registry always contains the in-process built-ins. External services +/// are connected and described before construction succeeds, so callers never +/// observe a partially registered service set. +#[derive(Clone)] +pub struct MiddlewareRegistry { + services: Arc>>, + external: Arc>, +} + +impl std::fmt::Debug for MiddlewareRegistry { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MiddlewareRegistry") + .field("service_count", &self.services.len()) + .field("external_count", &self.external.len()) + .finish() + } +} + +#[derive(Clone)] +struct RegisteredExternalService { + registration: ExternalMiddlewareService, + binding_ids: Vec, +} + +impl Default for MiddlewareRegistry { fn default() -> Self { Self { - state: Arc::clone(&IN_PROCESS_SERVICE), + services: Arc::new(vec![Arc::clone(&IN_PROCESS_SERVICE)]), + external: Arc::new(Vec::new()), } } } +fn validate_registration(registration: &ExternalMiddlewareService) -> Result<()> { + if registration.name.trim().is_empty() { + return Err(miette!( + "external middleware registration name cannot be empty" + )); + } + if !registration.allow_insecure { + return Err(miette!( + "middleware registration '{}' must set allow_insecure = true; TLS is not supported in V1", + registration.name + )); + } + if !registration.endpoint.starts_with("http://") { + return Err(miette!( + "middleware registration '{}' endpoint must use http:// in the local-development-only V1", + registration.name + )); + } + if registration.max_body_bytes == 0 { + return Err(miette!( + "middleware registration '{}' max_body_bytes must be greater than zero", + registration.name + )); + } + Ok(()) +} + +fn validate_external_manifest( + registration: &ExternalMiddlewareService, + manifest: &MiddlewareManifest, + operator_max_body_bytes: usize, + known_binding_ids: &mut HashSet, +) -> Result> { + if manifest.api_version != API_VERSION { + return Err(miette!( + "middleware registration '{}' reports unsupported API version '{}'", + registration.name, + manifest.api_version + )); + } + if manifest.bindings.is_empty() { + return Err(miette!( + "middleware registration '{}' describes no bindings", + registration.name + )); + } + + let mut described_ids = Vec::with_capacity(manifest.bindings.len()); + for binding in &manifest.bindings { + if binding.id.trim().is_empty() { + return Err(miette!( + "middleware registration '{}' describes an empty binding id", + registration.name + )); + } + if binding.id.starts_with("openshell/") { + return Err(miette!( + "external middleware registration '{}' cannot claim reserved binding '{}'", + registration.name, + binding.id + )); + } + if binding.operation != HTTP_REQUEST_OPERATION || binding.phase != PRE_CREDENTIALS_PHASE { + return Err(miette!( + "middleware binding '{}' must support {HTTP_REQUEST_OPERATION}/{PRE_CREDENTIALS_PHASE}", + binding.id + )); + } + let advertised = usize::try_from(binding.max_body_bytes).map_err(|_| { + miette!( + "middleware binding '{}' reports a body limit too large for this platform", + binding.id + ) + })?; + if advertised == 0 { + return Err(miette!( + "middleware binding '{}' must advertise a non-zero body limit", + binding.id + )); + } + if operator_max_body_bytes > advertised { + return Err(miette!( + "middleware registration '{}' max_body_bytes ({operator_max_body_bytes}) exceeds binding '{}' capability ({advertised})", + registration.name, + binding.id + )); + } + if !known_binding_ids.insert(binding.id.clone()) { + return Err(miette!( + "middleware binding '{}' is described by more than one service", + binding.id + )); + } + described_ids.push(binding.id.clone()); + } + Ok(described_ids) +} + +impl MiddlewareRegistry { + /// Connect and validate every external service registration. + pub async fn connect_external(registrations: Vec) -> Result { + let mut services = vec![Arc::clone(&IN_PROCESS_SERVICE)]; + let mut external = Vec::with_capacity(registrations.len()); + let mut registration_names = HashSet::new(); + let mut binding_ids = HashSet::from([BUILTIN_SECRETS.to_string()]); + + for registration in registrations { + validate_registration(®istration)?; + if !registration_names.insert(registration.name.clone()) { + return Err(miette!( + "duplicate external middleware registration name '{}'", + registration.name + )); + } + + let operator_max_body_bytes = + usize::try_from(registration.max_body_bytes).map_err(|_| { + miette!( + "middleware registration '{}' body limit is too large for this platform", + registration.name + ) + })?; + let service = Arc::new( + remote::RemoteMiddlewareService::connect( + ®istration.name, + ®istration.endpoint, + ) + .await?, + ); + let manifest = service + .describe(Request::new(())) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware registration '{}' Describe failed: {}", + registration.name, + safe_reason(&error.to_string()) + ) + })?; + let described_ids = validate_external_manifest( + ®istration, + &manifest, + operator_max_body_bytes, + &mut binding_ids, + )?; + let manifest_cell = OnceCell::new(); + manifest_cell + .set(manifest) + .map_err(|_| miette!("middleware manifest cache initialized twice"))?; + services.push(Arc::new(MiddlewareServiceState { + service, + manifest: manifest_cell, + operator_max_body_bytes: Some(operator_max_body_bytes), + })); + external.push(RegisteredExternalService { + registration, + binding_ids: described_ids, + }); + } + + Ok(Self { + services: Arc::new(services), + external: Arc::new(external), + }) + } + + /// Validate implementation-owned configuration for every middleware entry. + pub async fn validate_policy_configs(&self, policy: &SandboxPolicy) -> Result<()> { + let runner = ChainRunner::from_registry(self.clone()); + for config in &policy.network_middlewares { + runner + .validate_config( + &config.middleware, + config.config.clone().unwrap_or_default(), + ) + .await + .map_err(|error| { + miette!( + "middleware config '{}' is invalid: {}", + config.name, + safe_reason(&error.to_string()) + ) + })?; + } + Ok(()) + } + + /// Check that every policy binding still belongs to the current static + /// registry without making a network call. + pub fn ensure_policy_bindings_registered(&self, policy: &SandboxPolicy) -> Result<()> { + for config in &policy.network_middlewares { + let registered = config.middleware == BUILTIN_SECRETS + || self.external.iter().any(|service| { + service + .binding_ids + .iter() + .any(|binding| binding == &config.middleware) + }); + if !registered { + return Err(miette!( + "middleware binding '{}' used by config '{}' is not registered", + config.middleware, + config.name + )); + } + } + Ok(()) + } + + /// Return only external services referenced by the effective policy. + pub fn required_external_services( + &self, + policy: Option<&SandboxPolicy>, + ) -> Vec { + let Some(policy) = policy else { + return Vec::new(); + }; + let selected: HashSet<&str> = policy + .network_middlewares + .iter() + .map(|config| config.middleware.as_str()) + .collect(); + self.external + .iter() + .filter(|service| { + service + .binding_ids + .iter() + .any(|binding| selected.contains(binding.as_str())) + }) + .map(|service| service.registration.clone()) + .collect() + } +} + +impl Default for ChainRunner { + fn default() -> Self { + Self::from_registry(MiddlewareRegistry::default()) + } +} + impl ChainRunner { pub fn new(service: Arc) -> Self { Self { - state: Arc::new(MiddlewareServiceState { - service, - manifest: OnceCell::new(), + registry: Arc::new(MiddlewareRegistry { + services: Arc::new(vec![Arc::new(MiddlewareServiceState { + service, + manifest: OnceCell::new(), + operator_max_body_bytes: None, + })]), + external: Arc::new(Vec::new()), }), } } - async fn manifest(&self) -> Result<&MiddlewareManifest> { - self.state - .manifest - .get_or_try_init(|| async { - self.state - .service - .describe(Request::new(())) - .await - .map(tonic::Response::into_inner) - .map_err(|error| { - miette!( - "middleware Describe failed: {}", - safe_reason(&error.to_string()) - ) - }) - }) - .await + pub fn from_registry(registry: MiddlewareRegistry) -> Self { + Self { + registry: Arc::new(registry), + } + } + + async fn manifests(&self) -> Result, MiddlewareManifest)>> { + let mut manifests = Vec::with_capacity(self.registry.services.len()); + for state in self.registry.services.iter() { + let manifest = state + .manifest + .get_or_try_init(|| async { + state + .service + .describe(Request::new(())) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware Describe failed: {}", + safe_reason(&error.to_string()) + ) + }) + }) + .await?; + manifests.push((Arc::clone(state), manifest.clone())); + } + Ok(manifests) } pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result> { - let manifest = self.manifest().await?; + let manifests = self.manifests().await?; entries .iter() .map(|entry| { - let binding = manifest - .bindings - .iter() - .find(|binding| binding.id == entry.implementation) - .cloned(); + let described = manifests.iter().find_map(|(state, manifest)| { + manifest + .bindings + .iter() + .find(|binding| binding.id == entry.implementation) + .cloned() + .map(|binding| (Arc::clone(state), binding)) + }); + let (service, binding) = described.map_or((None, None), |(service, binding)| { + (Some(service), Some(binding)) + }); let max_body_bytes = binding .as_ref() .map(|binding| { - usize::try_from(binding.max_body_bytes).map_err(|_| { + let advertised = usize::try_from(binding.max_body_bytes).map_err(|_| { miette!( "middleware binding '{}' reports a body limit too large for this platform", binding.id ) - }) + })?; + Ok::<_, miette::Report>(service + .as_ref() + .and_then(|state| state.operator_max_body_bytes) + .unwrap_or(advertised)) }) .transpose()? .unwrap_or(0); Ok(DescribedChainEntry { entry: entry.clone(), + service, binding, max_body_bytes, }) @@ -265,6 +568,44 @@ impl ChainRunner { .collect() } + pub async fn validate_config( + &self, + implementation: &str, + config: prost_types::Struct, + ) -> Result<()> { + let manifests = self.manifests().await?; + let Some((state, _)) = manifests.iter().find(|(_, manifest)| { + manifest + .bindings + .iter() + .any(|binding| binding.id == implementation) + }) else { + return Err(miette!( + "middleware binding '{implementation}' is not registered" + )); + }; + let response = state + .service + .validate_config(Request::new(ValidateConfigRequest { + api_version: API_VERSION.into(), + binding_id: implementation.into(), + config: Some(config), + })) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware ValidateConfig failed: {}", + safe_reason(&error.to_string()) + ) + })?; + if response.valid { + Ok(()) + } else { + Err(miette!("{}", safe_reason(&response.reason))) + } + } + pub async fn evaluate( &self, entries: &[ChainEntry], @@ -320,8 +661,10 @@ impl ChainRunner { } } let evaluation = build_evaluation(entry, binding, &input, &headers, &body); - let result = match self - .state + let Some(service) = entry.service.as_ref() else { + unreachable!("described binding always has a service") + }; + let result = match service .service .evaluate_http_request(Request::new(evaluation)) .await @@ -545,7 +888,10 @@ pub(crate) fn safe_reason(reason: &str) -> String { #[cfg(test)] mod tests { use super::*; - use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; + use openshell_core::proto::middleware::v1::supervisor_middleware_server::{ + SupervisorMiddleware, SupervisorMiddlewareServer, + }; + use tokio_stream::wrappers::TcpListenerStream; fn entry(name: &str, on_error: OnError) -> ChainEntry { ChainEntry { @@ -736,7 +1082,7 @@ mod tests { async fn validate_config( &self, - _request: Request, + _request: Request, ) -> std::result::Result< tonic::Response, tonic::Status, @@ -780,6 +1126,51 @@ mod tests { } } + fn external_registration(max_body_bytes: u64) -> ExternalMiddlewareService { + ExternalMiddlewareService { + name: "local-guard-service".into(), + endpoint: "http://127.0.0.1:50051".into(), + allow_insecure: true, + max_body_bytes, + } + } + + async fn registry_with_external( + service: Arc, + registration: ExternalMiddlewareService, + ) -> MiddlewareRegistry { + let manifest = service + .describe(Request::new(())) + .await + .expect("describe test service") + .into_inner(); + let operator_max_body_bytes = usize::try_from(registration.max_body_bytes).unwrap(); + let mut known = HashSet::from([BUILTIN_SECRETS.to_string()]); + let binding_ids = validate_external_manifest( + ®istration, + &manifest, + operator_max_body_bytes, + &mut known, + ) + .expect("valid external manifest"); + let manifest_cell = OnceCell::new(); + manifest_cell.set(manifest).expect("manifest cache"); + MiddlewareRegistry { + services: Arc::new(vec![ + Arc::clone(&IN_PROCESS_SERVICE), + Arc::new(MiddlewareServiceState { + service, + manifest: manifest_cell, + operator_max_body_bytes: Some(operator_max_body_bytes), + }), + ]), + external: Arc::new(vec![RegisteredExternalService { + registration, + binding_ids, + }]), + } + } + #[tokio::test] async fn descriptors_are_resolved_from_any_middleware_service() { let runner = ChainRunner::new(Arc::new(ScriptedService { @@ -815,6 +1206,138 @@ mod tests { assert!(outcome.allowed); } + #[tokio::test] + async fn mixed_builtin_and_external_chain_uses_operator_limit() { + let external = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: allow_result(), + }); + let registry = registry_with_external(external, external_registration(1024)).await; + let runner = ChainRunner::from_registry(registry); + let external_entry = ChainEntry { + name: "external".into(), + implementation: "example/content-guard".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let entries = [entry("builtin", OnError::FailClosed), external_entry]; + + let described = runner + .describe_chain(&entries) + .await + .expect("describe chain"); + assert_eq!(described[0].max_body_bytes(), 256 * 1024); + assert_eq!(described[1].max_body_bytes(), 1024); + + let outcome = runner + .evaluate_described(&described, input(r#"password="top-secret""#)) + .await + .expect("evaluate mixed chain"); + assert!(outcome.allowed); + assert_eq!(outcome.applied.len(), 2); + assert_eq!( + String::from_utf8(outcome.body).expect("utf8"), + r#"password="[REDACTED]""# + ); + } + + #[test] + fn external_manifest_rejects_operator_limit_above_capability() { + let registration = external_registration(4097); + let manifest = MiddlewareManifest { + api_version: API_VERSION.into(), + name: "example/service".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/content-guard".into(), + operation: HTTP_REQUEST_OPERATION.into(), + phase: PRE_CREDENTIALS_PHASE.into(), + max_body_bytes: 4096, + }], + }; + let error = validate_external_manifest( + ®istration, + &manifest, + 4097, + &mut HashSet::from([BUILTIN_SECRETS.to_string()]), + ) + .expect_err("operator limit must fit capability"); + assert!(error.to_string().contains("exceeds")); + } + + #[test] + fn external_registration_requires_explicit_insecure_opt_in() { + let mut registration = external_registration(4096); + registration.allow_insecure = false; + let error = validate_registration(®istration).expect_err("opt-in required"); + assert!(error.to_string().contains("allow_insecure")); + } + + #[tokio::test] + async fn external_registry_invokes_remote_service_over_grpc() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test middleware"); + let address = listener.local_addr().expect("test middleware address"); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tonic::transport::Server::builder() + .add_service(SupervisorMiddlewareServer::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: allow_result(), + })) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = shutdown_rx.await; + }); + let server_task = tokio::spawn(server); + + let mut registration = external_registration(1024); + registration.endpoint = format!("http://{address}"); + let registry = MiddlewareRegistry::connect_external(vec![registration.clone()]) + .await + .expect("connect external middleware"); + let policy = SandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "guard".into(), + middleware: "example/content-guard".into(), + config: Some(prost_types::Struct::default()), + on_error: "fail_closed".into(), + endpoints: None, + }], + ..Default::default() + }; + + registry + .validate_policy_configs(&policy) + .await + .expect("remote config validates"); + assert_eq!( + registry.required_external_services(Some(&policy)), + vec![registration] + ); + + let outcome = ChainRunner::from_registry(registry) + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }], + input("hello"), + ) + .await + .expect("remote evaluation"); + assert!(outcome.allowed); + + let _ = shutdown_tx.send(()); + server_task + .await + .expect("join test middleware") + .expect("serve"); + } + #[tokio::test] async fn deny_decision_short_circuits_chain() { let runner = ChainRunner::new(Arc::new(scripted_service( diff --git a/crates/openshell-supervisor-middleware/src/remote.rs b/crates/openshell-supervisor-middleware/src/remote.rs new file mode 100644 index 0000000000..dd147788b9 --- /dev/null +++ b/crates/openshell-supervisor-middleware/src/remote.rs @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use miette::{IntoDiagnostic, Result, WrapErr}; +use openshell_core::proto::middleware::v1::supervisor_middleware_client::SupervisorMiddlewareClient; +use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; +use openshell_core::proto::{ + HttpRequestEvaluation, HttpRequestResult, MiddlewareManifest, ValidateConfigRequest, + ValidateConfigResponse, +}; +use tonic::transport::{Channel, Endpoint}; +use tonic::{Request, Response, Status}; + +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const RPC_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone)] +pub struct RemoteMiddlewareService { + registration_name: String, + client: SupervisorMiddlewareClient, +} + +impl RemoteMiddlewareService { + pub async fn connect(registration_name: &str, endpoint: &str) -> Result { + let channel = Endpoint::from_shared(endpoint.to_string()) + .into_diagnostic() + .wrap_err_with(|| { + format!("middleware registration '{registration_name}' has an invalid endpoint") + })? + .connect_timeout(CONNECT_TIMEOUT) + .connect() + .await + .into_diagnostic() + .wrap_err_with(|| { + format!( + "middleware registration '{registration_name}' could not connect to {endpoint}" + ) + })?; + Ok(Self { + registration_name: registration_name.to_string(), + client: SupervisorMiddlewareClient::new(channel), + }) + } + + async fn with_timeout( + &self, + operation: &'static str, + future: impl Future, Status>>, + ) -> std::result::Result, Status> { + tokio::time::timeout(RPC_TIMEOUT, future) + .await + .map_err(|_| { + Status::deadline_exceeded(format!( + "middleware '{}' {operation} timed out", + self.registration_name + )) + })? + } +} + +#[tonic::async_trait] +impl SupervisorMiddleware for RemoteMiddlewareService { + async fn describe( + &self, + request: Request<()>, + ) -> std::result::Result, Status> { + let mut client = self.client.clone(); + self.with_timeout("Describe", client.describe(request)) + .await + } + + async fn validate_config( + &self, + request: Request, + ) -> std::result::Result, Status> { + let mut client = self.client.clone(); + self.with_timeout("ValidateConfig", client.validate_config(request)) + .await + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> std::result::Result, Status> { + let mut client = self.client.clone(); + self.with_timeout("EvaluateHttpRequest", client.evaluate_http_request(request)) + .await + } +} diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 8383b6bb2b..84853f7517 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -454,35 +454,41 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; - let req = - match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) - .await? - { - MiddlewareApplyResult::Allowed(req) => req, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; - return Ok(()); - } - }; + let req = match apply_middleware_chain( + req, + client, + ctx, + chain, + engine.middleware_runner(), + engine.generation_guard(), + ) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let outcome = crate::l7::rest::relay_http_request_with_options_guarded( &req, client, @@ -786,9 +792,11 @@ pub(crate) async fn apply_middleware_chain, + runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, ) -> Result { - apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, generation_guard).await + apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, runner, generation_guard) + .await } pub(crate) async fn apply_middleware_chain_for_scheme( @@ -797,12 +805,12 @@ pub(crate) async fn apply_middleware_chain_for_scheme, + runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); } - let runner = openshell_supervisor_middleware::ChainRunner::default(); let chain = runner.describe_chain(&chain).await?; let max_body_bytes = middleware_chain_body_limit(&chain).expect("non-empty middleware chain has a body limit"); @@ -1221,35 +1229,41 @@ where if allowed || config.enforcement == EnforcementMode::Audit { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; - let req = - match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) - .await? - { - MiddlewareApplyResult::Allowed(req) => req, - MiddlewareApplyResult::Denied(reason) => { - provider - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; - return Ok(()); - } - }; + let req = match apply_middleware_chain( + req, + client, + ctx, + chain, + engine.middleware_runner(), + engine.generation_guard(), + ) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + provider + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let req_with_auth = match crate::l7::token_grant_injection::inject_if_needed(req, ctx).await { Ok(req) => req, @@ -1490,35 +1504,41 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; - let req = - match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) - .await? - { - MiddlewareApplyResult::Allowed(req) => req, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; - return Ok(()); - } - }; + let req = match apply_middleware_chain( + req, + client, + ctx, + chain, + engine.middleware_runner(), + engine.generation_guard(), + ) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; // Future MCP response/SSE introspection or rewrite would hook here // before returning upstream bytes. The current policy schema has no // trusted-annotations or version-profile field, so MCP responses and @@ -1714,35 +1734,41 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; - let req = - match apply_middleware_chain(req, client, ctx, chain, engine.generation_guard()) - .await? - { - MiddlewareApplyResult::Allowed(req) => req, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; - return Ok(()); - } - }; + let req = match apply_middleware_chain( + req, + client, + ctx, + chain, + engine.middleware_runner(), + engine.generation_guard(), + ) + .await? + { + MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Denied(reason) => { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + }; let outcome = crate::l7::rest::relay_http_request_with_resolver_guarded( &req, client, @@ -2170,7 +2196,9 @@ where if generation != generation_guard.captured_generation() { return Ok(()); } - match apply_middleware_chain(req, client, ctx, chain, generation_guard).await? { + let runner = engine.middleware_runner()?; + match apply_middleware_chain(req, client, ctx, chain, &runner, generation_guard).await? + { MiddlewareApplyResult::Allowed(req) => req, MiddlewareApplyResult::Denied(reason) => { crate::l7::rest::RestProvider::default() diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index c4e7739963..9e1427dcde 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -13,11 +13,11 @@ use openshell_core::policy::{ }; use openshell_core::proto::SandboxPolicy as ProtoSandboxPolicy; use openshell_policy::L7ConfigStanza; -use openshell_supervisor_middleware::ChainEntry; +use openshell_supervisor_middleware::{ChainEntry, ChainRunner, MiddlewareRegistry}; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{ - Arc, Mutex, + Arc, Mutex, RwLock, atomic::{AtomicU64, Ordering}, }; @@ -73,6 +73,7 @@ pub struct SandboxConfig { pub struct OpaEngine { engine: Mutex, generation: Arc, + middleware_runner: RwLock, } /// Generation guard captured when an HTTP tunnel or request path starts. @@ -112,6 +113,7 @@ impl PolicyGenerationGuard { pub struct TunnelPolicyEngine { engine: Mutex, generation_guard: PolicyGenerationGuard, + middleware_runner: ChainRunner, } impl TunnelPolicyEngine { @@ -135,6 +137,10 @@ impl TunnelPolicyEngine { &self.engine } + pub(crate) fn middleware_runner(&self) -> &ChainRunner { + &self.middleware_runner + } + /// Query the ordered middleware chain for a destination within this tunnel. pub fn query_middleware_chain(&self, input: &NetworkInput) -> Result> { let mut engine = self @@ -164,6 +170,7 @@ impl OpaEngine { Ok(Self { engine: Mutex::new(engine), generation: Arc::new(AtomicU64::new(0)), + middleware_runner: RwLock::new(ChainRunner::default()), }) } @@ -182,6 +189,7 @@ impl OpaEngine { Ok(Self { engine: Mutex::new(engine), generation: Arc::new(AtomicU64::new(0)), + middleware_runner: RwLock::new(ChainRunner::default()), }) } @@ -254,6 +262,7 @@ impl OpaEngine { Ok(Self { engine: Mutex::new(engine), generation: Arc::new(AtomicU64::new(0)), + middleware_runner: RwLock::new(ChainRunner::default()), }) } @@ -451,6 +460,25 @@ impl OpaEngine { self.generation.load(Ordering::Acquire) } + /// Replace the complete middleware service registry and invalidate + /// existing tunnels so subsequent requests use the new service set. + pub fn replace_middleware_registry(&self, registry: MiddlewareRegistry) -> Result<()> { + let mut runner = self + .middleware_runner + .write() + .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; + *runner = ChainRunner::from_registry(registry); + self.generation.fetch_add(1, Ordering::AcqRel); + Ok(()) + } + + pub(crate) fn middleware_runner(&self) -> Result { + self.middleware_runner + .read() + .map(|runner| runner.clone()) + .map_err(|_| miette::miette!("middleware runner lock poisoned")) + } + /// Return a guard for a previously captured policy generation. pub fn generation_guard(&self, expected_generation: u64) -> Result { let generation = self.current_generation(); @@ -662,6 +690,7 @@ impl OpaEngine { captured_generation: generation, current_generation: Arc::clone(&self.generation), }, + middleware_runner: self.middleware_runner()?, }) } } @@ -2941,6 +2970,7 @@ network_policies: let engine = OpaEngine { engine: Mutex::new(rego), generation: Arc::new(AtomicU64::new(0)), + middleware_runner: RwLock::new(ChainRunner::default()), }; let input = l7_websocket_graphql_input( "realtime.graphql.com", diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index afec98666d..8616c3b2c6 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4218,6 +4218,7 @@ async fn handle_forward_proxy( return Ok(()); } if !chain.is_empty() { + let middleware_runner = opa_engine.middleware_runner()?; let request = crate::l7::rest::request_from_buffered_http( method, middleware_path, @@ -4230,6 +4231,7 @@ async fn handle_forward_proxy( &l7_ctx, &scheme, chain, + &middleware_runner, &forward_generation_guard, ) .await? diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 88b82870de..c289671905 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -103,6 +103,14 @@ guest_tls_key = "/etc/openshell/certs/client-key.pem" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 +# Local-development-only external supervisor middleware. The endpoint must be +# reachable from both the gateway and sandbox supervisors. +[[openshell.gateway.middleware]] +name = "local-content-guard" +endpoint = "http://host.openshell.internal:50051" +allow_insecure = true +max_body_bytes = 262144 + # Gateway listener TLS (distinct from the per-driver guest_tls_*). [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" @@ -140,6 +148,26 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. +## External Supervisor Middleware + +Register external supervisor middleware with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. + +```toml +[[openshell.gateway.middleware]] +name = "local-content-guard" +endpoint = "http://host.openshell.internal:50051" +allow_insecure = true +max_body_bytes = 262144 +``` + +Each service implements the supervisor middleware gRPC contract and may expose multiple binding IDs through `Describe`. Policies reference those binding IDs, not the registration `name`. The gateway rejects duplicate binding IDs across services and prevents external services from claiming the reserved `openshell/` namespace. + +The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. + +`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero and no larger than each binding's advertised limit. OpenShell rejects an oversized value instead of silently clamping it. + +External middleware is a local-development preview. The endpoint must use plaintext `http://`, and `allow_insecure = true` is required as an explicit acknowledgement that inspected request content is sent without transport encryption or peer authentication. TLS, authentication, health checks, and runtime registration are not supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address when both runtimes resolve it. + `image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. ## Driver References diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 049ad47979..69ff06dc35 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -20,6 +20,7 @@ filesystem_policy: { ... } landlock: { ... } process: { ... } network_policies: { ... } +network_middlewares: [ ... ] ``` | Field | Type | Required | Category | Description | @@ -29,6 +30,7 @@ network_policies: { ... } | `landlock` | object | No | Static | Configures Landlock LSM enforcement behavior. | | `process` | object | No | Static | Sets the user and group the agent process runs as. | | `network_policies` | map | No | Dynamic | Declares which binaries can reach which network endpoints. | +| `network_middlewares` | list | No | Dynamic | Selects ordered HTTP request middleware by destination host. | Static fields are set at sandbox creation time. Changing them requires destroying and recreating the sandbox. Dynamic fields can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. @@ -472,7 +474,35 @@ Identifies an executable that is permitted to use the associated endpoints. |---|---|---|---| | `path` | string | Yes | Filesystem path to the executable. Supports glob patterns with `*` and `**`. For example, `/sandbox/.vscode-server/**` matches any executable under that directory tree. | -### Full Example +## Network Middleware + +**Category:** Dynamic + +An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once in list order before provider credential injection. + +```yaml showLineNumbers={false} +network_middlewares: + - name: redact-secrets + middleware: openshell/secrets + config: + secrets: redact + on_error: fail_closed + endpoints: + include: ["*.example.com"] + exclude: ["trusted.example.com"] +``` + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | string | Yes | Policy-local config name. Names must be unique within the list. | +| `middleware` | string | Yes | Built-in or operator-registered binding ID. `openshell/` is reserved for built-ins. | +| `config` | object | No | Implementation-owned configuration validated by the selected middleware. | +| `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | +| `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists. Exclusions take precedence. | + +Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. + +## Full Example The following policy grants read-only GitHub API access and npm registry access: diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 212ba76fce..295acb64c1 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -12,7 +12,7 @@ Use this page to apply and iterate policy changes on running sandboxes. For a fu ## Policy Structure -A policy has static sections `filesystem_policy`, `landlock`, and `process` that are locked at sandbox creation, and a dynamic section `network_policies` that is hot-reloadable on a running sandbox. +A policy has static sections `filesystem_policy`, `landlock`, and `process` that are locked at sandbox creation, and dynamic `network_policies` and `network_middlewares` sections that are hot-reloadable on a running sandbox. ```yaml wordWrap showLineNumbers={false} version: 1 @@ -44,6 +44,17 @@ network_policies: binaries: - path: /usr/bin/curl +# Dynamic: ordered middleware selected independently by admitted host. +network_middlewares: + - name: redact-secrets + middleware: openshell/secrets + config: + secrets: redact + on_error: fail_closed + endpoints: + include: ["api.example.com"] + exclude: [] + ``` Static sections are locked at sandbox creation. Changing them requires destroying and recreating the sandbox. @@ -57,6 +68,29 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in | `landlock` | Static | Configures Landlock LSM enforcement behavior. Set `compatibility` to `best_effort` (skip individual inaccessible paths while applying remaining rules) or `hard_requirement` (fail if any path is inaccessible or the required kernel ABI is unavailable). Refer to the [Policy Schema Reference](/reference/policy-schema#landlock) for the full behavior table. | | `process` | Static | Sets the OS-level identity for the agent process. `run_as_user` and `run_as_group` default to `sandbox`. Root (`root` or `0`) is rejected. The agent also runs with seccomp filters that block dangerous system calls. | | `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` goes through the proxy, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints without `protocol` allow the TCP stream through without inspecting payloads.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | +| `network_middlewares` | Dynamic | Declares ordered HTTP request middleware configs. After network and L7 policy admit a request, OpenShell matches each config's host selectors independently and runs matching entries in declaration order before credential injection. | + +## Supervisor Middleware + +Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. + +```yaml +network_middlewares: + - name: redact-secrets + middleware: openshell/secrets + config: + secrets: redact + on_error: fail_closed + endpoints: + include: ["*.example.com"] + exclude: ["trusted.example.com"] +``` + +Matching entries run once each in top-level declaration order. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. + +`openshell/secrets` is built into the supervisor. External binding IDs must be registered by the gateway operator before a policy can reference them; see [External Supervisor Middleware](/reference/gateway-config#external-supervisor-middleware). The gateway calls the implementation's `ValidateConfig` before accepting the policy. + +`on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect; policy validation rejects a required selector that can cover a `tls: skip` endpoint. ## Baseline Filesystem Paths diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 04cbd67767..afec587232 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -352,4 +352,20 @@ message GetSandboxConfigResponse { // Fingerprint for provider credential inputs attached to this sandbox. // Changes when attached provider names or attached provider records change. uint64 provider_env_revision = 8; + // Operator-registered external middleware services required by the effective + // policy. Built-in middleware is not included. + repeated ExternalMiddlewareService external_middleware = 9; +} + +// Connection details for one operator-registered external middleware service. +// V1 supports only explicitly enabled plaintext gRPC for local development. +message ExternalMiddlewareService { + // Operator-facing registration name used for diagnostics. + string name = 1; + // gRPC endpoint reachable from the sandbox supervisor. + string endpoint = 2; + // Explicit acknowledgement that request content is sent without TLS. + bool allow_insecure = 3; + // Operator-owned body limit applied to every binding exposed by the service. + uint64 max_body_bytes = 4; } From 762887a6eb72c9ec5c668a665c7b0c6e327b2b60 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 29 Jun 2026 15:38:08 -0700 Subject: [PATCH 08/59] refactor(supervisor-middleware): clarify service contract Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 4 +- crates/openshell-core/src/grpc_client.rs | 4 +- crates/openshell-sandbox/src/lib.rs | 27 ++++---- crates/openshell-server/src/config_file.rs | 10 +-- crates/openshell-server/src/grpc/policy.rs | 17 +++-- crates/openshell-server/src/lib.rs | 2 +- .../src/lib.rs | 62 +++++++++---------- docs/reference/gateway-config.mdx | 12 ++-- docs/reference/policy-schema.mdx | 4 ++ docs/sandboxes/policies.mdx | 6 +- proto/sandbox.proto | 10 +-- 11 files changed, 85 insertions(+), 73 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index deec00f320..d63c4fbaab 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -69,7 +69,7 @@ are relayed but are not currently parsed for policy enforcement. For admitted HTTP requests, the proxy can run an ordered supervisor middleware chain before credential injection. Host selectors choose the chain independently of the network rule that admitted the request. Built-ins run in-process; -operator-registered external services are called directly from the supervisor +operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep the last-known-good service registry when a live config reload fails. @@ -184,7 +184,7 @@ quickly. - If gateway config polling fails, the sandbox keeps its last-known-good policy. - If a live policy update is invalid, the supervisor rejects it and keeps the current policy. -- If an external middleware call fails, the selected config's `on_error` +- If an operator-run middleware call fails, the selected config's `on_error` behavior decides whether to deny the request or continue without that stage. - Existing raw byte streams are connection scoped. Dynamic policy changes apply to new connections or the next parsed HTTP request where the proxy can safely diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 57f72dca61..836c7880c4 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -728,7 +728,7 @@ pub struct SettingsPollResult { /// When `policy_source` is `Global`, the version of the global policy revision. pub global_policy_version: u32, pub provider_env_revision: u64, - pub external_middleware: Vec, + pub supervisor_middleware_services: Vec, } pub struct ProviderEnvironmentResult { @@ -773,7 +773,7 @@ impl CachedOpenShellClient { settings: inner.settings, global_policy_version: inner.global_policy_version, provider_env_revision: inner.provider_env_revision, - external_middleware: inner.external_middleware, + supervisor_middleware_services: inner.supervisor_middleware_services, }) } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 30da1c292b..5e640d73ff 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1483,8 +1483,8 @@ async fn load_policy( info!("Creating OPA engine from proto policy data"); let engine = OpaEngine::from_proto(&proto_policy)?; let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_external( - sandbox_config.external_middleware, + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + sandbox_config.supervisor_middleware_services, ) .await?; engine.replace_middleware_registry(middleware_registry)?; @@ -1638,7 +1638,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let mut current_config_revision: u64 = 0; let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; let mut current_policy_hash = String::new(); - let mut current_external_middleware = Vec::new(); + let mut current_middleware_services = Vec::new(); let mut current_settings: std::collections::HashMap< String, openshell_core::proto::EffectiveSetting, @@ -1650,7 +1650,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); current_config_revision = result.config_revision; current_policy_hash = result.policy_hash.clone(); - current_external_middleware = result.external_middleware; + current_middleware_services = result.supervisor_middleware_services; current_settings = result.settings; debug!( config_revision = current_config_revision, @@ -1680,7 +1680,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } let policy_changed = result.policy_hash != current_policy_hash; - let middleware_changed = result.external_middleware != current_external_middleware; + let middleware_changed = + result.supervisor_middleware_services != current_middleware_services; // Log which settings changed. log_setting_changes(¤t_settings, &result.settings); @@ -1740,26 +1741,26 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } if middleware_changed { - match openshell_supervisor_middleware::MiddlewareRegistry::connect_external( - result.external_middleware.clone(), + match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + result.supervisor_middleware_services.clone(), ) .await .and_then(|registry| ctx.opa_engine.replace_middleware_registry(registry)) { Ok(()) => { - current_external_middleware = result.external_middleware.clone(); + current_middleware_services = result.supervisor_middleware_services.clone(); ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "loaded") .unmapped( - "external_middleware_count", - serde_json::json!(current_external_middleware.len()) + "supervisor_middleware_service_count", + serde_json::json!(current_middleware_services.len()) ) .message(format!( - "External middleware registry reloaded [service_count:{}]", - current_external_middleware.len() + "Supervisor middleware registry reloaded [service_count:{}]", + current_middleware_services.len() )) .build() ); @@ -1771,7 +1772,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .status(StatusId::Failure) .state(StateId::Other, "failed") .message(format!( - "External middleware registry reload failed, keeping last-known-good registry [error:{error}]" + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" )) .build() ); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 13c7e9ebb2..4b0fbc919c 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -25,7 +25,7 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use openshell_core::config::ComputeDriverKind; -use openshell_core::proto::ExternalMiddlewareService; +use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{GatewayAuthConfig, GatewayJwtConfig, MtlsAuthConfig, OidcConfig, TlsConfig}; use serde::{Deserialize, Serialize}; @@ -153,7 +153,7 @@ pub struct GatewayFileSection { pub gateway_jwt: Option, // ── Supervisor middleware ───────────────────────────────────────────── - /// Statically registered external middleware services. Registration is + /// Statically registered supervisor middleware services. Registration is /// operator-owned and changes require a gateway restart. #[serde(default)] pub middleware: Vec, @@ -167,7 +167,7 @@ pub struct GatewayFileSection { pub database_url: Option, } -/// One `[[openshell.gateway.middleware]]` external middleware registration. +/// One `[[openshell.gateway.middleware]]` supervisor middleware registration. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MiddlewareServiceFileConfig { @@ -182,7 +182,7 @@ pub struct MiddlewareServiceFileConfig { pub max_body_bytes: u64, } -impl From<&MiddlewareServiceFileConfig> for ExternalMiddlewareService { +impl From<&MiddlewareServiceFileConfig> for SupervisorMiddlewareService { fn from(config: &MiddlewareServiceFileConfig) -> Self { Self { name: config.name.clone(), @@ -435,7 +435,7 @@ allow_unauthenticated_users = true } #[test] - fn parses_external_middleware_registration() { + fn parses_supervisor_middleware_registration() { let toml = r#" [[openshell.gateway.middleware]] name = "local-guard" diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index c29de71ffd..9587e72955 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1230,14 +1230,13 @@ pub(super) async fn handle_get_sandbox_config( } let settings = merge_effective_settings(&global_settings, &sandbox_settings)?; - let external_middleware = state - .middleware_registry - .required_external_services(policy.as_ref()); + let supervisor_middleware_services = + state.middleware_registry.required_services(policy.as_ref()); let config_revision = compute_config_revision( policy.as_ref(), &settings, policy_source, - &external_middleware, + &supervisor_middleware_services, ); let provider_env_revision = compute_provider_env_revision(state.store.as_ref(), &sandbox_provider_names).await?; @@ -1251,7 +1250,7 @@ pub(super) async fn handle_get_sandbox_config( policy_source: policy_source.into(), global_policy_version, provider_env_revision, - external_middleware, + supervisor_middleware_services, })) } @@ -3144,7 +3143,7 @@ fn compute_config_revision( policy: Option<&ProtoSandboxPolicy>, settings: &HashMap, policy_source: PolicySource, - external_middleware: &[openshell_core::proto::ExternalMiddlewareService], + supervisor_middleware_services: &[openshell_core::proto::SupervisorMiddlewareService], ) -> u64 { let mut hasher = Sha256::new(); hasher.update((policy_source as i32).to_le_bytes()); @@ -3177,7 +3176,7 @@ fn compute_config_revision( } } } - let mut middleware = external_middleware.iter().collect::>(); + let mut middleware = supervisor_middleware_services.iter().collect::>(); middleware.sort_by(|left, right| left.name.cmp(&right.name)); for service in middleware { hasher.update(service.encode_to_vec()); @@ -9327,10 +9326,10 @@ mod tests { } #[test] - fn config_revision_changes_when_external_middleware_changes() { + fn config_revision_changes_when_supervisor_middleware_services_change() { let policy = ProtoSandboxPolicy::default(); let settings = HashMap::new(); - let service = openshell_core::proto::ExternalMiddlewareService { + let service = openshell_core::proto::SupervisorMiddlewareService { name: "local-guard".into(), endpoint: "http://127.0.0.1:50051".into(), allow_insecure: true, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index bca8abe2e7..bd974d4b4e 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -242,7 +242,7 @@ pub(crate) async fn run_server( }) .unwrap_or_default(); let middleware_registry = Arc::new( - MiddlewareRegistry::connect_external(middleware_registrations) + MiddlewareRegistry::connect_services(middleware_registrations) .await .map_err(|error| Error::config(format!("middleware registration failed: {error}")))?, ); diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 828179d186..ebc5817e45 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -15,9 +15,9 @@ pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - Decision, ExternalMiddlewareService, Finding, HttpRequestEvaluation, HttpRequestTarget, - MiddlewareBinding, MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, - ValidateConfigRequest, + Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, + MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, + SupervisorMiddlewareService, ValidateConfigRequest, }; use tokio::sync::OnceCell; use tonic::Request; @@ -206,13 +206,13 @@ static IN_PROCESS_SERVICE: LazyLock> = LazyLock::new /// Validated middleware services available to a gateway or one supervisor. /// -/// The registry always contains the in-process built-ins. External services -/// are connected and described before construction succeeds, so callers never +/// The registry always contains the in-process built-ins. Operator-registered +/// services are connected and described before construction succeeds, so callers never /// observe a partially registered service set. #[derive(Clone)] pub struct MiddlewareRegistry { services: Arc>>, - external: Arc>, + registered_services: Arc>, } impl std::fmt::Debug for MiddlewareRegistry { @@ -220,14 +220,14 @@ impl std::fmt::Debug for MiddlewareRegistry { formatter .debug_struct("MiddlewareRegistry") .field("service_count", &self.services.len()) - .field("external_count", &self.external.len()) + .field("registered_service_count", &self.registered_services.len()) .finish() } } #[derive(Clone)] -struct RegisteredExternalService { - registration: ExternalMiddlewareService, +struct RegisteredMiddlewareService { + registration: SupervisorMiddlewareService, binding_ids: Vec, } @@ -235,15 +235,15 @@ impl Default for MiddlewareRegistry { fn default() -> Self { Self { services: Arc::new(vec![Arc::clone(&IN_PROCESS_SERVICE)]), - external: Arc::new(Vec::new()), + registered_services: Arc::new(Vec::new()), } } } -fn validate_registration(registration: &ExternalMiddlewareService) -> Result<()> { +fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<()> { if registration.name.trim().is_empty() { return Err(miette!( - "external middleware registration name cannot be empty" + "supervisor middleware registration name cannot be empty" )); } if !registration.allow_insecure { @@ -268,7 +268,7 @@ fn validate_registration(registration: &ExternalMiddlewareService) -> Result<()> } fn validate_external_manifest( - registration: &ExternalMiddlewareService, + registration: &SupervisorMiddlewareService, manifest: &MiddlewareManifest, operator_max_body_bytes: usize, known_binding_ids: &mut HashSet, @@ -339,10 +339,10 @@ fn validate_external_manifest( } impl MiddlewareRegistry { - /// Connect and validate every external service registration. - pub async fn connect_external(registrations: Vec) -> Result { + /// Connect and validate every operator-provided service registration. + pub async fn connect_services(registrations: Vec) -> Result { let mut services = vec![Arc::clone(&IN_PROCESS_SERVICE)]; - let mut external = Vec::with_capacity(registrations.len()); + let mut registered_services = Vec::with_capacity(registrations.len()); let mut registration_names = HashSet::new(); let mut binding_ids = HashSet::from([BUILTIN_SECRETS.to_string()]); @@ -350,7 +350,7 @@ impl MiddlewareRegistry { validate_registration(®istration)?; if !registration_names.insert(registration.name.clone()) { return Err(miette!( - "duplicate external middleware registration name '{}'", + "duplicate supervisor middleware registration name '{}'", registration.name )); } @@ -395,7 +395,7 @@ impl MiddlewareRegistry { manifest: manifest_cell, operator_max_body_bytes: Some(operator_max_body_bytes), })); - external.push(RegisteredExternalService { + registered_services.push(RegisteredMiddlewareService { registration, binding_ids: described_ids, }); @@ -403,7 +403,7 @@ impl MiddlewareRegistry { Ok(Self { services: Arc::new(services), - external: Arc::new(external), + registered_services: Arc::new(registered_services), }) } @@ -433,7 +433,7 @@ impl MiddlewareRegistry { pub fn ensure_policy_bindings_registered(&self, policy: &SandboxPolicy) -> Result<()> { for config in &policy.network_middlewares { let registered = config.middleware == BUILTIN_SECRETS - || self.external.iter().any(|service| { + || self.registered_services.iter().any(|service| { service .binding_ids .iter() @@ -450,11 +450,11 @@ impl MiddlewareRegistry { Ok(()) } - /// Return only external services referenced by the effective policy. - pub fn required_external_services( + /// Return only operator-registered services referenced by the effective policy. + pub fn required_services( &self, policy: Option<&SandboxPolicy>, - ) -> Vec { + ) -> Vec { let Some(policy) = policy else { return Vec::new(); }; @@ -463,7 +463,7 @@ impl MiddlewareRegistry { .iter() .map(|config| config.middleware.as_str()) .collect(); - self.external + self.registered_services .iter() .filter(|service| { service @@ -491,7 +491,7 @@ impl ChainRunner { manifest: OnceCell::new(), operator_max_body_bytes: None, })]), - external: Arc::new(Vec::new()), + registered_services: Arc::new(Vec::new()), }), } } @@ -1126,8 +1126,8 @@ mod tests { } } - fn external_registration(max_body_bytes: u64) -> ExternalMiddlewareService { - ExternalMiddlewareService { + fn external_registration(max_body_bytes: u64) -> SupervisorMiddlewareService { + SupervisorMiddlewareService { name: "local-guard-service".into(), endpoint: "http://127.0.0.1:50051".into(), allow_insecure: true, @@ -1137,7 +1137,7 @@ mod tests { async fn registry_with_external( service: Arc, - registration: ExternalMiddlewareService, + registration: SupervisorMiddlewareService, ) -> MiddlewareRegistry { let manifest = service .describe(Request::new(())) @@ -1164,7 +1164,7 @@ mod tests { operator_max_body_bytes: Some(operator_max_body_bytes), }), ]), - external: Arc::new(vec![RegisteredExternalService { + registered_services: Arc::new(vec![RegisteredMiddlewareService { registration, binding_ids, }]), @@ -1294,7 +1294,7 @@ mod tests { let mut registration = external_registration(1024); registration.endpoint = format!("http://{address}"); - let registry = MiddlewareRegistry::connect_external(vec![registration.clone()]) + let registry = MiddlewareRegistry::connect_services(vec![registration.clone()]) .await .expect("connect external middleware"); let policy = SandboxPolicy { @@ -1313,7 +1313,7 @@ mod tests { .await .expect("remote config validates"); assert_eq!( - registry.required_external_services(Some(&policy)), + registry.required_services(Some(&policy)), vec![registration] ); diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index c289671905..1b3554c203 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -148,9 +148,13 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. -## External Supervisor Middleware +## Supervisor Middleware Services -Register external supervisor middleware with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. + +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. + + +Register operator-run supervisor middleware services with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. ```toml [[openshell.gateway.middleware]] @@ -160,13 +164,13 @@ allow_insecure = true max_body_bytes = 262144 ``` -Each service implements the supervisor middleware gRPC contract and may expose multiple binding IDs through `Describe`. Policies reference those binding IDs, not the registration `name`. The gateway rejects duplicate binding IDs across services and prevents external services from claiming the reserved `openshell/` namespace. +Each service implements the supervisor middleware gRPC contract and may expose multiple binding IDs through `Describe`. Policies reference those binding IDs, not the registration `name`. The gateway rejects duplicate binding IDs across services and prevents operator-run services from claiming the reserved `openshell/` namespace. The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. `max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero and no larger than each binding's advertised limit. OpenShell rejects an oversized value instead of silently clamping it. -External middleware is a local-development preview. The endpoint must use plaintext `http://`, and `allow_insecure = true` is required as an explicit acknowledgement that inspected request content is sent without transport encryption or peer authentication. TLS, authentication, health checks, and runtime registration are not supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address when both runtimes resolve it. +The service endpoint must use plaintext `http://`, and `allow_insecure = true` is required as an explicit acknowledgement that inspected request content is sent without transport encryption or peer authentication. TLS, authentication, health checks, and runtime registration are not supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address when both runtimes resolve it. `image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 69ff06dc35..9279b6f663 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -476,6 +476,10 @@ Identifies an executable that is permitted to use the associated endpoints. ## Network Middleware + +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. + + **Category:** Dynamic An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once in list order before provider credential injection. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 295acb64c1..5280cc0853 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -72,6 +72,10 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in ## Supervisor Middleware + +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. + + Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. ```yaml @@ -88,7 +92,7 @@ network_middlewares: Matching entries run once each in top-level declaration order. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. -`openshell/secrets` is built into the supervisor. External binding IDs must be registered by the gateway operator before a policy can reference them; see [External Supervisor Middleware](/reference/gateway-config#external-supervisor-middleware). The gateway calls the implementation's `ValidateConfig` before accepting the policy. +`openshell/secrets` is built into the supervisor. Operator-provided binding IDs must be registered before a policy can reference them; see [Supervisor Middleware Services](/reference/gateway-config#supervisor-middleware-services). The gateway calls the implementation's `ValidateConfig` before accepting the policy. `on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect; policy validation rejects a required selector that can cover a `tls: skip` endpoint. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index afec587232..644fd86cba 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -352,14 +352,14 @@ message GetSandboxConfigResponse { // Fingerprint for provider credential inputs attached to this sandbox. // Changes when attached provider names or attached provider records change. uint64 provider_env_revision = 8; - // Operator-registered external middleware services required by the effective - // policy. Built-in middleware is not included. - repeated ExternalMiddlewareService external_middleware = 9; + // Operator-registered supervisor middleware services required by the + // effective policy. Built-in middleware is not included. + repeated SupervisorMiddlewareService supervisor_middleware_services = 9; } -// Connection details for one operator-registered external middleware service. +// Connection details for one operator-registered supervisor middleware service. // V1 supports only explicitly enabled plaintext gRPC for local development. -message ExternalMiddlewareService { +message SupervisorMiddlewareService { // Operator-facing registration name used for diagnostics. string name = 1; // gRPC endpoint reachable from the sandbox supervisor. From 31662d8601ce8dcffd2dfe822e07966bac9d01f6 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 29 Jun 2026 20:18:37 -0700 Subject: [PATCH 09/59] docs(supervisor-middleware): refine preview warning Signed-off-by: Piotr Mlocek --- docs/reference/gateway-config.mdx | 2 +- docs/reference/policy-schema.mdx | 2 +- docs/sandboxes/policies.mdx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 1b3554c203..7036238a20 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -151,7 +151,7 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth ## Supervisor Middleware Services -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. Register operator-run supervisor middleware services with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 9279b6f663..6ef48d869a 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -477,7 +477,7 @@ Identifies an executable that is permitted to use the associated endpoints. ## Network Middleware -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. **Category:** Dynamic diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 5280cc0853..8e71440e5f 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -73,7 +73,7 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in ## Supervisor Middleware -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware, not for production or long-lived integrations. +Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. From 2304e2fcc28c6af267d9b60139b8fde0b3763b5b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 10:22:43 -0700 Subject: [PATCH 10/59] docs(extensibility): add supervisor middleware guide Signed-off-by: Piotr Mlocek --- docs/extensibility/supervisor-middleware.mdx | 141 +++++++++++++++++++ docs/index.yml | 2 + docs/reference/gateway-config.mdx | 6 +- docs/reference/policy-schema.mdx | 6 +- docs/sandboxes/policies.mdx | 8 +- 5 files changed, 150 insertions(+), 13 deletions(-) create mode 100644 docs/extensibility/supervisor-middleware.mdx diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx new file mode 100644 index 0000000000..320495e59a --- /dev/null +++ b/docs/extensibility/supervisor-middleware.mdx @@ -0,0 +1,141 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Supervisor Middleware" +sidebar-title: "Supervisor Middleware" +description: "Configure and operate built-in and operator-run middleware for sandbox HTTP requests." +keywords: "Generative AI, Cybersecurity, AI Agents, Supervisor Middleware, Extensibility, Request Filtering" +--- + +Supervisor middleware adds ordered request-processing stages to allowed HTTP egress. Middleware runs after network and L7 policy admit a request and before OpenShell injects provider credentials. A stage can allow or deny the request, replace its body, add approved headers, and report audit-safe findings. + +Middleware selection is independent of the network policy rule that admitted the request. OpenShell matches middleware by destination host, so the same middleware applies consistently across broad, specific, user-authored, and provider-derived network policies. + +## Request Flow + +For each inspected HTTP request, the supervisor: + +1. Evaluates network and L7 policy. +2. Selects middleware whose host selectors match the admitted destination. +3. Buffers the request body using the smallest body limit in the selected chain. +4. Runs matching middleware in policy declaration order. +5. Applies allowed transformations, injects provider credentials, and forwards the request. + +Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. + +## Choose a Middleware Type + +| Type | Registration | Body limit | Deployment | +| --- | --- | --- | --- | +| Built-in | None | Defined by OpenShell | Runs inside the supervisor | +| Operator-run service | Required in gateway TOML | Set by the operator, up to the service capability | Runs as a separate service reachable by the gateway and supervisors | + +`openshell/secrets` is the built-in middleware currently available. It identifies common secret patterns in UTF-8 request bodies and replaces matched values before the request leaves the sandbox. + +Operator-run services expose one or more binding IDs. Policies reference a binding ID, such as `example/content-guard`, rather than the gateway registration name. + +## Register a Middleware Service + +Start an operator-run service before starting the gateway, then add a registration to the local gateway TOML: + +```toml +[[openshell.gateway.middleware]] +name = "local-content-guard" +endpoint = "http://host.openshell.internal:50051" +allow_insecure = true +max_body_bytes = 262144 +``` + +| Field | Description | +| --- | --- | +| `name` | Operator-facing registration name used in diagnostics. Policies do not reference this value. | +| `endpoint` | Service address reachable from both the gateway and sandbox supervisors. | +| `allow_insecure` | Required acknowledgement for the currently supported plaintext endpoint. | +| `max_body_bytes` | Operator limit applied to every binding exposed by the service. | + +The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. + +Registration is static. Restart the gateway after adding, removing, or changing a service. See [Gateway Configuration](/reference/gateway-config#supervisor-middleware-services) for the complete gateway TOML context. + +## Apply Middleware with Policy + +Add middleware configs to the top-level `network_middlewares` list: + +```yaml +network_middlewares: + - name: redact-secrets + middleware: openshell/secrets + config: + secrets: redact + on_error: fail_closed + endpoints: + include: ["*.example.com"] + exclude: ["trusted.example.com"] +``` + +Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, implementation-owned `config`, failure behavior, and host selectors. + +`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints. + +Matching configs run once each in top-level declaration order. Different config names may reference the same binding and run as separate stages. Config names must be unique. + +See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. + +## Configure Failure Behavior + +`on_error` controls what happens when middleware is unavailable, rejects its configuration, returns an invalid result, or exceeds a body limit. + +| Value | Behavior | +| --- | --- | +| `fail_closed` | Denies the request when the middleware stage fails. This is the default. | +| `fail_open` | Skips the failed stage and continues the request through the remaining chain. | + +Use `fail_open` only when bypassing the middleware preserves the intended security policy. OpenShell emits a detection finding when a failed stage is bypassed. + +An explicit deny decision always stops the chain and denies the request, regardless of `on_error`. + +## Set Body Limits + +Every middleware binding declares the largest request or replacement body it supports. + +- Built-in middleware uses its OpenShell-defined limit. +- Each operator-run registration sets `max_body_bytes` no higher than the service capability. +- A selected chain buffers using its smallest stage limit. +- The same per-stage limit applies to request bodies and replacement bodies. + +The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure and follows that config's `on_error` behavior. + +## Operate Middleware Services + +Plan startup and updates around these boundaries: + +- Start registered services before the gateway. The gateway validates every registration during startup. +- Keep service endpoints reachable from both the gateway and sandbox supervisors. The supervisors call operator-run services directly on the request path. +- Restart the gateway after changing registrations. +- Keep required services available before creating or updating policies. The gateway validates implementation-owned config before persisting a policy. +- Treat `fail_open` as an explicit availability-over-enforcement decision. + +When the effective sandbox configuration changes, a running supervisor validates the new service registry before installing it. If the reload fails, the supervisor keeps its last-known-good registry and emits a configuration failure event. + +## Observe Middleware + +Middleware activity is emitted through OpenShell's OCSF logging: + +- Each invocation records its policy-local middleware name, binding, decision, transformation state, and failure state. +- A bypass under `fail_open` emits a detection finding. +- A required stage that fails closed emits a high-severity detection finding. +- Findings include the service-provided type and label plus aggregate counts. Middleware services should keep those fields audit-safe and omit request content or matched values. +- Registry reload success and failure are emitted as configuration state changes. + +See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. + +## Current Limitations + +- Middleware applies only to HTTP requests parsed by the supervisor. +- The supported operation and phase are `HttpRequest/pre_credentials`. +- Selection uses destination host include and exclude patterns. +- Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. +- Operator-run services currently use explicitly enabled plaintext `http://` endpoints. +- TLS, service authentication, health checks, and runtime registration are not available. + +For a runnable operator workflow, see the [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard). diff --git a/docs/index.yml b/docs/index.yml index b2443e4afd..45db451a78 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -19,6 +19,8 @@ navigation: title: "Manage OpenShell" - folder: providers title: "Providers" +- folder: extensibility + title: "Extensibility" - folder: observability title: "Observability" - folder: kubernetes diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 7036238a20..cad7a4e888 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -150,10 +150,6 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth ## Supervisor Middleware Services - -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. - - Register operator-run supervisor middleware services with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. ```toml @@ -172,6 +168,8 @@ The gateway connects to every registered service and validates `Describe` before The service endpoint must use plaintext `http://`, and `allow_insecure = true` is required as an explicit acknowledgement that inspected request content is sent without transport encryption or peer authentication. TLS, authentication, health checks, and runtime registration are not supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address when both runtimes resolve it. +See [Supervisor Middleware](/extensibility/supervisor-middleware) for selection, failure, body-limit, and operational guidance. + `image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. ## Driver References diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 6ef48d869a..e36c535be9 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -476,10 +476,6 @@ Identifies an executable that is permitted to use the associated endpoints. ## Network Middleware - -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. - - **Category:** Dynamic An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once in list order before provider credential injection. @@ -506,6 +502,8 @@ network_middlewares: Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. +See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, failure behavior, body limits, and operational guidance. + ## Full Example The following policy grants read-only GitHub API access and npm registry access: diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 8e71440e5f..1353fd640c 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -72,10 +72,6 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in ## Supervisor Middleware - -Supervisor middleware is a research preview. Its policy and service contracts may change without compatibility guarantees. Use it only to prototype and evaluate middleware integrations. - - Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. ```yaml @@ -92,10 +88,12 @@ network_middlewares: Matching entries run once each in top-level declaration order. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. -`openshell/secrets` is built into the supervisor. Operator-provided binding IDs must be registered before a policy can reference them; see [Supervisor Middleware Services](/reference/gateway-config#supervisor-middleware-services). The gateway calls the implementation's `ValidateConfig` before accepting the policy. +`openshell/secrets` is built into the supervisor. Operator-provided binding IDs must be registered before a policy can reference them. The gateway validates implementation-owned config before accepting the policy. `on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect; policy validation rejects a required selector that can cover a `tls: skip` endpoint. +See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, chain ordering, body limits, failure behavior, and operations. + ## Baseline Filesystem Paths When a sandbox runs in proxy mode (the default), OpenShell automatically adds baseline filesystem paths required for the sandbox child process to function: `/usr`, `/lib`, `/etc`, `/var/log` (read-only) and `/sandbox`, `/tmp` (read-write). Paths like `/app` are included in the baseline set but are only added if they exist in the container image. From 97bef95a7a4b44dd97cad1a7b7a7cb653e168179 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 12:41:09 -0700 Subject: [PATCH 11/59] fix(server): remove stale middleware import Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index bd974d4b4e..86afe4ef42 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -55,7 +55,6 @@ mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; use openshell_core::{ComputeDriverKind, Config, Error, Result}; use openshell_supervisor_middleware::MiddlewareRegistry; -use serde::Deserialize; use std::collections::HashMap; use std::io::ErrorKind; use std::net::SocketAddr; From ab2daba91ff79e10f82dcc52764746bf239be91e Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 13:26:06 -0700 Subject: [PATCH 12/59] fix(network): remove needless test struct updates Signed-off-by: Piotr Mlocek --- crates/openshell-supervisor-network/src/opa.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 9e1427dcde..f0aea9c389 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -6698,7 +6698,6 @@ network_policies: path: link_path, ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { @@ -6777,7 +6776,6 @@ network_policies: path: link_path, ..Default::default() }], - ..Default::default() }, ); let proto = ProtoSandboxPolicy { From a820dd289778c5f39df6294e3c5906a8463a81a3 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 13:48:26 -0700 Subject: [PATCH 13/59] fix(middleware): avoid enabling core telemetry Signed-off-by: Piotr Mlocek --- crates/openshell-supervisor-middleware/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index e5e53618df..0af6e70c29 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -11,7 +11,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -openshell-core = { path = "../openshell-core" } +openshell-core = { path = "../openshell-core", default-features = false } miette = { workspace = true } prost-types = { workspace = true } From 49424a074f0ec41138c09e6f9965ab5b402a26bd Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 14:47:35 -0700 Subject: [PATCH 14/59] refactor(supervisor-middleware): simplify service endpoints Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/config_file.rs | 14 ++-- crates/openshell-server/src/grpc/policy.rs | 3 +- .../Cargo.toml | 2 +- .../src/lib.rs | 39 ++++++----- .../src/remote.rs | 23 +++++-- docs/extensibility/supervisor-middleware.mdx | 10 ++- docs/reference/gateway-config.mdx | 12 ++-- proto/middleware.proto | 66 +++++++++++++++++++ proto/sandbox.proto | 10 +-- 9 files changed, 127 insertions(+), 52 deletions(-) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 4b0fbc919c..7306c80e7e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -174,10 +174,7 @@ pub struct MiddlewareServiceFileConfig { /// Operator-facing name used for diagnostics. pub name: String, /// Plaintext gRPC endpoint reachable by the gateway and supervisors. - pub endpoint: String, - /// Required explicit opt-in to the local-development-only insecure mode. - #[serde(default)] - pub allow_insecure: bool, + pub grpc_endpoint: String, /// Operator-owned body limit for every binding exposed by this service. pub max_body_bytes: u64, } @@ -186,8 +183,7 @@ impl From<&MiddlewareServiceFileConfig> for SupervisorMiddlewareService { fn from(config: &MiddlewareServiceFileConfig) -> Self { Self { name: config.name.clone(), - endpoint: config.endpoint.clone(), - allow_insecure: config.allow_insecure, + grpc_endpoint: config.grpc_endpoint.clone(), max_body_bytes: config.max_body_bytes, } } @@ -439,8 +435,7 @@ allow_unauthenticated_users = true let toml = r#" [[openshell.gateway.middleware]] name = "local-guard" -endpoint = "http://127.0.0.1:50051" -allow_insecure = true +grpc_endpoint = "http://127.0.0.1:50051" max_body_bytes = 262144 "#; let tmp = write_tmp(toml); @@ -449,8 +444,7 @@ max_body_bytes = 262144 file.openshell.gateway.middleware, vec![MiddlewareServiceFileConfig { name: "local-guard".into(), - endpoint: "http://127.0.0.1:50051".into(), - allow_insecure: true, + grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes: 262_144, }] ); diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 9587e72955..97d4598132 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -9331,8 +9331,7 @@ mod tests { let settings = HashMap::new(); let service = openshell_core::proto::SupervisorMiddlewareService { name: "local-guard".into(), - endpoint: "http://127.0.0.1:50051".into(), - allow_insecure: true, + grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes: 1024, }; diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index 0af6e70c29..f36fc854b6 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -17,7 +17,7 @@ miette = { workspace = true } prost-types = { workspace = true } regex = { workspace = true } tokio = { workspace = true } -tonic = { workspace = true, features = ["channel", "server"] } +tonic = { workspace = true, features = ["channel", "server", "tls-native-roots"] } [dev-dependencies] tokio-stream = { workspace = true, features = ["net"] } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index ebc5817e45..00324d75fd 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -246,15 +246,11 @@ fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<( "supervisor middleware registration name cannot be empty" )); } - if !registration.allow_insecure { - return Err(miette!( - "middleware registration '{}' must set allow_insecure = true; TLS is not supported in V1", - registration.name - )); - } - if !registration.endpoint.starts_with("http://") { + if !registration.grpc_endpoint.starts_with("http://") + && !registration.grpc_endpoint.starts_with("https://") + { return Err(miette!( - "middleware registration '{}' endpoint must use http:// in the local-development-only V1", + "middleware registration '{}' grpc_endpoint must use http:// or https://", registration.name )); } @@ -365,7 +361,7 @@ impl MiddlewareRegistry { let service = Arc::new( remote::RemoteMiddlewareService::connect( ®istration.name, - ®istration.endpoint, + ®istration.grpc_endpoint, ) .await?, ); @@ -1129,8 +1125,7 @@ mod tests { fn external_registration(max_body_bytes: u64) -> SupervisorMiddlewareService { SupervisorMiddlewareService { name: "local-guard-service".into(), - endpoint: "http://127.0.0.1:50051".into(), - allow_insecure: true, + grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes, } } @@ -1267,11 +1262,23 @@ mod tests { } #[test] - fn external_registration_requires_explicit_insecure_opt_in() { + fn external_registration_accepts_http_and_https_grpc_endpoints() { + for grpc_endpoint in [ + "http://127.0.0.1:50051", + "https://middleware.example.com:443", + ] { + let mut registration = external_registration(4096); + registration.grpc_endpoint = grpc_endpoint.into(); + validate_registration(®istration).expect("supported gRPC endpoint scheme"); + } + } + + #[test] + fn external_registration_rejects_unsupported_grpc_endpoint_scheme() { let mut registration = external_registration(4096); - registration.allow_insecure = false; - let error = validate_registration(®istration).expect_err("opt-in required"); - assert!(error.to_string().contains("allow_insecure")); + registration.grpc_endpoint = "ftp://middleware.example.com".into(); + let error = validate_registration(®istration).expect_err("unsupported scheme"); + assert!(error.to_string().contains("http:// or https://")); } #[tokio::test] @@ -1293,7 +1300,7 @@ mod tests { let server_task = tokio::spawn(server); let mut registration = external_registration(1024); - registration.endpoint = format!("http://{address}"); + registration.grpc_endpoint = format!("http://{address}"); let registry = MiddlewareRegistry::connect_services(vec![registration.clone()]) .await .expect("connect external middleware"); diff --git a/crates/openshell-supervisor-middleware/src/remote.rs b/crates/openshell-supervisor-middleware/src/remote.rs index dd147788b9..7645ed811f 100644 --- a/crates/openshell-supervisor-middleware/src/remote.rs +++ b/crates/openshell-supervisor-middleware/src/remote.rs @@ -10,7 +10,7 @@ use openshell_core::proto::{ HttpRequestEvaluation, HttpRequestResult, MiddlewareManifest, ValidateConfigRequest, ValidateConfigResponse, }; -use tonic::transport::{Channel, Endpoint}; +use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; use tonic::{Request, Response, Status}; const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); @@ -23,19 +23,30 @@ pub struct RemoteMiddlewareService { } impl RemoteMiddlewareService { - pub async fn connect(registration_name: &str, endpoint: &str) -> Result { - let channel = Endpoint::from_shared(endpoint.to_string()) + pub async fn connect(registration_name: &str, grpc_endpoint: &str) -> Result { + let mut endpoint = Endpoint::from_shared(grpc_endpoint.to_string()) .into_diagnostic() .wrap_err_with(|| { - format!("middleware registration '{registration_name}' has an invalid endpoint") - })? + format!( + "middleware registration '{registration_name}' has an invalid grpc_endpoint" + ) + })?; + if grpc_endpoint.starts_with("https://") { + endpoint = endpoint + .tls_config(ClientTlsConfig::new().with_enabled_roots()) + .into_diagnostic() + .wrap_err_with(|| { + format!("middleware registration '{registration_name}' could not configure TLS") + })?; + } + let channel = endpoint .connect_timeout(CONNECT_TIMEOUT) .connect() .await .into_diagnostic() .wrap_err_with(|| { format!( - "middleware registration '{registration_name}' could not connect to {endpoint}" + "middleware registration '{registration_name}' could not connect to {grpc_endpoint}" ) })?; Ok(Self { diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 320495e59a..0ebd518bd9 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -41,16 +41,14 @@ Start an operator-run service before starting the gateway, then add a registrati ```toml [[openshell.gateway.middleware]] name = "local-content-guard" -endpoint = "http://host.openshell.internal:50051" -allow_insecure = true +grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 ``` | Field | Description | | --- | --- | | `name` | Operator-facing registration name used in diagnostics. Policies do not reference this value. | -| `endpoint` | Service address reachable from both the gateway and sandbox supervisors. | -| `allow_insecure` | Required acknowledgement for the currently supported plaintext endpoint. | +| `grpc_endpoint` | Service address reachable from both the gateway and sandbox supervisors. Supports plaintext `http://` and TLS `https://` with platform trust roots. | | `max_body_bytes` | Operator limit applied to every binding exposed by the service. | The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. @@ -135,7 +133,7 @@ See [Logging](/observability/logging) for log access and [OCSF JSON Export](/obs - The supported operation and phase are `HttpRequest/pre_credentials`. - Selection uses destination host include and exclude patterns. - Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. -- Operator-run services currently use explicitly enabled plaintext `http://` endpoints. -- TLS, service authentication, health checks, and runtime registration are not available. +- Operator-run services support plaintext `http://` and TLS `https://` endpoints. HTTPS certificates must chain to a CA in the platform trust store. +- Custom trust roots, client authentication, health checks, and runtime registration are not available. For a runnable operator workflow, see the [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index cad7a4e888..64540e5128 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -103,12 +103,11 @@ guest_tls_key = "/etc/openshell/certs/client-key.pem" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 -# Local-development-only external supervisor middleware. The endpoint must be -# reachable from both the gateway and sandbox supervisors. +# Operator-run supervisor middleware. The gRPC endpoint must be reachable from +# both the gateway and sandbox supervisors. [[openshell.gateway.middleware]] name = "local-content-guard" -endpoint = "http://host.openshell.internal:50051" -allow_insecure = true +grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 # Gateway listener TLS (distinct from the per-driver guest_tls_*). @@ -155,8 +154,7 @@ Register operator-run supervisor middleware services with one or more `[[openshe ```toml [[openshell.gateway.middleware]] name = "local-content-guard" -endpoint = "http://host.openshell.internal:50051" -allow_insecure = true +grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 ``` @@ -166,7 +164,7 @@ The gateway connects to every registered service and validates `Describe` before `max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero and no larger than each binding's advertised limit. OpenShell rejects an oversized value instead of silently clamping it. -The service endpoint must use plaintext `http://`, and `allow_insecure = true` is required as an explicit acknowledgement that inspected request content is sent without transport encryption or peer authentication. TLS, authentication, health checks, and runtime registration are not supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address when both runtimes resolve it. +The service `grpc_endpoint` currently supports plaintext `http://` and TLS `https://` using the platform trust store. Custom trust roots, client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. See [Supervisor Middleware](/extensibility/supervisor-middleware) for selection, failure, body-limit, and operational guidance. diff --git a/proto/middleware.proto b/proto/middleware.proto index 2944227d8a..9b988b930e 100644 --- a/proto/middleware.proto +++ b/proto/middleware.proto @@ -8,90 +8,156 @@ package openshell.middleware.v1; import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; +// SupervisorMiddleware lets an operator-run service inspect and transform +// sandbox HTTP egress before OpenShell injects credentials. service SupervisorMiddleware { + // Describe returns the service manifest and declared bindings. rpc Describe(google.protobuf.Empty) returns (MiddlewareManifest); + + // ValidateConfig checks service-specific configuration for one binding. rpc ValidateConfig(ValidateConfigRequest) returns (ValidateConfigResponse); + + // EvaluateHttpRequest returns an allow, deny, or mutation decision for one + // buffered HTTP request. rpc EvaluateHttpRequest(HttpRequestEvaluation) returns (HttpRequestResult); } +// MiddlewareManifest describes one service and the bindings it exposes. message MiddlewareManifest { + // Middleware protocol version implemented by the service. string api_version = 1; + // Human-readable service name used for diagnostics. string name = 2; + // Service-defined version string used for diagnostics. string service_version = 3; + // Bindings exposed by this service. repeated MiddlewareBinding bindings = 4; } +// MiddlewareBinding declares one operation and phase supported by a service. message MiddlewareBinding { + // Stable binding id used by policy configuration and audit logs. string id = 1; + // Supported operation name. V1 supports "HttpRequest". string operation = 2; + // Supported evaluation phase. V1 supports "pre_credentials". string phase = 3; // Maximum request or replacement body this binding can process. uint64 max_body_bytes = 4; } +// ValidateConfigRequest contains one policy configuration to validate. message ValidateConfigRequest { + // Middleware protocol version selected by OpenShell. string api_version = 1; + // Manifest binding id associated with this configuration. string binding_id = 2; + // Service-specific policy configuration. google.protobuf.Struct config = 3; } +// ValidateConfigResponse reports whether a policy configuration is accepted. message ValidateConfigResponse { + // True when the service accepts the configuration. bool valid = 1; + // Human-readable validation failure reason. Empty when valid is true. string reason = 2; } +// HttpRequestEvaluation contains one buffered HTTP request to evaluate. message HttpRequestEvaluation { + // Middleware protocol version selected by OpenShell. string api_version = 1; + // Manifest binding id selected for this evaluation. string binding_id = 2; + // Evaluation phase selected for this request. string phase = 3; + // Sandbox and request identity available to the supervisor. RequestContext context = 4; + // Validated service-specific policy configuration. google.protobuf.Struct config = 5; + // Destination and HTTP request target. HttpRequestTarget target = 6; + // HTTP request headers before OpenShell injects credentials. map headers = 7; + // Buffered request body. Empty for a bodyless request. bytes body = 8; } +// RequestContext identifies the sandbox request being evaluated. message RequestContext { + // Request id used to correlate middleware and supervisor logs. string request_id = 1; + // Sandbox id that originated the request. string sandbox_id = 2; + // Workload process that originated the request, when available. Process originating_process = 3; } +// HttpRequestTarget describes the admitted HTTP destination and request target. message HttpRequestTarget { + // Request scheme, such as "http" or "https". string scheme = 1; + // Destination hostname selected by network policy. string host = 2; + // Destination TCP port. uint32 port = 3; + // HTTP request method. string method = 4; + // Request path without the query string. string path = 5; + // Raw request query string without the leading question mark. string query = 6; } +// Process identifies a workload process and its executable ancestry. message Process { + // Executable path for the originating process. string binary = 1; + // Process id within the sandbox. uint32 pid = 2; + // Executable paths for ancestor processes, nearest parent first. repeated string ancestors = 3; } +// Decision controls whether OpenShell continues processing the request. enum Decision { + // Invalid response value handled according to the policy failure mode. DECISION_UNSPECIFIED = 0; + // Continue processing the request and apply any returned mutations. DECISION_ALLOW = 1; + // Deny the request before credentials are injected or data is sent upstream. DECISION_DENY = 2; } +// Finding is an audit-safe observation produced during evaluation. message Finding { + // Stable, service-defined finding type. string type = 1; + // Human-readable finding label that does not contain request content. string label = 2; + // Number of matching observations represented by this finding. uint32 count = 3; + // Service-defined confidence level. string confidence = 4; + // Service-defined severity level. string severity = 5; } +// HttpRequestResult contains the decision and optional request mutations. message HttpRequestResult { + // Allow or deny decision for this request. Decision decision = 1; + // Human-readable reason used for diagnostics and denied responses. string reason = 2; + // Replacement request body when has_body is true. bytes body = 3; + // True when body should replace the request body, including with an empty body. bool has_body = 4; + // Request headers to add before forwarding. Protected headers are rejected. map add_headers = 5; + // Audit-safe findings produced during evaluation. repeated Finding findings = 6; + // Non-secret service-defined metadata included in diagnostics. map metadata = 7; } diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 644fd86cba..c4018573d6 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -79,8 +79,12 @@ message NetworkMiddlewareConfig { MiddlewareEndpointSelector endpoints = 5; } +// Host selector controlling which admitted destinations use a middleware config. message MiddlewareEndpointSelector { + // Exact host or DNS glob patterns included in the selection. repeated string include = 1; + // Exact host or DNS glob patterns removed from the selection. + // Exclusions take precedence over inclusions. repeated string exclude = 2; } @@ -358,14 +362,12 @@ message GetSandboxConfigResponse { } // Connection details for one operator-registered supervisor middleware service. -// V1 supports only explicitly enabled plaintext gRPC for local development. +// V1 supports plaintext and server-authenticated TLS gRPC. message SupervisorMiddlewareService { // Operator-facing registration name used for diagnostics. string name = 1; // gRPC endpoint reachable from the sandbox supervisor. - string endpoint = 2; - // Explicit acknowledgement that request content is sent without TLS. - bool allow_insecure = 3; + string grpc_endpoint = 2; // Operator-owned body limit applied to every binding exposed by the service. uint64 max_body_bytes = 4; } From 3e69c5f4ac50977af3df89117aa07b357801427a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 17:31:09 -0700 Subject: [PATCH 15/59] fix(supervisor-middleware): keep sandbox startup resilient to middleware outages An unreachable operator-registered middleware service previously aborted sandbox startup via a hard error in load_policy, contradicting the per-request on_error contract and the resilient live-reload path. Retry the initial connect and, on failure, degrade to the built-in registry so matched requests are governed by each config's on_error (deny for fail_closed, allow for fail_open) instead of blocking the whole sandbox. The policy poll loop now reconciles the registry on every poll while an install is pending, so a recovered service is adopted without waiting for a config change; a failed reconcile also no longer blocks unrelated policy updates. Signed-off-by: Piotr Mlocek --- crates/openshell-sandbox/src/lib.rs | 159 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 50 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 5e640d73ff..25750ee14a 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -128,7 +128,7 @@ pub async fn run_sandbox( // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); - let (mut policy, opa_engine, retained_proto) = load_policy( + let (mut policy, opa_engine, retained_proto, middleware_install_pending) = load_policy( sandbox_id.clone(), sandbox, openshell_endpoint.clone(), @@ -423,6 +423,7 @@ pub async fn run_sandbox( ocsf_enabled: poll_ocsf_enabled, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, + middleware_install_pending, }; tokio::spawn(async move { @@ -1370,6 +1371,11 @@ async fn load_policy( SandboxPolicy, Option>, Option, + // True when operator-registered middleware could not be connected at + // startup and the engine kept the built-in registry. The policy poll loop + // retries the install so a recovered service is picked up without a config + // change. + bool, )> { // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { @@ -1399,7 +1405,8 @@ async fn load_policy( process: config.process, }; enrich_sandbox_baseline_paths(&mut policy); - return Ok((policy, Some(Arc::new(engine)), None)); + // File mode has no operator-registered middleware to connect. + return Ok((policy, Some(Arc::new(engine)), None, false)); } // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data @@ -1482,16 +1489,45 @@ async fn load_policy( // engine is rebuilt with the real PID for symlink resolution. info!("Creating OPA engine from proto policy data"); let engine = OpaEngine::from_proto(&proto_policy)?; - let middleware_registry = + // Connect operator-registered middleware services. A connect/describe + // failure must not abort sandbox startup: unlike the previous hard + // failure, we degrade to the built-in registry and let each request's + // `on_error` policy govern matched traffic (deny for fail_closed, allow + // for fail_open). The policy poll loop retries the install so a + // recovered service is picked up without a config change. This mirrors + // the resilient live-reload path. + let middleware_services = sandbox_config.supervisor_middleware_services.clone(); + let middleware_install_pending = match grpc_retry("Middleware connect", || { openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - sandbox_config.supervisor_middleware_services, + middleware_services.clone(), ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; + }) + .await + .and_then(|registry| engine.replace_middleware_registry(registry)) + { + Ok(()) => false, + Err(error) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + true + } + }; let opa_engine = Some(Arc::new(engine)); let policy = SandboxPolicy::try_from(proto_policy.clone())?; - return Ok((policy, opa_engine, Some(proto_policy))); + return Ok((policy, opa_engine, Some(proto_policy), middleware_install_pending)); } // No policy source available @@ -1627,6 +1663,10 @@ struct PolicyPollLoopContext { ocsf_enabled: Arc, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, + /// True when `load_policy` degraded to the built-in middleware registry + /// because operator services could not be connected at startup. The poll + /// loop retries the install until it succeeds. + middleware_install_pending: bool, } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { @@ -1639,6 +1679,10 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; let mut current_policy_hash = String::new(); let mut current_middleware_services = Vec::new(); + // Set when a middleware install is outstanding (degraded at startup or a + // failed reload). Drives a retry on every poll, independent of the config + // revision, so a recovered operator service is picked up promptly. + let mut middleware_sync_pending = ctx.middleware_install_pending; let mut current_settings: std::collections::HashMap< String, openshell_core::proto::EffectiveSetting, @@ -1674,14 +1718,70 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } }; + // Reconcile the supervisor middleware registry before evaluating the + // rest of the config. This runs independently of the config revision so + // an install that degraded at startup (or failed on an earlier poll) is + // retried here, letting a recovered operator service be picked up + // without waiting for a policy change. A failure keeps the + // last-known-good registry; the request path stays governed by each + // middleware's `on_error` policy, and a config change is still applied + // below rather than being blocked by the middleware outage. + if middleware_sync_pending + || result.supervisor_middleware_services != current_middleware_services + { + match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + result.supervisor_middleware_services.clone(), + ) + .await + .and_then(|registry| ctx.opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + current_middleware_services = result.supervisor_middleware_services.clone(); + middleware_sync_pending = false; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(current_middleware_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + current_middleware_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during a + // sustained outage. The startup degrade path emits its own + // event. + if !middleware_sync_pending { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + middleware_sync_pending = true; + } + } + } + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; if result.config_revision == current_config_revision && !provider_env_changed { continue; } let policy_changed = result.policy_hash != current_policy_hash; - let middleware_changed = - result.supervisor_middleware_services != current_middleware_services; // Log which settings changed. log_setting_changes(¤t_settings, &result.settings); @@ -1740,47 +1840,6 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } } - if middleware_changed { - match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - result.supervisor_middleware_services.clone(), - ) - .await - .and_then(|registry| ctx.opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - current_middleware_services = result.supervisor_middleware_services.clone(); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(current_middleware_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - current_middleware_services.len() - )) - .build() - ); - } - Err(error) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - continue; - } - } - } - // Only reload OPA when the policy payload actually changed. if policy_changed { let Some(policy) = result.policy.as_ref() else { From 9bdc7f7736937e2cce793c96f6a4ede1dbff2b3f Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 30 Jun 2026 17:31:17 -0700 Subject: [PATCH 16/59] fix(supervisor-middleware): ignore unresolved bindings in chain body limit A chain entry whose binding did not resolve reported a zero body limit, which dragged the whole chain's buffer cap to zero and spuriously failed body-bearing requests over capacity even when a resolved middleware could have processed them. Exclude unresolved entries from the limit via a new DescribedChainEntry::is_resolved(); when no entry resolves, skip buffering and apply each entry's on_error directly. Also fix two parallel-test flakes found while validating the change: - Build middleware OCSF events into a Vec and assert on it directly instead of capturing through the global tracing pipeline, whose callsite-interest cache is process-global and raced under parallel runs. - Accumulate the websocket deny response until the reason marker arrives rather than assuming a single read returns the full body. Signed-off-by: Piotr Mlocek --- .../src/lib.rs | 27 +++ .../src/l7/relay.rs | 201 ++++++++++++++---- 2 files changed, 188 insertions(+), 40 deletions(-) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 00324d75fd..27302dfe46 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -102,6 +102,14 @@ impl DescribedChainEntry { pub fn on_error(&self) -> OnError { self.entry.on_error } + + /// True when this entry resolved to a registered binding and will be + /// evaluated. When false, the binding is absent from the current registry + /// and the entry is handled entirely by its `on_error` policy, so it + /// imposes no body-buffering limit on the chain. + pub fn is_resolved(&self) -> bool { + self.binding.is_some() + } } #[derive(Debug, Clone)] @@ -1166,6 +1174,25 @@ mod tests { } } + #[tokio::test] + async fn describe_chain_marks_resolved_and_unresolved_entries() { + let unresolved = ChainEntry { + name: "missing".into(), + implementation: "third-party/missing".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }; + let described = ChainRunner::default() + .describe_chain(&[entry("redact", OnError::FailClosed), unresolved]) + .await + .expect("describe chain"); + // The built-in resolves and reports its real limit; the missing binding + // does not resolve and must not contribute a body limit. + assert!(described[0].is_resolved()); + assert_eq!(described[0].max_body_bytes(), 256 * 1024); + assert!(!described[1].is_resolved()); + } + #[tokio::test] async fn descriptors_are_resolved_from_any_middleware_service() { let runner = ChainRunner::new(Arc::new(ScriptedService { diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 84853f7517..3e92a32182 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -778,11 +778,18 @@ pub(crate) enum MiddlewareApplyResult { Denied(String), } +/// Smallest body-buffering limit across the entries that actually resolved to a +/// registered binding. Unresolved entries (`is_resolved() == false`) report a +/// zero limit and are excluded here: they are handled by their `on_error` policy +/// in `evaluate_described` without inspecting the body, so letting a zero drag +/// the chain limit to zero would spuriously fail the whole chain over capacity. +/// Returns `None` when no entry resolved, so the caller can skip buffering. fn middleware_chain_body_limit( chain: &[openshell_supervisor_middleware::DescribedChainEntry], ) -> Option { chain .iter() + .filter(|entry| entry.is_resolved()) .map(openshell_supervisor_middleware::DescribedChainEntry::max_body_bytes) .min() } @@ -812,8 +819,20 @@ pub(crate) async fn apply_middleware_chain_for_scheme crate::opa::NetworkInput { } } -fn emit_middleware_events( +/// Build the OCSF events describing a middleware chain outcome, in emission +/// order. Separated from `emit_middleware_events` so tests can assert on the +/// events deterministically without routing through the global tracing pipeline, +/// whose callsite-interest cache is process-global and races under parallel +/// tests. +fn middleware_events( ctx: &L7EvalContext, req: &crate::l7::provider::L7Request, outcome: &openshell_supervisor_middleware::ChainOutcome, -) { +) -> Vec { + let mut events = Vec::new(); for invocation in &outcome.applied { let allowed = invocation.decision == openshell_core::proto::Decision::Allow; let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -1000,7 +1025,7 @@ fn emit_middleware_events( invocation.failed )) .build(); - ocsf_emit!(event); + events.push(event); // A middleware that failed but was bypassed under `fail_open` is an // enforcement failure operators must be able to alert on, even though the @@ -1021,7 +1046,7 @@ fn emit_middleware_events( invocation.name )) .build(); - ocsf_emit!(event); + events.push(event); } } if !outcome.allowed && outcome.reason.starts_with("middleware_failed:") { @@ -1033,7 +1058,7 @@ fn emit_middleware_events( )) .message("Required supervisor middleware failed closed") .build(); - ocsf_emit!(event); + events.push(event); } for finding in &outcome.findings { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) @@ -1055,6 +1080,19 @@ fn emit_middleware_events( finding.finding.r#type, finding.finding.count )) .build(); + events.push(event); + } + events +} + +/// Emit the OCSF events describing a middleware chain outcome through the +/// tracing pipeline. +fn emit_middleware_events( + ctx: &L7EvalContext, + req: &crate::l7::provider::L7Request, + outcome: &openshell_supervisor_middleware::ChainOutcome, +) { + for event in middleware_events(ctx, req, outcome) { ocsf_emit!(event); } } @@ -3051,6 +3089,101 @@ network_policies: )); } + #[tokio::test] + async fn body_limit_ignores_unresolved_entries() { + use openshell_supervisor_middleware::{ChainEntry, ChainRunner, OnError}; + + let resolved = ChainEntry { + name: "redact".into(), + implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let unresolved = ChainEntry { + name: "missing".into(), + implementation: "third-party/missing".into(), + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }; + + // A single unresolved (0-limit) entry must not drag the chain limit to + // zero: the buffer limit reflects only the resolved built-in. + let mixed = ChainRunner::default() + .describe_chain(&[resolved, unresolved.clone()]) + .await + .expect("describe mixed chain"); + assert_eq!(middleware_chain_body_limit(&mixed), Some(256 * 1024)); + + // When nothing resolves, there is no body limit and the caller skips + // buffering entirely. + let none = ChainRunner::default() + .describe_chain(std::slice::from_ref(&unresolved)) + .await + .expect("describe unresolved chain"); + assert_eq!(middleware_chain_body_limit(&none), None); + } + + #[tokio::test] + async fn all_unresolved_fail_open_forwards_body_unbuffered() { + // A chain whose only entry is an unregistered binding has no resolvable + // body limit. Under fail_open the request must pass through with its + // body intact rather than being denied over a phantom zero-byte cap. + let (config, tunnel_engine, ctx) = + middleware_relay_context("third-party/missing", "fail_open"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"api_key":"sk-1234567890abcdef"}"#; + let request = format!( + "POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut upstream_request = [0u8; 1024]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_request), + ) + .await + .expect("request should reach upstream") + .unwrap(); + let upstream_request = String::from_utf8_lossy(&upstream_request[..n]); + // No middleware ran, so the body is forwarded verbatim. + assert!(upstream_request.contains(r#""api_key":"sk-1234567890abcdef""#)); + + upstream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut client_response = [0u8; 512]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + app.read(&mut client_response), + ) + .await + .expect("response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("204 No Content")); + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[test] fn middleware_keeps_the_raw_request_query() { let query = raw_query_from_request_headers( @@ -3095,36 +3228,14 @@ network_policies: assert_eq!(input.scheme, "http"); } - /// Tracing layer that captures emitted `OcsfEvent`s for assertions. - struct OcsfCaptureLayer(Arc>>); - - impl tracing_subscriber::Layer for OcsfCaptureLayer { - fn on_event( - &self, - event: &tracing::Event<'_>, - _ctx: tracing_subscriber::layer::Context<'_, S>, - ) { - if event.metadata().target() == openshell_ocsf::OCSF_TARGET - && let Some(ocsf_event) = openshell_ocsf::clone_current_event() - { - self.0.lock().unwrap().push(ocsf_event); - } - } - } - #[test] fn middleware_ocsf_events_are_audit_safe() { use openshell_supervisor_middleware::{ ChainOutcome, MiddlewareInvocation, NamespacedFinding, }; - use tracing_subscriber::layer::SubscriberExt; const RAW_SECRET: &str = "sk-RAWSECRETVALUE0123456789"; - let events = Arc::new(std::sync::Mutex::new(Vec::new())); - let subscriber = tracing_subscriber::registry().with(OcsfCaptureLayer(Arc::clone(&events))); - let _guard = tracing::subscriber::set_default(subscriber); - let ctx = L7EvalContext { host: "api.example.test".into(), port: 443, @@ -3171,23 +3282,26 @@ network_policies: }], }; - emit_middleware_events(&ctx, &req, &outcome); + // Build the events directly rather than routing through the global + // tracing pipeline: its callsite-interest cache is process-global, so a + // parallel test that emits OCSF with no subscriber installed can cache + // the callsite as disabled and make captured-event assertions flaky. + let events = middleware_events(&ctx, &req, &outcome); - let captured = events.lock().unwrap(); // Per-invocation decisions are HTTP Activity (class 4002). assert!( - captured.iter().any(|e| e.class_uid() == 4002), + events.iter().any(|e| e.class_uid() == 4002), "expected an HTTP Activity event for the middleware invocation" ); // Findings are Detection Finding (class 2004) with the finding's severity. - let finding_event = captured + let finding_event = events .iter() .find(|e| e.class_uid() == 2004) .expect("expected a Detection Finding event"); assert_eq!(finding_event.base().severity, SeverityId::Medium); // No raw payload material may appear in any emitted event. - let serialized = serde_json::to_string(&*captured).expect("serialize events"); + let serialized = serde_json::to_string(&events).expect("serialize events"); assert!( !serialized.contains(RAW_SECRET), "raw secret leaked into OCSF events: {serialized}" @@ -3364,12 +3478,19 @@ network_policies: .await .unwrap(); - let mut response = [0u8; 512]; - let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) - .await - .expect("denial should reach client") - .unwrap(); - let response = String::from_utf8_lossy(&response[..n]); + // Accumulate until the reason marker arrives: the deny response can be + // delivered in more than one write, so a single read may return only the + // status line and flake the body assertion. + let mut response = Vec::new(); + let mut buf = [0u8; 512]; + while !String::from_utf8_lossy(&response).contains("middleware_failed") { + match tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut buf)).await { + Ok(Ok(0)) | Err(_) => break, // clean EOF, or no more data before the deadline + Ok(Ok(n)) => response.extend_from_slice(&buf[..n]), + Ok(Err(e)) => panic!("read from relay failed: {e}"), + } + } + let response = String::from_utf8_lossy(&response); assert!(response.contains("403 Forbidden")); assert!(response.contains("middleware_failed")); From 9d4b6315e628fad3ac49840706dbbb91d2cb6d8b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 2 Jul 2026 13:16:45 -0700 Subject: [PATCH 17/59] fix(supervisor-middleware): harden policy enforcement Signed-off-by: Piotr Mlocek --- crates/openshell-sandbox/src/lib.rs | 7 +- crates/openshell-server/src/grpc/policy.rs | 32 ++++++ .../src/lib.rs | 98 ++++++++++++++++--- .../src/l7/relay.rs | 78 ++++++++++++++- .../src/l7/rest.rs | 29 +++++- 5 files changed, 228 insertions(+), 16 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 25750ee14a..a0fa859375 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1527,7 +1527,12 @@ async fn load_policy( let opa_engine = Some(Arc::new(engine)); let policy = SandboxPolicy::try_from(proto_policy.clone())?; - return Ok((policy, opa_engine, Some(proto_policy), middleware_install_pending)); + return Ok(( + policy, + opa_engine, + Some(proto_policy), + middleware_install_pending, + )); } // No policy source available diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 97d4598132..b1e37f87ed 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3135,6 +3135,18 @@ fn deterministic_policy_hash(policy: &ProtoSandboxPolicy) -> String { hasher.update(key.as_bytes()); hasher.update(value.encode_to_vec()); } + if !policy.network_middlewares.is_empty() { + hasher.update(b"network_middlewares"); + for middleware in &policy.network_middlewares { + let encoded = middleware.encode_to_vec(); + hasher.update( + u64::try_from(encoded.len()) + .expect("protobuf payload length fits in u64") + .to_le_bytes(), + ); + hasher.update(encoded); + } + } hex::encode(hasher.finalize()) } @@ -9315,6 +9327,26 @@ mod tests { assert_ne!(rev_a, rev_b); } + #[test] + fn policy_hash_changes_when_network_middlewares_change() { + let policy_a = ProtoSandboxPolicy::default(); + let policy_b = ProtoSandboxPolicy { + network_middlewares: vec![openshell_core::proto::NetworkMiddlewareConfig { + name: "redact-secrets".into(), + middleware: "openshell/secrets".into(), + on_error: "fail_closed".into(), + ..Default::default() + }], + ..Default::default() + }; + + assert_ne!( + deterministic_policy_hash(&policy_a), + deterministic_policy_hash(&policy_b), + "middleware-only policy changes must produce a new policy hash" + ); + } + #[test] fn config_revision_changes_when_policy_source_changes() { let policy = ProtoSandboxPolicy::default(); diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 27302dfe46..d2f8642b18 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -712,6 +712,37 @@ impl ChainRunner { } }; + if decision == Decision::Deny { + for finding in result.findings { + findings.push(NamespacedFinding { + middleware: entry.entry.name.clone(), + finding, + }); + } + if !result.metadata.is_empty() { + metadata.insert( + entry.entry.name.clone(), + result.metadata.into_iter().collect(), + ); + } + applied.push(MiddlewareInvocation { + name: entry.entry.name.clone(), + implementation: entry.entry.implementation.clone(), + decision, + transformed: false, + failed: false, + }); + return Ok(ChainOutcome { + allowed: false, + reason: safe_reason(&result.reason), + body, + added_headers, + findings, + metadata, + applied, + }); + } + if result.has_body && result.body.len() > entry.max_body_bytes { match apply_on_error(entry, "response_body_over_capacity", &mut applied) { OnErrorAction::FailOpen => continue, @@ -774,17 +805,6 @@ impl ChainRunner { transformed, failed: false, }); - if decision == Decision::Deny { - return Ok(ChainOutcome { - allowed: false, - reason: safe_reason(&result.reason), - body, - added_headers, - findings, - metadata, - applied, - }); - } } Ok(ChainOutcome { @@ -1399,6 +1419,62 @@ mod tests { assert!(!outcome.applied[0].failed); } + #[tokio::test] + async fn deny_decision_ignores_unsafe_mutations_under_fail_open() { + let runner = ChainRunner::new(Arc::new(scripted_service( + openshell_core::proto::HttpRequestResult { + decision: Decision::Deny as i32, + reason: "blocked_by_policy".into(), + add_headers: std::iter::once(( + "x-openshell-middleware-inject".to_string(), + "ok\r\nHost: evil".to_string(), + )) + .collect(), + ..allow_result() + }, + ))); + + let outcome = runner + .evaluate(&[entry("guard", OnError::FailOpen)], input("hello")) + .await + .expect("evaluate"); + + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "blocked_by_policy"); + assert!(outcome.added_headers.is_empty()); + assert_eq!(outcome.applied.len(), 1); + assert_eq!(outcome.applied[0].decision, Decision::Deny); + assert!(!outcome.applied[0].failed); + } + + #[tokio::test] + async fn deny_decision_ignores_oversized_replacement_under_fail_open() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + binding_id: BUILTIN_SECRETS.into(), + max_body_bytes: 4, + result: openshell_core::proto::HttpRequestResult { + decision: Decision::Deny as i32, + reason: "blocked_by_policy".into(), + body: b"too large".to_vec(), + has_body: true, + ..allow_result() + }, + })); + + let outcome = runner + .evaluate(&[entry("guard", OnError::FailOpen)], input("safe")) + .await + .expect("evaluate"); + + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "blocked_by_policy"); + assert_eq!(outcome.body, b"safe"); + assert_eq!(outcome.applied.len(), 1); + assert_eq!(outcome.applied[0].decision, Decision::Deny); + assert!(!outcome.applied[0].transformed); + assert!(!outcome.applied[0].failed); + } + #[tokio::test] async fn metadata_and_findings_are_namespaced_per_config() { let runner = ChainRunner::new(Arc::new(scripted_service( diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 3e92a32182..61daf33459 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -824,7 +824,14 @@ pub(crate) async fn apply_middleware_chain_for_scheme break, // clean EOF, or no more data before the deadline Ok(Ok(n)) => response.extend_from_slice(&buf[..n]), Ok(Err(e)) => panic!("read from relay failed: {e}"), diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 15825d1b23..4f2d37f087 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -828,7 +828,7 @@ pub(crate) enum BufferResult { OverCapacity { recoverable: bool }, } -pub(crate) async fn buffer_request_body_for_middleware( +pub(crate) async fn buffer_request_body_for_middleware( req: &L7Request, client: &mut C, generation_guard: Option<&PolicyGenerationGuard>, @@ -839,7 +839,7 @@ pub(crate) async fn buffer_request_body_for_middleware( .windows(4) .position(|w| w == b"\r\n\r\n") .map_or(req.raw_header.len(), |p| p + 4); - let headers = req.raw_header[..header_end].to_vec(); + let mut headers = req.raw_header[..header_end].to_vec(); let already_read = &req.raw_header[header_end..]; match req.body_length { BodyLength::None => Ok(BufferResult::Buffered(BufferedRequestBody { @@ -860,6 +860,9 @@ pub(crate) async fn buffer_request_body_for_middleware( let mut body = Vec::with_capacity(len); body.extend_from_slice(&already_read[..initial_len]); let mut remaining = len.saturating_sub(initial_len); + if remaining > 0 && already_read.is_empty() { + acknowledge_expect_continue(client, &mut headers).await?; + } let mut buf = [0u8; RELAY_BUF_SIZE]; while remaining > 0 { let to_read = remaining.min(buf.len()); @@ -887,6 +890,9 @@ pub(crate) async fn buffer_request_body_for_middleware( // we have already consumed wire bytes from the client stream and // cannot re-enter the normal raw relay path without a separate // splice-through buffer. + if already_read.is_empty() { + acknowledge_expect_continue(client, &mut headers).await?; + } Ok( collect_chunked_body(client, already_read, generation_guard, Some(max_body_bytes)) .await @@ -898,6 +904,25 @@ pub(crate) async fn buffer_request_body_for_middleware( } } +async fn acknowledge_expect_continue( + client: &mut C, + headers: &mut Vec, +) -> Result<()> { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + if !has_expect_continue(header_str) { + return Ok(()); + } + + client + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .await + .into_diagnostic()?; + client.flush().await.into_diagnostic()?; + *headers = strip_header(headers, "expect")?; + Ok(()) +} + pub(crate) fn rebuild_request_with_buffered_body( req: &L7Request, headers: &[u8], From 2b7cf4e1f9d4d70efa614998d8034637b9c8c90a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 2 Jul 2026 14:08:08 -0700 Subject: [PATCH 18/59] feat(ocsf): enrich middleware shorthand logs Signed-off-by: Piotr Mlocek --- crates/openshell-ocsf/src/format/shorthand.rs | 129 ++++++++++++++++-- docs/observability/logging.mdx | 9 +- 2 files changed, 125 insertions(+), 13 deletions(-) diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 3143b6d68e..8765f1e065 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -7,7 +7,22 @@ use crate::events::OcsfEvent; use crate::events::base_event::BaseEventData; -use crate::objects::Url; +use crate::objects::{Evidence, Url}; + +fn finding_evidence_value<'a>(evidences: Option<&'a [Evidence]>, key: &str) -> Option<&'a str> { + evidences? + .iter() + .filter_map(|evidence| evidence.data.as_ref()?.as_object()) + .find_map(|data| data.get(key)?.as_str()) +} + +fn message_bool_value<'a>(message: Option<&'a str>, key: &str) -> Option<&'a str> { + let prefix = format!("{key}="); + message? + .split_ascii_whitespace() + .find_map(|field| field.strip_prefix(&prefix)) + .filter(|value| matches!(*value, "true" | "false")) +} /// Format a timestamp (ms since epoch) as `HH:MM:SS.mmm`. /// @@ -195,10 +210,32 @@ impl OcsfEvent { .and_then(|r| r.url.as_ref()) .map(Url::to_display_string) .unwrap_or_default(); + let transformed = e + .firewall_rule + .as_ref() + .filter(|rule| rule.rule_type == "middleware") + .and_then(|_| message_bool_value(e.base.message.as_deref(), "transformed")); + let failed = e + .firewall_rule + .as_ref() + .filter(|rule| rule.rule_type == "middleware") + .and_then(|_| message_bool_value(e.base.message.as_deref(), "failed")); let rule_ctx = e .firewall_rule .as_ref() - .map(|r| format!(" [policy:{} engine:{}]", r.name, r.rule_type)) + .map(|r| { + let mut context = vec![ + format!("policy:{}", r.name), + format!("engine:{}", r.rule_type), + ]; + if let Some(value) = transformed { + context.push(format!("transformed:{value}")); + } + if let Some(value) = failed { + context.push(format!("failed:{value}")); + } + format!(" [{}]", context.join(" ")) + }) .unwrap_or_default(); // For denied events, surface the reason from status_detail let reason_ctx = if action == "DENIED" { @@ -280,16 +317,26 @@ impl OcsfEvent { } Self::DetectionFinding(e) => { - let disposition = e - .disposition - .map_or_else(|| "UNKNOWN".to_string(), |d| d.label().to_uppercase()); + let disposition = e.disposition.map_or_else( + || e.base.activity_name.to_uppercase(), + |d| d.label().to_uppercase(), + ); let title = &e.finding_info.title; - let confidence_ctx = e - .confidence - .map(|c| format!(" [confidence:{}]", c.label().to_lowercase())) - .unwrap_or_default(); - - format!("FINDING:{disposition} {sev} \"{title}\"{confidence_ctx}") + let mut context = vec![format!("type:{}", e.finding_info.uid)]; + if let Some(middleware) = + finding_evidence_value(e.evidences.as_deref(), "middleware") + { + context.push(format!("middleware:{middleware}")); + } + if let Some(count) = finding_evidence_value(e.evidences.as_deref(), "count") { + context.push(format!("count:{count}")); + } + if let Some(confidence) = e.confidence { + context.push(format!("confidence:{}", confidence.label().to_lowercase())); + } + let context = format!(" [{}]", context.join(" ")); + + format!("FINDING:{disposition} {sev} \"{title}\"{context}") } Self::ApplicationLifecycle(e) => { @@ -534,6 +581,37 @@ mod tests { ); } + #[test] + fn test_http_activity_shorthand_includes_middleware_outcome() { + let mut base = base(4002, "HTTP Activity", 4, "Network Activity", 99, "Other"); + base.set_message( + "MIDDLEWARE prototype-content-guard example/content-guard decision=Allow transformed=false failed=true", + ); + let event = OcsfEvent::HttpActivity(HttpActivityEvent { + base, + http_request: Some(HttpRequest::new( + "POST", + Url::new("http", "httpbin.org", "/anything", 443), + )), + http_response: None, + src_endpoint: None, + dst_endpoint: None, + proxy_endpoint: None, + actor: None, + firewall_rule: Some(FirewallRule::new("httpbin", "middleware")), + action: Some(ActionId::Allowed), + disposition: Some(DispositionId::Allowed), + observation_point_id: None, + is_src_dst_assignment_known: None, + }); + + let shorthand = event.format_shorthand(); + assert_eq!( + shorthand, + "HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware transformed:false failed:true]" + ); + } + #[test] fn test_network_activity_shorthand_denied_shows_reason() { let mut b = base(4001, "Network Activity", 4, "Network Activity", 1, "Open"); @@ -863,8 +941,35 @@ mod tests { let shorthand = event.format_shorthand(); assert_eq!( shorthand, - "FINDING:BLOCKED [HIGH] \"NSSH1 Nonce Replay Attack\" [confidence:high]" + "FINDING:BLOCKED [HIGH] \"NSSH1 Nonce Replay Attack\" [type:nssh1-replay-abc confidence:high]" + ); + } + + #[test] + fn test_detection_finding_shorthand_uses_activity_and_safe_evidence() { + let event = OcsfEvent::DetectionFinding(DetectionFindingEvent { + base: base(2004, "Detection Finding", 2, "Findings", 1, "Create"), + finding_info: FindingInfo::new("content_guard.match", "configured content matched"), + evidences: Some(vec![Evidence::from_pairs(&[ + ("middleware", "prototype-content-guard"), + ("count", "1"), + ("matched_content", "must-not-appear"), + ])]), + attacks: None, + remediation: None, + is_alert: None, + confidence: None, + risk_level: None, + action: None, + disposition: None, + }); + + let shorthand = event.format_shorthand(); + assert_eq!( + shorthand, + "FINDING:CREATE [INFO] \"configured content matched\" [type:content_guard.match middleware:prototype-content-guard count:1]" ); + assert!(!shorthand.contains("must-not-appear")); } #[test] diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index dcfe9f19d4..2066181071 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -94,7 +94,7 @@ CLASS:ACTIVITY [SEVERITY] ACTION DETAILS [CONTEXT] - SSH: peer address and authentication type - Process: `name(pid)` with exit code or command line - Config: description of what changed -- Finding: quoted title with confidence level +- Finding: quoted title with the stable finding type, optional confidence, and allowlisted evidence fields when available **Context** in brackets at the end provides the policy rule and enforcement engine that produced the decision. @@ -130,6 +130,13 @@ An HTTP request to a non-default port. HTTP log URLs include the port whenever i OCSF HTTP:GET [INFO] ALLOWED GET http://api.internal.corp:8080/v1/status [policy:internal_api engine:opa] ``` +A supervisor middleware HTTP event records whether it transformed the request. If the middleware also emits a finding, that remains a separate event: + +```text +OCSF HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware transformed:true failed:false] +OCSF FINDING:CREATE [MED] "configured content matched" [type:content_guard.match middleware:prototype-content-guard count:1] +``` + Proxy and SSH servers ready: ```text From 0ef948bff0b16405ecb7e6b44d43520ab40df519 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 2 Jul 2026 15:34:52 -0700 Subject: [PATCH 19/59] fix(policy): initialize network middleware test fixtures Signed-off-by: Piotr Mlocek --- crates/openshell-policy/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 29585a6233..efcca2fcaa 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -2678,6 +2678,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), + network_middlewares: Vec::new(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } @@ -2693,6 +2694,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), + network_middlewares: Vec::new(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } @@ -2771,6 +2773,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), + network_middlewares: Vec::new(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } From 400ca0fa6c87a2ac8580aa1826f37ebca7c42cb6 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 2 Jul 2026 16:39:23 -0700 Subject: [PATCH 20/59] fix(policy): preserve immutable validation precedence Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/grpc/policy.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index b1e37f87ed..21d7e02540 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1848,12 +1848,14 @@ async fn handle_update_config_inner( validate_no_reserved_provider_policy_keys(&new_policy)?; } + if let Some(baseline_policy) = spec.policy.as_ref() { + validate_static_fields_unchanged(baseline_policy, &new_policy)?; + } + validate_policy_safety(&new_policy)?; crate::middleware::validate_policy(state.middleware_registry.as_ref(), &new_policy).await?; - if let Some(baseline_policy) = spec.policy.as_ref() { - validate_static_fields_unchanged(baseline_policy, &new_policy)?; - } else { + if spec.policy.is_none() { // Backfill spec.policy using CAS (first-time policy discovery) let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; let sandbox_id = sandbox.object_id().to_string(); From 1f6aec16e84bc6b7ff9777ae7f87483d7bb38624 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 2 Jul 2026 18:00:20 -0700 Subject: [PATCH 21/59] feat(ocsf): log middleware denial reasons Signed-off-by: Piotr Mlocek --- crates/openshell-ocsf/src/format/shorthand.rs | 76 +++++++++---------- .../src/l7/relay.rs | 42 +++++++++- docs/extensibility/supervisor-middleware.mdx | 1 + docs/observability/logging.mdx | 4 +- proto/middleware.proto | 4 +- 5 files changed, 84 insertions(+), 43 deletions(-) diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 8765f1e065..22bd1759b1 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -97,22 +97,20 @@ fn truncate_with_ellipsis(text: &str, max: usize) -> String { format!("{}...", &text[..end]) } -/// Format a `[reason:...]` tag from `status_detail` (or `message` fallback) -/// for denied events. Returns an empty string if neither field is set. -fn reason_tag(base: &BaseEventData) -> String { - let text = base - .status_detail - .as_deref() - .or(base.message.as_deref()) - .unwrap_or(""); +fn reason_text(text: Option<&str>) -> Option { + let text = text?; if text.is_empty() { - return String::new(); + return None; } let text = text.replace(['\n', '\r'], " "); - format!( - " [reason:{}]", - truncate_with_ellipsis(&text, MAX_REASON_LEN) - ) + Some(truncate_with_ellipsis(&text, MAX_REASON_LEN)) +} + +/// Format a `[reason:...]` tag from `status_detail` (or `message` fallback) +/// for denied events. Returns an empty string if neither field is set. +fn reason_tag(base: &BaseEventData) -> String { + reason_text(base.status_detail.as_deref().or(base.message.as_deref())) + .map_or_else(String::new, |text| format!(" [reason:{text}]")) } fn message_tag(base: &BaseEventData) -> String { @@ -210,35 +208,37 @@ impl OcsfEvent { .and_then(|r| r.url.as_ref()) .map(Url::to_display_string) .unwrap_or_default(); - let transformed = e + let is_middleware = e .firewall_rule .as_ref() - .filter(|rule| rule.rule_type == "middleware") - .and_then(|_| message_bool_value(e.base.message.as_deref(), "transformed")); - let failed = e - .firewall_rule - .as_ref() - .filter(|rule| rule.rule_type == "middleware") - .and_then(|_| message_bool_value(e.base.message.as_deref(), "failed")); + .is_some_and(|rule| rule.rule_type == "middleware"); let rule_ctx = e .firewall_rule .as_ref() - .map(|r| { - let mut context = vec![ - format!("policy:{}", r.name), - format!("engine:{}", r.rule_type), - ]; - if let Some(value) = transformed { - context.push(format!("transformed:{value}")); - } - if let Some(value) = failed { - context.push(format!("failed:{value}")); - } - format!(" [{}]", context.join(" ")) - }) + .map(|r| format!(" [policy:{} engine:{}]", r.name, r.rule_type)) .unwrap_or_default(); - // For denied events, surface the reason from status_detail - let reason_ctx = if action == "DENIED" { + let outcome_ctx = if is_middleware { + let mut context = Vec::new(); + if let Some(value) = + message_bool_value(e.base.message.as_deref(), "transformed") + { + context.push(format!("transformed:{value}")); + } + if let Some(value) = message_bool_value(e.base.message.as_deref(), "failed") { + context.push(format!("failed:{value}")); + } + if action == "DENIED" + && let Some(reason) = reason_text(e.base.status_detail.as_deref()) + { + // Keep the free-form reason last so the preceding fields remain easy to parse. + context.push(format!("reason:{reason}")); + } + if context.is_empty() { + String::new() + } else { + format!(" [{}]", context.join(" ")) + } + } else if action == "DENIED" { reason_tag(&e.base) } else { String::new() @@ -255,7 +255,7 @@ impl OcsfEvent { (false, true) => format!(" {action}"), (false, false) => format!(" {action}{arrow}"), }; - format!("HTTP:{method} {sev}{detail}{rule_ctx}{reason_ctx}") + format!("HTTP:{method} {sev}{detail}{rule_ctx}{outcome_ctx}") } Self::SshActivity(e) => { @@ -608,7 +608,7 @@ mod tests { let shorthand = event.format_shorthand(); assert_eq!( shorthand, - "HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware transformed:false failed:true]" + "HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware] [transformed:false failed:true]" ); } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 61daf33459..29bbece568 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1000,7 +1000,7 @@ fn middleware_events( let mut events = Vec::new(); for invocation in &outcome.applied { let allowed = invocation.decision == openshell_core::proto::Decision::Allow; - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .action(if allowed { ActionId::Allowed @@ -1030,8 +1030,13 @@ fn middleware_events( invocation.decision, invocation.transformed, invocation.failed - )) - .build(); + )); + if !allowed && !outcome.reason.is_empty() { + event = event + .status(StatusId::Failure) + .status_detail(&outcome.reason); + } + let event = event.build(); events.push(event); // A middleware that failed but was bypassed under `fail_open` is an @@ -3381,6 +3386,37 @@ network_policies: ); // Safe finding metadata is still present. assert!(serialized.contains("secret.common")); + + let denied_outcome = ChainOutcome { + allowed: false, + reason: "request matched configured policy".into(), + body: Vec::new(), + added_headers: BTreeMap::new(), + findings: Vec::new(), + metadata: BTreeMap::new(), + applied: vec![MiddlewareInvocation { + name: "content-guard".into(), + implementation: "example/content-guard".into(), + decision: openshell_core::proto::Decision::Deny, + transformed: false, + failed: false, + }], + }; + let denied_events = middleware_events(&ctx, &req, &denied_outcome); + let denied_http = denied_events + .iter() + .find(|event| event.class_uid() == 4002) + .expect("expected denied HTTP Activity event"); + assert_eq!( + denied_http.base().status_detail.as_deref(), + Some("request matched configured policy") + ); + assert_eq!( + denied_http.format_shorthand(), + "HTTP:POST [MED] DENIED POST http://api.example.test:443/v1/messages \ + [policy:rest_api engine:middleware] \ + [transformed:false failed:false reason:request matched configured policy]" + ); } #[tokio::test] diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 0ebd518bd9..f2ea0c98a9 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -120,6 +120,7 @@ When the effective sandbox configuration changes, a running supervisor validates Middleware activity is emitted through OpenShell's OCSF logging: - Each invocation records its policy-local middleware name, binding, decision, transformation state, and failure state. +- A denied invocation records the service-provided audit-safe reason without request content, configured terms, credentials, or other secrets. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. - Findings include the service-provided type and label plus aggregate counts. Middleware services should keep those fields audit-safe and omit request content or matched values. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 2066181071..31281de881 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -133,7 +133,7 @@ OCSF HTTP:GET [INFO] ALLOWED GET http://api.internal.corp:8080/v1/status [policy A supervisor middleware HTTP event records whether it transformed the request. If the middleware also emits a finding, that remains a separate event: ```text -OCSF HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware transformed:true failed:false] +OCSF HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware] [transformed:true failed:false] OCSF FINDING:CREATE [MED] "configured content matched" [type:content_guard.match middleware:prototype-content-guard count:1] ``` @@ -167,6 +167,8 @@ OCSF CONFIG:LOADED [INFO] Policy reloaded successfully [policy_hash:0cc0c2b52557 Denied `NET:` and `HTTP:` events carry a `[reason:...]` suffix that surfaces the decision detail from the event's `status_detail` field. The reason helps distinguish between policy misses, SSRF hardening, and L7 enforcement without inspecting the full OCSF JSONL record. +For supervisor middleware denials, `status_detail` contains the service-provided audit-safe reason from the gRPC response. + Common reason phrases emitted by the sandbox include: | Reason | Meaning | diff --git a/proto/middleware.proto b/proto/middleware.proto index 9b988b930e..7a4eb28dfe 100644 --- a/proto/middleware.proto +++ b/proto/middleware.proto @@ -148,7 +148,9 @@ message Finding { message HttpRequestResult { // Allow or deny decision for this request. Decision decision = 1; - // Human-readable reason used for diagnostics and denied responses. + // Audit-safe human-readable reason used for diagnostics, denied responses, + // and security logs. Must not contain request content, configured terms, + // credentials, or other secrets. string reason = 2; // Replacement request body when has_body is true. bytes body = 3; From 6fdc5e3e736d068231278b2a081aceb8860881e8 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 11:43:26 -0700 Subject: [PATCH 22/59] refactor(ocsf): decouple shorthand from middleware Signed-off-by: Piotr Mlocek --- crates/openshell-ocsf/src/builders/finding.rs | 14 ++ crates/openshell-ocsf/src/builders/http.rs | 16 +++ crates/openshell-ocsf/src/format/shorthand.rs | 123 ++++++++---------- .../src/l7/relay.rs | 19 ++- docs/observability/logging.mdx | 8 +- 5 files changed, 101 insertions(+), 79 deletions(-) diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index 177c9159df..fecb1e9275 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -25,6 +25,7 @@ pub struct DetectionFindingBuilder<'a> { risk_level: Option, message: Option, log_source: Option, + unmapped: serde_json::Map, } impl<'a> DetectionFindingBuilder<'a> { @@ -45,6 +46,7 @@ impl<'a> DetectionFindingBuilder<'a> { risk_level: None, message: None, log_source: None, + unmapped: serde_json::Map::new(), } } @@ -74,6 +76,13 @@ impl<'a> DetectionFindingBuilder<'a> { self } + /// Add a source-specific attribute that is not defined by the OCSF class. + #[must_use] + pub fn unmapped(mut self, key: &str, value: impl Into) -> Self { + self.unmapped.insert(key.to_string(), value.into()); + self + } + /// Add a remediation description. #[must_use] pub fn remediation(mut self, desc: &str) -> Self { @@ -122,6 +131,9 @@ impl<'a> DetectionFindingBuilder<'a> { self.severity, metadata, ); + if !self.unmapped.is_empty() { + base.unmapped = Some(serde_json::Value::Object(self.unmapped)); + } self.ctx.apply_common_fields(&mut base, None, self.message); OcsfEvent::DetectionFinding(DetectionFindingEvent { @@ -169,6 +181,7 @@ mod tests { .is_alert(true) .confidence(ConfidenceId::High) .risk_level(RiskLevelId::High) + .unmapped("source_rule", "nonce_replay") .finding_info( FindingInfo::new("nssh1-replay-abc", "NSSH1 Nonce Replay Attack") .with_desc("A nonce was replayed."), @@ -188,5 +201,6 @@ mod tests { assert_eq!(json["finding_info"]["title"], "NSSH1 Nonce Replay Attack"); assert_eq!(json["is_alert"], true); assert_eq!(json["confidence"], "High"); + assert_eq!(json["unmapped"]["source_rule"], "nonce_replay"); } } diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index 4fa33eb1ca..ae91cf7c41 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -25,6 +25,7 @@ pub struct HttpActivityBuilder<'a> { firewall_rule: Option, message: Option, status_detail: Option, + unmapped: serde_json::Map, } impl<'a> HttpActivityBuilder<'a> { @@ -45,6 +46,7 @@ impl<'a> HttpActivityBuilder<'a> { firewall_rule: None, message: None, status_detail: None, + unmapped: serde_json::Map::new(), } } @@ -69,6 +71,13 @@ impl<'a> HttpActivityBuilder<'a> { self } + /// Add a source-specific attribute that is not defined by the OCSF class. + #[must_use] + pub fn unmapped(mut self, key: &str, value: impl Into) -> Self { + self.unmapped.insert(key.to_string(), value.into()); + self + } + #[must_use] pub fn build(self) -> OcsfEvent { let activity_name = self.activity.http_label().to_string(); @@ -86,6 +95,9 @@ impl<'a> HttpActivityBuilder<'a> { if let Some(detail) = self.status_detail { base.set_status_detail(detail); } + if !self.unmapped.is_empty() { + base.unmapped = Some(serde_json::Value::Object(self.unmapped)); + } self.ctx .apply_common_fields(&mut base, self.status, self.message); @@ -157,11 +169,15 @@ mod tests { .firewall_rule("aws_iam", "ssrf") .message("FORWARD blocked: allowed_ips check failed") .status_detail("resolves to always-blocked address") + .unmapped("attempt", 2) + .unmapped("cached", true) .build(); let json = event.to_json().unwrap(); assert_eq!(json["class_uid"], 4002); assert_eq!(json["status_detail"], "resolves to always-blocked address"); + assert_eq!(json["unmapped"]["attempt"], 2); + assert_eq!(json["unmapped"]["cached"], true); assert_eq!(json["action_id"], 2); // Denied } } diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 22bd1759b1..5e5f932448 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -7,22 +7,7 @@ use crate::events::OcsfEvent; use crate::events::base_event::BaseEventData; -use crate::objects::{Evidence, Url}; - -fn finding_evidence_value<'a>(evidences: Option<&'a [Evidence]>, key: &str) -> Option<&'a str> { - evidences? - .iter() - .filter_map(|evidence| evidence.data.as_ref()?.as_object()) - .find_map(|data| data.get(key)?.as_str()) -} - -fn message_bool_value<'a>(message: Option<&'a str>, key: &str) -> Option<&'a str> { - let prefix = format!("{key}="); - message? - .split_ascii_whitespace() - .find_map(|field| field.strip_prefix(&prefix)) - .filter(|value| matches!(*value, "true" | "false")) -} +use crate::objects::Url; /// Format a timestamp (ms since epoch) as `HH:MM:SS.mmm`. /// @@ -113,6 +98,43 @@ fn reason_tag(base: &BaseEventData) -> String { .map_or_else(String::new, |text| format!(" [reason:{text}]")) } +fn unmapped_fields(base: &BaseEventData) -> Vec { + base.unmapped + .as_ref() + .and_then(serde_json::Value::as_object) + .into_iter() + .flatten() + .filter_map(|(key, value)| { + let value = match value { + serde_json::Value::Bool(value) => value.to_string(), + serde_json::Value::Number(value) => value.to_string(), + serde_json::Value::String(value) => { + let value = value.replace(['\n', '\r'], " "); + truncate_with_ellipsis(&value, MAX_REASON_LEN) + } + _ => return None, + }; + Some(format!("{key}:{value}")) + }) + .collect() +} + +fn unmapped_context(base: &BaseEventData, include_reason: bool) -> String { + let mut fields = unmapped_fields(base); + + if include_reason + && let Some(reason) = reason_text(base.status_detail.as_deref().or(base.message.as_deref())) + { + fields.push(format!("reason:{reason}")); + } + + if fields.is_empty() { + String::new() + } else { + format!(" [{}]", fields.join(" ")) + } +} + fn message_tag(base: &BaseEventData) -> String { let text = base.message.as_deref().unwrap_or(""); if text.is_empty() { @@ -208,41 +230,12 @@ impl OcsfEvent { .and_then(|r| r.url.as_ref()) .map(Url::to_display_string) .unwrap_or_default(); - let is_middleware = e - .firewall_rule - .as_ref() - .is_some_and(|rule| rule.rule_type == "middleware"); let rule_ctx = e .firewall_rule .as_ref() .map(|r| format!(" [policy:{} engine:{}]", r.name, r.rule_type)) .unwrap_or_default(); - let outcome_ctx = if is_middleware { - let mut context = Vec::new(); - if let Some(value) = - message_bool_value(e.base.message.as_deref(), "transformed") - { - context.push(format!("transformed:{value}")); - } - if let Some(value) = message_bool_value(e.base.message.as_deref(), "failed") { - context.push(format!("failed:{value}")); - } - if action == "DENIED" - && let Some(reason) = reason_text(e.base.status_detail.as_deref()) - { - // Keep the free-form reason last so the preceding fields remain easy to parse. - context.push(format!("reason:{reason}")); - } - if context.is_empty() { - String::new() - } else { - format!(" [{}]", context.join(" ")) - } - } else if action == "DENIED" { - reason_tag(&e.base) - } else { - String::new() - }; + let outcome_ctx = unmapped_context(&e.base, action == "DENIED"); let arrow = if actor_str.is_empty() { format!(" {method} {url_str}") } else { @@ -323,14 +316,7 @@ impl OcsfEvent { ); let title = &e.finding_info.title; let mut context = vec![format!("type:{}", e.finding_info.uid)]; - if let Some(middleware) = - finding_evidence_value(e.evidences.as_deref(), "middleware") - { - context.push(format!("middleware:{middleware}")); - } - if let Some(count) = finding_evidence_value(e.evidences.as_deref(), "count") { - context.push(format!("count:{count}")); - } + context.extend(unmapped_fields(&e.base)); if let Some(confidence) = e.confidence { context.push(format!("confidence:{}", confidence.label().to_lowercase())); } @@ -582,11 +568,10 @@ mod tests { } #[test] - fn test_http_activity_shorthand_includes_middleware_outcome() { + fn test_http_activity_shorthand_includes_unmapped_attributes() { let mut base = base(4002, "HTTP Activity", 4, "Network Activity", 99, "Other"); - base.set_message( - "MIDDLEWARE prototype-content-guard example/content-guard decision=Allow transformed=false failed=true", - ); + base.add_unmapped("attempt", serde_json::json!(2)); + base.add_unmapped("cached", serde_json::json!(true)); let event = OcsfEvent::HttpActivity(HttpActivityEvent { base, http_request: Some(HttpRequest::new( @@ -598,7 +583,7 @@ mod tests { dst_endpoint: None, proxy_endpoint: None, actor: None, - firewall_rule: Some(FirewallRule::new("httpbin", "middleware")), + firewall_rule: Some(FirewallRule::new("httpbin", "extension")), action: Some(ActionId::Allowed), disposition: Some(DispositionId::Allowed), observation_point_id: None, @@ -608,7 +593,7 @@ mod tests { let shorthand = event.format_shorthand(); assert_eq!( shorthand, - "HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware] [transformed:false failed:true]" + "HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:extension] [attempt:2 cached:true]" ); } @@ -946,15 +931,17 @@ mod tests { } #[test] - fn test_detection_finding_shorthand_uses_activity_and_safe_evidence() { + fn test_detection_finding_shorthand_uses_activity_and_safe_unmapped_attributes() { + let mut base = base(2004, "Detection Finding", 2, "Findings", 1, "Create"); + base.add_unmapped("count", serde_json::json!(1)); + base.add_unmapped("source", serde_json::json!("content_guard")); let event = OcsfEvent::DetectionFinding(DetectionFindingEvent { - base: base(2004, "Detection Finding", 2, "Findings", 1, "Create"), + base, finding_info: FindingInfo::new("content_guard.match", "configured content matched"), - evidences: Some(vec![Evidence::from_pairs(&[ - ("middleware", "prototype-content-guard"), - ("count", "1"), - ("matched_content", "must-not-appear"), - ])]), + evidences: Some(vec![Evidence::from_pairs(&[( + "matched_content", + "must-not-appear", + )])]), attacks: None, remediation: None, is_alert: None, @@ -967,7 +954,7 @@ mod tests { let shorthand = event.format_shorthand(); assert_eq!( shorthand, - "FINDING:CREATE [INFO] \"configured content matched\" [type:content_guard.match middleware:prototype-content-guard count:1]" + "FINDING:CREATE [INFO] \"configured content matched\" [type:content_guard.match count:1 source:content_guard]" ); assert!(!shorthand.contains("must-not-appear")); } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 29bbece568..329d4b42da 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1023,13 +1023,11 @@ fn middleware_events( )) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "middleware") + .unmapped("transformed", invocation.transformed) + .unmapped("failed", invocation.failed) .message(format!( - "MIDDLEWARE {} {} decision={:?} transformed={} failed={}", - invocation.name, - invocation.implementation, - invocation.decision, - invocation.transformed, - invocation.failed + "MIDDLEWARE {} {} decision={:?}", + invocation.name, invocation.implementation, invocation.decision )); if !allowed && !outcome.reason.is_empty() { event = event @@ -1053,6 +1051,8 @@ fn middleware_events( ("middleware", invocation.name.as_str()), ("implementation", invocation.implementation.as_str()), ]) + .unmapped("middleware", invocation.name.as_str()) + .unmapped("implementation", invocation.implementation.as_str()) .message(format!( "Middleware {} failed and was bypassed (fail_open)", invocation.name @@ -1087,6 +1087,8 @@ fn middleware_events( ("middleware", &finding.middleware), ("count", &finding.finding.count.to_string()), ]) + .unmapped("middleware", finding.middleware.as_str()) + .unmapped("count", finding.finding.count) .message(format!( "Middleware finding {} count={}", finding.finding.r#type, finding.finding.count @@ -3411,11 +3413,14 @@ network_policies: denied_http.base().status_detail.as_deref(), Some("request matched configured policy") ); + let denied_json = denied_http.to_json().expect("serialize denied event"); + assert_eq!(denied_json["unmapped"]["transformed"], false); + assert_eq!(denied_json["unmapped"]["failed"], false); assert_eq!( denied_http.format_shorthand(), "HTTP:POST [MED] DENIED POST http://api.example.test:443/v1/messages \ [policy:rest_api engine:middleware] \ - [transformed:false failed:false reason:request matched configured policy]" + [failed:false transformed:false reason:request matched configured policy]" ); } diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 31281de881..6cd24c70e2 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -94,9 +94,9 @@ CLASS:ACTIVITY [SEVERITY] ACTION DETAILS [CONTEXT] - SSH: peer address and authentication type - Process: `name(pid)` with exit code or command line - Config: description of what changed -- Finding: quoted title with the stable finding type, optional confidence, and allowlisted evidence fields when available +- Finding: quoted title with the stable finding type, optional confidence, and source-specific context attributes when available -**Context** in brackets at the end provides the policy rule and enforcement engine that produced the decision. +**Context** in brackets provides structured fields such as policy provenance, source-specific attributes, and denial reasons. ### Examples @@ -133,8 +133,8 @@ OCSF HTTP:GET [INFO] ALLOWED GET http://api.internal.corp:8080/v1/status [policy A supervisor middleware HTTP event records whether it transformed the request. If the middleware also emits a finding, that remains a separate event: ```text -OCSF HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware] [transformed:true failed:false] -OCSF FINDING:CREATE [MED] "configured content matched" [type:content_guard.match middleware:prototype-content-guard count:1] +OCSF HTTP:POST [INFO] ALLOWED POST http://httpbin.org:443/anything [policy:httpbin engine:middleware] [failed:false transformed:true] +OCSF FINDING:CREATE [MED] "configured content matched" [type:content_guard.match count:1 middleware:prototype-content-guard] ``` Proxy and SSH servers ready: From d27085cb73566d9588cc3f7f202090aa25af926b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 12:08:51 -0700 Subject: [PATCH 23/59] refactor(supervisor-middleware): clarify runtime integration Signed-off-by: Piotr Mlocek --- crates/openshell-sandbox/src/lib.rs | 176 +++++---- .../src/l7/middleware.rs | 353 ++++++++++++++++++ .../src/l7/mod.rs | 1 + .../src/l7/relay.rs | 353 +----------------- .../openshell-supervisor-network/src/opa.rs | 117 +----- .../openshell-supervisor-network/src/proxy.rs | 6 +- 6 files changed, 467 insertions(+), 539 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/l7/middleware.rs diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index a0fa859375..b8c4eeb9a5 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -128,7 +128,7 @@ pub async fn run_sandbox( // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); - let (mut policy, opa_engine, retained_proto, middleware_install_pending) = load_policy( + let (mut policy, opa_engine, retained_proto, middleware_registry_status) = load_policy( sandbox_id.clone(), sandbox, openshell_endpoint.clone(), @@ -423,7 +423,7 @@ pub async fn run_sandbox( ocsf_enabled: poll_ocsf_enabled, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, - middleware_install_pending, + middleware_registry_status, }; tokio::spawn(async move { @@ -1371,11 +1371,7 @@ async fn load_policy( SandboxPolicy, Option>, Option, - // True when operator-registered middleware could not be connected at - // startup and the engine kept the built-in registry. The policy poll loop - // retries the install so a recovered service is picked up without a config - // change. - bool, + MiddlewareRegistryStatus, )> { // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { @@ -1406,7 +1402,12 @@ async fn load_policy( }; enrich_sandbox_baseline_paths(&mut policy); // File mode has no operator-registered middleware to connect. - return Ok((policy, Some(Arc::new(engine)), None, false)); + return Ok(( + policy, + Some(Arc::new(engine)), + None, + MiddlewareRegistryStatus::Synchronized, + )); } // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data @@ -1490,14 +1491,11 @@ async fn load_policy( info!("Creating OPA engine from proto policy data"); let engine = OpaEngine::from_proto(&proto_policy)?; // Connect operator-registered middleware services. A connect/describe - // failure must not abort sandbox startup: unlike the previous hard - // failure, we degrade to the built-in registry and let each request's - // `on_error` policy govern matched traffic (deny for fail_closed, allow - // for fail_open). The policy poll loop retries the install so a - // recovered service is picked up without a config change. This mirrors - // the resilient live-reload path. + // failure keeps the built-in registry active so each request's + // `on_error` policy governs matched traffic. The policy poll loop + // retries the install without waiting for a config change. let middleware_services = sandbox_config.supervisor_middleware_services.clone(); - let middleware_install_pending = match grpc_retry("Middleware connect", || { + let middleware_registry_status = match grpc_retry("Middleware connect", || { openshell_supervisor_middleware::MiddlewareRegistry::connect_services( middleware_services.clone(), ) @@ -1505,7 +1503,7 @@ async fn load_policy( .await .and_then(|registry| engine.replace_middleware_registry(registry)) { - Ok(()) => false, + Ok(()) => MiddlewareRegistryStatus::Synchronized, Err(error) => { ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) @@ -1521,7 +1519,7 @@ async fn load_policy( )) .build() ); - true + MiddlewareRegistryStatus::NeedsReconciliation } }; let opa_engine = Some(Arc::new(engine)); @@ -1531,7 +1529,7 @@ async fn load_policy( policy, opa_engine, Some(proto_policy), - middleware_install_pending, + middleware_registry_status, )); } @@ -1651,6 +1649,12 @@ fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::S } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiddlewareRegistryStatus { + Synchronized, + NeedsReconciliation, +} + /// Background loop that polls the server for policy updates. /// /// When a new version is detected, attempts to reload the OPA engine via @@ -1668,10 +1672,65 @@ struct PolicyPollLoopContext { ocsf_enabled: Arc, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, - /// True when `load_policy` degraded to the built-in middleware registry - /// because operator services could not be connected at startup. The poll - /// loop retries the install until it succeeds. - middleware_install_pending: bool, + middleware_registry_status: MiddlewareRegistryStatus, +} + +async fn reconcile_middleware_registry( + opa_engine: &OpaEngine, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + current_services: &mut Vec, + status: &mut MiddlewareRegistryStatus, +) { + if *status == MiddlewareRegistryStatus::Synchronized + && desired_services == current_services.as_slice() + { + return; + } + + match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + desired_services.to_vec(), + ) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + current_services.clear(); + current_services.extend_from_slice(desired_services); + *status = MiddlewareRegistryStatus::Synchronized; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(current_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + current_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during an outage. + if *status == MiddlewareRegistryStatus::Synchronized { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + *status = MiddlewareRegistryStatus::NeedsReconciliation; + } + } } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { @@ -1684,10 +1743,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; let mut current_policy_hash = String::new(); let mut current_middleware_services = Vec::new(); - // Set when a middleware install is outstanding (degraded at startup or a - // failed reload). Drives a retry on every poll, independent of the config - // revision, so a recovered operator service is picked up promptly. - let mut middleware_sync_pending = ctx.middleware_install_pending; + let mut middleware_registry_status = ctx.middleware_registry_status; let mut current_settings: std::collections::HashMap< String, openshell_core::proto::EffectiveSetting, @@ -1723,63 +1779,17 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } }; - // Reconcile the supervisor middleware registry before evaluating the - // rest of the config. This runs independently of the config revision so - // an install that degraded at startup (or failed on an earlier poll) is - // retried here, letting a recovered operator service be picked up - // without waiting for a policy change. A failure keeps the - // last-known-good registry; the request path stays governed by each - // middleware's `on_error` policy, and a config change is still applied - // below rather than being blocked by the middleware outage. - if middleware_sync_pending - || result.supervisor_middleware_services != current_middleware_services - { - match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - result.supervisor_middleware_services.clone(), - ) - .await - .and_then(|registry| ctx.opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - current_middleware_services = result.supervisor_middleware_services.clone(); - middleware_sync_pending = false; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(current_middleware_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - current_middleware_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during a - // sustained outage. The startup degrade path emits its own - // event. - if !middleware_sync_pending { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - middleware_sync_pending = true; - } - } - } + // Reconcile independently of the config revision so a recovered + // operator service is picked up without waiting for a policy change. + // Failure preserves the last-known-good registry and does not block + // the remaining config updates below. + reconcile_middleware_registry( + &ctx.opa_engine, + &result.supervisor_middleware_services, + &mut current_middleware_services, + &mut middleware_registry_status, + ) + .await; let provider_env_changed = result.provider_env_revision != current_provider_env_revision; if result.config_revision == current_config_revision && !provider_env_changed { diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs new file mode 100644 index 0000000000..d8a8b6c300 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor middleware application for L7 requests. + +use crate::l7::relay::L7EvalContext; +use crate::opa::PolicyGenerationGuard; +use miette::{Result, miette}; +use openshell_ocsf::{ + ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, + HttpActivityBuilder, HttpRequest, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, +}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub(crate) enum MiddlewareApplyResult { + Allowed(crate::l7::provider::L7Request), + Denied(String), +} + +/// Smallest body-buffering limit across the entries that actually resolved to a +/// registered binding. Unresolved entries (`is_resolved() == false`) report a +/// zero limit and are excluded here: they are handled by their `on_error` policy +/// in `evaluate_described` without inspecting the body, so letting a zero drag +/// the chain limit to zero would spuriously fail the whole chain over capacity. +/// Returns `None` when no entry resolved, so the caller can skip buffering. +pub(super) fn middleware_chain_body_limit( + chain: &[openshell_supervisor_middleware::DescribedChainEntry], +) -> Option { + chain + .iter() + .filter(|entry| entry.is_resolved()) + .map(openshell_supervisor_middleware::DescribedChainEntry::max_body_bytes) + .min() +} + +pub(crate) async fn apply_middleware_chain( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, +) -> Result { + apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, runner, generation_guard) + .await +} + +pub(crate) async fn apply_middleware_chain_for_scheme( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, +) -> Result { + if chain.is_empty() { + return Ok(MiddlewareApplyResult::Allowed(req)); + } + let chain = runner.describe_chain(&chain).await?; + let Some(max_body_bytes) = middleware_chain_body_limit(&chain) else { + // No entry resolved to a registered binding, so nothing inspects the + // body. Apply each entry's `on_error` policy without buffering (an + // unresolved binding is handled before the body is read) and forward + // the original request unchanged if the chain allows. + let input = middleware_request_input( + scheme, + &req, + ctx, + BTreeMap::new(), + String::new(), + Vec::new(), + ); + let outcome = runner.evaluate_described(&chain, input).await?; + emit_middleware_events(ctx, &req, &outcome); + return Ok(if outcome.allowed { + MiddlewareApplyResult::Allowed(req) + } else { + MiddlewareApplyResult::Denied(outcome.reason) + }); + }; + let buffered = match crate::l7::rest::buffer_request_body_for_middleware( + &req, + client, + Some(generation_guard), + max_body_bytes, + ) + .await? + { + crate::l7::rest::BufferResult::Buffered(buffered) => buffered, + crate::l7::rest::BufferResult::OverCapacity { recoverable } => { + return Ok(resolve_unbuffered_body(ctx, req, &chain, recoverable)); + } + }; + let headers = safe_middleware_headers(&buffered.headers)?; + let query = raw_query_from_request_headers(&buffered.headers)?; + let input = middleware_request_input(scheme, &req, ctx, headers, query, buffered.body); + let outcome = runner.evaluate_described(&chain, input).await?; + emit_middleware_events(ctx, &req, &outcome); + let rebuilt = crate::l7::rest::rebuild_request_with_buffered_body( + &req, + &buffered.headers, + &outcome.body, + &outcome.added_headers, + )?; + if outcome.allowed { + Ok(MiddlewareApplyResult::Allowed(rebuilt)) + } else { + Ok(MiddlewareApplyResult::Denied(outcome.reason)) + } +} + +pub(super) fn middleware_request_input( + scheme: &str, + req: &crate::l7::provider::L7Request, + ctx: &L7EvalContext, + headers: BTreeMap, + query: String, + body: Vec, +) -> openshell_supervisor_middleware::HttpRequestInput { + openshell_supervisor_middleware::HttpRequestInput { + request_id: uuid::Uuid::new_v4().to_string(), + sandbox_id: openshell_ocsf::ctx::ctx().sandbox_id.clone(), + scheme: scheme.into(), + host: ctx.host.clone(), + port: ctx.port, + method: req.action.clone(), + path: req.target.clone(), + query, + headers, + body, + } +} + +pub(super) fn raw_query_from_request_headers(headers: &[u8]) -> Result { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let target = header_str + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .ok_or_else(|| miette!("HTTP request line is missing a target"))?; + Ok(target + .split_once('?') + .map_or_else(String::new, |(_, query)| query.to_string())) +} + +/// Apply the chain's `on_error` policy when the request body cannot be buffered +/// for inspection because it exceeds the size cap. The RFC treats an unbufferable +/// body as an `on_error` event: it is denied unless every attached middleware is +/// `fail_open`, and passing it through is only safe when no bytes were consumed. +pub(super) fn resolve_unbuffered_body( + ctx: &L7EvalContext, + req: crate::l7::provider::L7Request, + chain: &[openshell_supervisor_middleware::DescribedChainEntry], + recoverable: bool, +) -> MiddlewareApplyResult { + let all_fail_open = chain + .iter() + .all(|entry| entry.on_error() == openshell_supervisor_middleware::OnError::FailOpen); + if recoverable && all_fail_open { + emit_middleware_body_unavailable(ctx, false); + return MiddlewareApplyResult::Allowed(req); + } + emit_middleware_body_unavailable(ctx, true); + MiddlewareApplyResult::Denied("middleware_failed: request_body_over_capacity".into()) +} + +fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if denied { + SeverityId::High + } else { + SeverityId::Medium + }) + .finding_info(FindingInfo::new( + "openshell.middleware.body_unavailable", + "Supervisor middleware could not inspect request body", + )) + .evidence_pairs(&[ + ("policy", ctx.policy_name.as_str()), + ("host", ctx.host.as_str()), + ("disposition", if denied { "denied" } else { "fail_open" }), + ]) + .message(if denied { + "Request body exceeded middleware inspection cap; denied" + } else { + "Request body exceeded middleware inspection cap; passed through (fail_open)" + }) + .build(); + ocsf_emit!(event); +} + +fn safe_middleware_headers(headers: &[u8]) -> Result> { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let mut out = BTreeMap::new(); + for line in header_str.lines().skip(1) { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let name = name.trim().to_ascii_lowercase(); + if name.is_empty() + || matches!( + name.as_str(), + "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" + ) + || name.starts_with("x-amz-") + || name.starts_with("x-openshell-credential") + { + continue; + } + out.insert(name, value.trim().to_string()); + } + Ok(out) +} + +pub(super) fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { + crate::opa::NetworkInput { + host: ctx.host.clone(), + port: ctx.port, + binary_path: PathBuf::from(&ctx.binary_path), + binary_sha256: String::new(), + ancestors: ctx.ancestors.iter().map(PathBuf::from).collect(), + cmdline_paths: ctx.cmdline_paths.iter().map(PathBuf::from).collect(), + } +} + +/// Build the OCSF events describing a middleware chain outcome, in emission +/// order. Separated from `emit_middleware_events` so tests can assert on the +/// events deterministically without routing through the global tracing pipeline, +/// whose callsite-interest cache is process-global and races under parallel +/// tests. +pub(super) fn middleware_events( + ctx: &L7EvalContext, + req: &crate::l7::provider::L7Request, + outcome: &openshell_supervisor_middleware::ChainOutcome, +) -> Vec { + let mut events = Vec::new(); + for invocation in &outcome.applied { + let allowed = invocation.decision == openshell_core::proto::Decision::Allow; + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(if allowed { + ActionId::Allowed + } else { + ActionId::Denied + }) + .disposition(if allowed { + DispositionId::Allowed + } else { + DispositionId::Blocked + }) + .severity(if allowed { + SeverityId::Informational + } else { + SeverityId::Medium + }) + .http_request(HttpRequest::new( + &req.action, + OcsfUrl::new("http", &ctx.host, &req.target, ctx.port), + )) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .firewall_rule(&ctx.policy_name, "middleware") + .unmapped("transformed", invocation.transformed) + .unmapped("failed", invocation.failed) + .message(format!( + "MIDDLEWARE {} {} decision={:?}", + invocation.name, invocation.implementation, invocation.decision + )); + if !allowed && !outcome.reason.is_empty() { + event = event + .status(StatusId::Failure) + .status_detail(&outcome.reason); + } + let event = event.build(); + events.push(event); + + // A middleware that failed but was bypassed under `fail_open` is an + // enforcement failure operators must be able to alert on, even though the + // request proceeded. + if invocation.failed && allowed { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Medium) + .finding_info(FindingInfo::new( + "openshell.middleware.failure", + "Supervisor middleware failed open", + )) + .evidence_pairs(&[ + ("middleware", invocation.name.as_str()), + ("implementation", invocation.implementation.as_str()), + ]) + .unmapped("middleware", invocation.name.as_str()) + .unmapped("implementation", invocation.implementation.as_str()) + .message(format!( + "Middleware {} failed and was bypassed (fail_open)", + invocation.name + )) + .build(); + events.push(event); + } + } + if !outcome.allowed && outcome.reason.starts_with("middleware_failed:") { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::High) + .finding_info(FindingInfo::new( + "openshell.middleware.failure", + "Supervisor middleware failure", + )) + .message("Required supervisor middleware failed closed") + .build(); + events.push(event); + } + for finding in &outcome.findings { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(match finding.finding.severity.as_str() { + "high" => SeverityId::High, + "low" => SeverityId::Low, + _ => SeverityId::Medium, + }) + .finding_info(FindingInfo::new( + &finding.finding.r#type, + &finding.finding.label, + )) + .evidence_pairs(&[ + ("middleware", &finding.middleware), + ("count", &finding.finding.count.to_string()), + ]) + .unmapped("middleware", finding.middleware.as_str()) + .unmapped("count", finding.finding.count) + .message(format!( + "Middleware finding {} count={}", + finding.finding.r#type, finding.finding.count + )) + .build(); + events.push(event); + } + events +} + +/// Emit the OCSF events describing a middleware chain outcome through the +/// tracing pipeline. +fn emit_middleware_events( + ctx: &L7EvalContext, + req: &crate::l7::provider::L7Request, + outcome: &openshell_supervisor_middleware::ChainOutcome, +) { + for event in middleware_events(ctx, req, outcome) { + ocsf_emit!(event); + } +} diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index d10c19b8e8..691aeb87d9 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -12,6 +12,7 @@ pub mod graphql; pub(crate) mod http; pub mod inference; pub mod jsonrpc; +pub(crate) mod middleware; pub mod path; pub mod provider; pub mod relay; diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 329d4b42da..87ab2f39f3 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,6 +7,14 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. +use crate::l7::middleware::{ + MiddlewareApplyResult, apply_middleware_chain, middleware_network_input, +}; +#[cfg(test)] +use crate::l7::middleware::{ + middleware_chain_body_limit, middleware_events, middleware_request_input, + raw_query_from_request_headers, resolve_unbuffered_body, +}; use crate::l7::provider::{L7Provider, RelayOutcome}; use crate::l7::rest::WebSocketExtensionMode; use crate::l7::{EnforcementMode, L7EndpointConfig, L7Protocol, L7RequestInfo}; @@ -15,12 +23,11 @@ use miette::{IntoDiagnostic, Result, miette}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::secrets::{self, SecretResolver}; use openshell_ocsf::{ - ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, - HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, - ocsf_emit, + ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, + NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; +#[cfg(test)] use std::collections::BTreeMap; -use std::path::PathBuf; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; @@ -773,344 +780,6 @@ fn jsonrpc_engine_type(protocol: L7Protocol) -> &'static str { } } -pub(crate) enum MiddlewareApplyResult { - Allowed(crate::l7::provider::L7Request), - Denied(String), -} - -/// Smallest body-buffering limit across the entries that actually resolved to a -/// registered binding. Unresolved entries (`is_resolved() == false`) report a -/// zero limit and are excluded here: they are handled by their `on_error` policy -/// in `evaluate_described` without inspecting the body, so letting a zero drag -/// the chain limit to zero would spuriously fail the whole chain over capacity. -/// Returns `None` when no entry resolved, so the caller can skip buffering. -fn middleware_chain_body_limit( - chain: &[openshell_supervisor_middleware::DescribedChainEntry], -) -> Option { - chain - .iter() - .filter(|entry| entry.is_resolved()) - .map(openshell_supervisor_middleware::DescribedChainEntry::max_body_bytes) - .min() -} - -pub(crate) async fn apply_middleware_chain( - req: crate::l7::provider::L7Request, - client: &mut C, - ctx: &L7EvalContext, - chain: Vec, - runner: &openshell_supervisor_middleware::ChainRunner, - generation_guard: &PolicyGenerationGuard, -) -> Result { - apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, runner, generation_guard) - .await -} - -pub(crate) async fn apply_middleware_chain_for_scheme( - req: crate::l7::provider::L7Request, - client: &mut C, - ctx: &L7EvalContext, - scheme: &str, - chain: Vec, - runner: &openshell_supervisor_middleware::ChainRunner, - generation_guard: &PolicyGenerationGuard, -) -> Result { - if chain.is_empty() { - return Ok(MiddlewareApplyResult::Allowed(req)); - } - let chain = runner.describe_chain(&chain).await?; - let Some(max_body_bytes) = middleware_chain_body_limit(&chain) else { - // No entry resolved to a registered binding, so nothing inspects the - // body. Apply each entry's `on_error` policy without buffering (an - // unresolved binding is handled before the body is read) and forward - // the original request unchanged if the chain allows. - let input = middleware_request_input( - scheme, - &req, - ctx, - BTreeMap::new(), - String::new(), - Vec::new(), - ); - let outcome = runner.evaluate_described(&chain, input).await?; - emit_middleware_events(ctx, &req, &outcome); - return Ok(if outcome.allowed { - MiddlewareApplyResult::Allowed(req) - } else { - MiddlewareApplyResult::Denied(outcome.reason) - }); - }; - let buffered = match crate::l7::rest::buffer_request_body_for_middleware( - &req, - client, - Some(generation_guard), - max_body_bytes, - ) - .await? - { - crate::l7::rest::BufferResult::Buffered(buffered) => buffered, - crate::l7::rest::BufferResult::OverCapacity { recoverable } => { - return Ok(resolve_unbuffered_body(ctx, req, &chain, recoverable)); - } - }; - let headers = safe_middleware_headers(&buffered.headers)?; - let query = raw_query_from_request_headers(&buffered.headers)?; - let input = middleware_request_input(scheme, &req, ctx, headers, query, buffered.body); - let outcome = runner.evaluate_described(&chain, input).await?; - emit_middleware_events(ctx, &req, &outcome); - let rebuilt = crate::l7::rest::rebuild_request_with_buffered_body( - &req, - &buffered.headers, - &outcome.body, - &outcome.added_headers, - )?; - if outcome.allowed { - Ok(MiddlewareApplyResult::Allowed(rebuilt)) - } else { - Ok(MiddlewareApplyResult::Denied(outcome.reason)) - } -} - -fn middleware_request_input( - scheme: &str, - req: &crate::l7::provider::L7Request, - ctx: &L7EvalContext, - headers: BTreeMap, - query: String, - body: Vec, -) -> openshell_supervisor_middleware::HttpRequestInput { - openshell_supervisor_middleware::HttpRequestInput { - request_id: uuid::Uuid::new_v4().to_string(), - sandbox_id: openshell_ocsf::ctx::ctx().sandbox_id.clone(), - scheme: scheme.into(), - host: ctx.host.clone(), - port: ctx.port, - method: req.action.clone(), - path: req.target.clone(), - query, - headers, - body, - } -} - -fn raw_query_from_request_headers(headers: &[u8]) -> Result { - let header_str = - std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - let target = header_str - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .ok_or_else(|| miette!("HTTP request line is missing a target"))?; - Ok(target - .split_once('?') - .map_or_else(String::new, |(_, query)| query.to_string())) -} - -/// Apply the chain's `on_error` policy when the request body cannot be buffered -/// for inspection because it exceeds the size cap. The RFC treats an unbufferable -/// body as an `on_error` event: it is denied unless every attached middleware is -/// `fail_open`, and passing it through is only safe when no bytes were consumed. -fn resolve_unbuffered_body( - ctx: &L7EvalContext, - req: crate::l7::provider::L7Request, - chain: &[openshell_supervisor_middleware::DescribedChainEntry], - recoverable: bool, -) -> MiddlewareApplyResult { - let all_fail_open = chain - .iter() - .all(|entry| entry.on_error() == openshell_supervisor_middleware::OnError::FailOpen); - if recoverable && all_fail_open { - emit_middleware_body_unavailable(ctx, false); - return MiddlewareApplyResult::Allowed(req); - } - emit_middleware_body_unavailable(ctx, true); - MiddlewareApplyResult::Denied("middleware_failed: request_body_over_capacity".into()) -} - -fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { - let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(if denied { - SeverityId::High - } else { - SeverityId::Medium - }) - .finding_info(FindingInfo::new( - "openshell.middleware.body_unavailable", - "Supervisor middleware could not inspect request body", - )) - .evidence_pairs(&[ - ("policy", ctx.policy_name.as_str()), - ("host", ctx.host.as_str()), - ("disposition", if denied { "denied" } else { "fail_open" }), - ]) - .message(if denied { - "Request body exceeded middleware inspection cap; denied" - } else { - "Request body exceeded middleware inspection cap; passed through (fail_open)" - }) - .build(); - ocsf_emit!(event); -} - -fn safe_middleware_headers(headers: &[u8]) -> Result> { - let header_str = - std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - let mut out = BTreeMap::new(); - for line in header_str.lines().skip(1) { - let Some((name, value)) = line.split_once(':') else { - continue; - }; - let name = name.trim().to_ascii_lowercase(); - if name.is_empty() - || matches!( - name.as_str(), - "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" - ) - || name.starts_with("x-amz-") - || name.starts_with("x-openshell-credential") - { - continue; - } - out.insert(name, value.trim().to_string()); - } - Ok(out) -} - -fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { - crate::opa::NetworkInput { - host: ctx.host.clone(), - port: ctx.port, - binary_path: PathBuf::from(&ctx.binary_path), - binary_sha256: String::new(), - ancestors: ctx.ancestors.iter().map(PathBuf::from).collect(), - cmdline_paths: ctx.cmdline_paths.iter().map(PathBuf::from).collect(), - } -} - -/// Build the OCSF events describing a middleware chain outcome, in emission -/// order. Separated from `emit_middleware_events` so tests can assert on the -/// events deterministically without routing through the global tracing pipeline, -/// whose callsite-interest cache is process-global and races under parallel -/// tests. -fn middleware_events( - ctx: &L7EvalContext, - req: &crate::l7::provider::L7Request, - outcome: &openshell_supervisor_middleware::ChainOutcome, -) -> Vec { - let mut events = Vec::new(); - for invocation in &outcome.applied { - let allowed = invocation.decision == openshell_core::proto::Decision::Allow; - let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(if allowed { - ActionId::Allowed - } else { - ActionId::Denied - }) - .disposition(if allowed { - DispositionId::Allowed - } else { - DispositionId::Blocked - }) - .severity(if allowed { - SeverityId::Informational - } else { - SeverityId::Medium - }) - .http_request(HttpRequest::new( - &req.action, - OcsfUrl::new("http", &ctx.host, &req.target, ctx.port), - )) - .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) - .firewall_rule(&ctx.policy_name, "middleware") - .unmapped("transformed", invocation.transformed) - .unmapped("failed", invocation.failed) - .message(format!( - "MIDDLEWARE {} {} decision={:?}", - invocation.name, invocation.implementation, invocation.decision - )); - if !allowed && !outcome.reason.is_empty() { - event = event - .status(StatusId::Failure) - .status_detail(&outcome.reason); - } - let event = event.build(); - events.push(event); - - // A middleware that failed but was bypassed under `fail_open` is an - // enforcement failure operators must be able to alert on, even though the - // request proceeded. - if invocation.failed && allowed { - let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(SeverityId::Medium) - .finding_info(FindingInfo::new( - "openshell.middleware.failure", - "Supervisor middleware failed open", - )) - .evidence_pairs(&[ - ("middleware", invocation.name.as_str()), - ("implementation", invocation.implementation.as_str()), - ]) - .unmapped("middleware", invocation.name.as_str()) - .unmapped("implementation", invocation.implementation.as_str()) - .message(format!( - "Middleware {} failed and was bypassed (fail_open)", - invocation.name - )) - .build(); - events.push(event); - } - } - if !outcome.allowed && outcome.reason.starts_with("middleware_failed:") { - let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(SeverityId::High) - .finding_info(FindingInfo::new( - "openshell.middleware.failure", - "Supervisor middleware failure", - )) - .message("Required supervisor middleware failed closed") - .build(); - events.push(event); - } - for finding in &outcome.findings { - let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(match finding.finding.severity.as_str() { - "high" => SeverityId::High, - "low" => SeverityId::Low, - _ => SeverityId::Medium, - }) - .finding_info(FindingInfo::new( - &finding.finding.r#type, - &finding.finding.label, - )) - .evidence_pairs(&[ - ("middleware", &finding.middleware), - ("count", &finding.finding.count.to_string()), - ]) - .unmapped("middleware", finding.middleware.as_str()) - .unmapped("count", finding.finding.count) - .message(format!( - "Middleware finding {} count={}", - finding.finding.r#type, finding.finding.count - )) - .build(); - events.push(event); - } - events -} - -/// Emit the OCSF events describing a middleware chain outcome through the -/// tracing pipeline. -fn emit_middleware_events( - ctx: &L7EvalContext, - req: &crate::l7::provider::L7Request, - outcome: &openshell_supervisor_middleware::ChainOutcome, -) { - for event in middleware_events(ctx, req, outcome) { - ocsf_emit!(event); - } -} - /// REST relay loop: parse request -> evaluate -> allow/deny -> relay response -> repeat. async fn relay_rest( config: &L7EndpointConfig, diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index f0aea9c389..8702c6e6e2 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -227,6 +227,7 @@ impl OpaEngine { let mut data: serde_json::Value = serde_json::from_str(&data_json_str) .map_err(|e| miette::miette!("internal: failed to parse proto JSON: {e}"))?; + // Validate BEFORE expanding presets let (errors, warnings) = crate::l7::validate_l7_policies(&data); for w in &warnings { openshell_ocsf::ocsf_emit!( @@ -272,27 +273,7 @@ impl OpaEngine { /// `allow_network` rule, and returns a `PolicyDecision` with the result, /// deny reason, and matched policy name. pub fn evaluate_network(&self, input: &NetworkInput) -> Result { - let ancestor_strs: Vec = input - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let cmdline_strs: Vec = input - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let input_json = serde_json::json!({ - "exec": { - "path": input.binary_path.to_string_lossy(), - "ancestors": ancestor_strs, - "cmdline_paths": cmdline_strs, - }, - "network": { - "host": input.host, - "port": input.port, - } - }); + let input_json = network_input_json(input); let mut engine = self .engine @@ -342,27 +323,7 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(NetworkAction, u64)> { - let ancestor_strs: Vec = input - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let cmdline_strs: Vec = input - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let input_json = serde_json::json!({ - "exec": { - "path": input.binary_path.to_string_lossy(), - "ancestors": ancestor_strs, - "cmdline_paths": cmdline_strs, - }, - "network": { - "host": input.host, - "port": input.port, - } - }); + let input_json = network_input_json(input); let mut engine = self .engine @@ -552,27 +513,7 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(Vec, u64)> { - let ancestor_strs: Vec = input - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let cmdline_strs: Vec = input - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let input_json = serde_json::json!({ - "exec": { - "path": input.binary_path.to_string_lossy(), - "ancestors": ancestor_strs, - "cmdline_paths": cmdline_strs, - }, - "network": { - "host": input.host, - "port": input.port, - } - }); + let input_json = network_input_json(input); let mut engine = self .engine @@ -630,27 +571,7 @@ impl OpaEngine { /// denial while preserving separate handling for `allowed_ips` and advisor /// proposals. pub fn query_exact_declared_endpoint_host(&self, input: &NetworkInput) -> Result { - let ancestor_strs: Vec = input - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let cmdline_strs: Vec = input - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(); - let input_json = serde_json::json!({ - "exec": { - "path": input.binary_path.to_string_lossy(), - "ancestors": ancestor_strs, - "cmdline_paths": cmdline_strs, - }, - "network": { - "host": input.host, - "port": input.port, - } - }); + let input_json = network_input_json(input); let mut engine = self .engine @@ -1711,7 +1632,7 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St "middleware": mw.middleware, }); if let Some(config) = &mw.config { - value["config"] = prost_struct_to_json(config); + value["config"] = openshell_core::proto_struct::struct_to_json_value(config); } if !mw.on_error.is_empty() { value["on_error"] = mw.on_error.clone().into(); @@ -1740,32 +1661,6 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St .to_string() } -fn prost_struct_to_json(config: &prost_types::Struct) -> serde_json::Value { - serde_json::Value::Object( - config - .fields - .iter() - .map(|(key, value)| (key.clone(), prost_value_to_json(value))) - .collect(), - ) -} - -fn prost_value_to_json(value: &prost_types::Value) -> serde_json::Value { - match value.kind.as_ref() { - Some(prost_types::value::Kind::NullValue(_)) | None => serde_json::Value::Null, - Some(prost_types::value::Kind::BoolValue(value)) => serde_json::Value::Bool(*value), - Some(prost_types::value::Kind::NumberValue(value)) => serde_json::Number::from_f64(*value) - .map_or(serde_json::Value::Null, serde_json::Value::Number), - Some(prost_types::value::Kind::StringValue(value)) => { - serde_json::Value::String(value.clone()) - } - Some(prost_types::value::Kind::ListValue(value)) => { - serde_json::Value::Array(value.values.iter().map(prost_value_to_json).collect()) - } - Some(prost_types::value::Kind::StructValue(value)) => prost_struct_to_json(value), - } -} - #[cfg(test)] #[allow( clippy::needless_raw_string_hashes, diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 8616c3b2c6..8a10ea3fdb 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4225,7 +4225,7 @@ async fn handle_forward_proxy( &upstream_target, forward_request_bytes, )?; - forward_request_bytes = match crate::l7::relay::apply_middleware_chain_for_scheme( + forward_request_bytes = match crate::l7::middleware::apply_middleware_chain_for_scheme( request, client, &l7_ctx, @@ -4236,8 +4236,8 @@ async fn handle_forward_proxy( ) .await? { - crate::l7::relay::MiddlewareApplyResult::Allowed(request) => request.raw_header, - crate::l7::relay::MiddlewareApplyResult::Denied(reason) => { + crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header, + crate::l7::middleware::MiddlewareApplyResult::Denied(reason) => { emit_activity_simple(activity_tx, true, "middleware"); respond( client, From d2a9791308f969117363aad5654953dfb7f668e6 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 12:23:27 -0700 Subject: [PATCH 24/59] refactor(middleware): move shared contracts to core Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 - architecture/sandbox.md | 4 +- crates/openshell-core/README.md | 7 ++ crates/openshell-core/src/lib.rs | 1 + crates/openshell-core/src/middleware.rs | 77 +++++++++++++++++++ crates/openshell-policy/Cargo.toml | 1 - crates/openshell-policy/src/lib.rs | 11 ++- .../src/builtins/mod.rs | 9 --- .../src/builtins/secrets.rs | 18 +---- .../src/lib.rs | 11 +-- .../src/l7/middleware.rs | 6 +- 11 files changed, 99 insertions(+), 47 deletions(-) create mode 100644 crates/openshell-core/src/middleware.rs diff --git a/Cargo.lock b/Cargo.lock index 6819aab32d..3552ea04ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3772,7 +3772,6 @@ dependencies = [ "glob", "miette", "openshell-core", - "openshell-supervisor-middleware", "prost-types", "serde", "serde_json", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index d63c4fbaab..bf09b3116c 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -72,7 +72,9 @@ of the network rule that admitted the request. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep -the last-known-good service registry when a live config reload fails. +the last-known-good service registry when a live config reload fails. Built-in +middleware identifiers and pure config validation live in `openshell-core` so +policy admission does not depend on the supervisor runtime implementation. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-core/README.md b/crates/openshell-core/README.md index 51da847b82..b40ae121ca 100644 --- a/crates/openshell-core/README.md +++ b/crates/openshell-core/README.md @@ -50,3 +50,10 @@ router agree on provider defaults. Profiles define: Do not duplicate provider-specific inference behavior in callers. Add shared behavior here, then consume it from the gateway, sandbox, and router. + +## Middleware Contracts + +Built-in supervisor middleware identifiers and pure configuration validation +live in `openshell_core::middleware`. Policy admission and the supervisor +runtime consume the same contract without introducing a dependency from the +policy crate to the supervisor implementation. diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 3212963691..9ba041dd85 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -23,6 +23,7 @@ pub mod grpc_client; pub mod image; pub mod inference; pub mod metadata; +pub mod middleware; pub mod net; pub mod paths; pub mod policy; diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs new file mode 100644 index 0000000000..6fb15b1c15 --- /dev/null +++ b/crates/openshell-core/src/middleware.rs @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared supervisor middleware identifiers and policy validation contracts. + +use miette::{Result, miette}; + +/// Binding identifier for the built-in secret redaction middleware. +pub const BUILTIN_SECRETS: &str = "openshell/secrets"; + +/// Validate policy-owned configuration for a built-in middleware. +pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { + match implementation { + BUILTIN_SECRETS => validate_secrets_config(config), + other => Err(miette!( + "middleware implementation '{other}' is not available in phase 1" + )), + } +} + +fn validate_secrets_config(config: &prost_types::Struct) -> Result<()> { + let mode = config + .fields + .get("secrets") + .and_then(|value| match value.kind.as_ref() { + Some(prost_types::value::Kind::StringValue(value)) => Some(value.as_str()), + _ => None, + }) + .unwrap_or("redact"); + if mode != "redact" { + return Err(miette!( + "{BUILTIN_SECRETS} only supports config.secrets: redact in phase 1" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn secrets_config_defaults_to_redact() { + validate_builtin_config(BUILTIN_SECRETS, &prost_types::Struct::default()).unwrap(); + } + + #[test] + fn secrets_config_rejects_unsupported_mode() { + let config = prost_types::Struct { + fields: std::iter::once(( + "secrets".to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("allow".into())), + }, + )) + .collect(), + }; + + let error = validate_builtin_config(BUILTIN_SECRETS, &config).unwrap_err(); + assert!( + error + .to_string() + .contains("only supports config.secrets: redact") + ); + } + + #[test] + fn rejects_unknown_builtin() { + let error = validate_builtin_config("openshell/unknown", &prost_types::Struct::default()) + .unwrap_err(); + assert!( + error + .to_string() + .contains("implementation 'openshell/unknown' is not available") + ); + } +} diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 073728db16..036964b722 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -13,7 +13,6 @@ repository.workspace = true [dependencies] glob = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } -openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } prost-types = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index efcca2fcaa..f09006b2e7 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1532,7 +1532,7 @@ pub fn validate_sandbox_policy( reason: "implementation must not be empty".to_string(), }); } else if middleware.middleware.starts_with("openshell/") - && middleware.middleware != openshell_supervisor_middleware::BUILTIN_SECRETS + && middleware.middleware != openshell_core::middleware::BUILTIN_SECRETS { violations.push(PolicyViolation::InvalidMiddlewareConfig { name: middleware.name.clone(), @@ -1572,12 +1572,11 @@ pub fn validate_sandbox_policy( } } - if middleware.middleware == openshell_supervisor_middleware::BUILTIN_SECRETS { + if middleware.middleware == openshell_core::middleware::BUILTIN_SECRETS { let config = middleware.config.clone().unwrap_or_default(); - if let Err(error) = openshell_supervisor_middleware::validate_builtin_config( - &middleware.middleware, - &config, - ) { + if let Err(error) = + openshell_core::middleware::validate_builtin_config(&middleware.middleware, &config) + { violations.push(PolicyViolation::InvalidBuiltinMiddlewareConfig { name: middleware.name.clone(), reason: error.to_string(), diff --git a/crates/openshell-supervisor-middleware/src/builtins/mod.rs b/crates/openshell-supervisor-middleware/src/builtins/mod.rs index 1db620220a..176f4786cf 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/mod.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/mod.rs @@ -10,15 +10,6 @@ pub fn describe() -> Vec { vec![secrets::describe()] } -pub fn validate_config(binding_id: &str, config: &prost_types::Struct) -> Result<()> { - match binding_id { - secrets::BINDING_ID => secrets::validate_config(config), - other => Err(miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - } -} - pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { match evaluation.binding_id.as_str() { secrets::BINDING_ID => secrets::evaluate_http_request(evaluation), diff --git a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs index d88ac080d8..716301c447 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs +++ b/crates/openshell-supervisor-middleware/src/builtins/secrets.rs @@ -10,7 +10,7 @@ use openshell_core::proto::{ }; use regex::Regex; -pub const BINDING_ID: &str = "openshell/secrets"; +pub const BINDING_ID: &str = openshell_core::middleware::BUILTIN_SECRETS; const OPERATION: &str = "HttpRequest"; const PHASE: &str = "pre_credentials"; const MAX_BODY_BYTES: u64 = 256 * 1024; @@ -54,21 +54,7 @@ static SECRET_PATTERNS: LazyLock<[SecretPattern; 2]> = LazyLock::new(|| { }); pub fn validate_config(config: &prost_types::Struct) -> Result<()> { - let mode = config - .fields - .get("secrets") - .and_then(|value| match value.kind.as_ref() { - Some(prost_types::value::Kind::StringValue(value)) => Some(value.as_str()), - _ => None, - }) - .unwrap_or("redact"); - if mode != "redact" { - return Err(miette!( - "{} only supports config.secrets: redact in phase 1", - BINDING_ID - )); - } - Ok(()) + openshell_core::middleware::validate_builtin_config(BINDING_ID, config) } pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index d2f8642b18..9b067b31e1 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -11,6 +11,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, LazyLock}; use miette::{Result, miette}; +pub use openshell_core::middleware::{BUILTIN_SECRETS, validate_builtin_config}; pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; @@ -23,18 +24,8 @@ use tokio::sync::OnceCell; use tonic::Request; pub const API_VERSION: &str = "openshell.middleware.v1"; -pub const BUILTIN_SECRETS: &str = builtins::secrets::BINDING_ID; const HTTP_REQUEST_OPERATION: &str = "HttpRequest"; const PRE_CREDENTIALS_PHASE: &str = "pre_credentials"; - -/// Validate the configuration for an in-process middleware implementation. -/// -/// Policy admission uses this same implementation-specific validation before a -/// configuration can reach the request path. -pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { - builtins::validate_config(implementation, config) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OnError { FailClosed, diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index d8a8b6c300..a33d06e377 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncWrite}; -pub(crate) enum MiddlewareApplyResult { +pub enum MiddlewareApplyResult { Allowed(crate::l7::provider::L7Request), Denied(String), } @@ -35,7 +35,7 @@ pub(super) fn middleware_chain_body_limit( .min() } -pub(crate) async fn apply_middleware_chain( +pub async fn apply_middleware_chain( req: crate::l7::provider::L7Request, client: &mut C, ctx: &L7EvalContext, @@ -47,7 +47,7 @@ pub(crate) async fn apply_middleware_chain( +pub async fn apply_middleware_chain_for_scheme( req: crate::l7::provider::L7Request, client: &mut C, ctx: &L7EvalContext, From 64439ea327b80bbc12bae145c4a8653450cfc5ec Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 13:05:06 -0700 Subject: [PATCH 25/59] refactor(policy): extract middleware serialization Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 4 +- crates/openshell-cli/src/run.rs | 36 +-- crates/openshell-core/README.md | 7 + crates/openshell-core/src/proto_struct.rs | 67 ++++- crates/openshell-driver-docker/src/tests.rs | 39 +-- .../openshell-driver-kubernetes/src/driver.rs | 37 +-- .../openshell-driver-podman/src/container.rs | 39 +-- crates/openshell-driver-podman/src/driver.rs | 39 +-- crates/openshell-policy/src/lib.rs | 280 ++---------------- crates/openshell-policy/src/middleware.rs | 217 ++++++++++++++ 10 files changed, 337 insertions(+), 428 deletions(-) create mode 100644 crates/openshell-policy/src/middleware.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index bf09b3116c..a9f1208e06 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,7 +74,9 @@ over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep the last-known-good service registry when a live config reload fails. Built-in middleware identifiers and pure config validation live in `openshell-core` so -policy admission does not depend on the supervisor runtime implementation. +policy admission does not depend on the supervisor runtime implementation. The +policy and runtime also share the core JSON/protobuf adapter for middleware +configuration, keeping serialization consistent across that boundary. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index f56bb71512..4eaf429c44 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1648,39 +1648,9 @@ fn parse_driver_config_json(value: &str) -> Result { )); }; - Ok(prost_types::Struct { - fields: fields - .into_iter() - .map(|(key, value)| json_to_protobuf_value(value).map(|value| (key, value))) - .collect::>()?, - }) -} - -fn json_to_protobuf_value(value: serde_json::Value) -> Result { - use prost_types::{ListValue, Struct, Value, value::Kind}; - - let kind = match value { - serde_json::Value::Null => Kind::NullValue(0), - serde_json::Value::Bool(value) => Kind::BoolValue(value), - serde_json::Value::Number(value) => Kind::NumberValue(value.as_f64().ok_or_else(|| { - miette!("--driver-config-json contains a number that cannot be represented") - })?), - serde_json::Value::String(value) => Kind::StringValue(value), - serde_json::Value::Array(values) => Kind::ListValue(ListValue { - values: values - .into_iter() - .map(json_to_protobuf_value) - .collect::>()?, - }), - serde_json::Value::Object(fields) => Kind::StructValue(Struct { - fields: fields - .into_iter() - .map(|(key, value)| json_to_protobuf_value(value).map(|value| (key, value))) - .collect::>()?, - }), - }; - - Ok(Value { kind: Some(kind) }) + openshell_core::proto_struct::json_object_to_struct(fields) + .into_diagnostic() + .wrap_err("--driver-config-json contains a value that cannot be represented") } fn validate_cpu_quantity(value: &str) -> Result { diff --git a/crates/openshell-core/README.md b/crates/openshell-core/README.md index b40ae121ca..e27ab167f5 100644 --- a/crates/openshell-core/README.md +++ b/crates/openshell-core/README.md @@ -57,3 +57,10 @@ Built-in supervisor middleware identifiers and pure configuration validation live in `openshell_core::middleware`. Policy admission and the supervisor runtime consume the same contract without introducing a dependency from the policy crate to the supervisor implementation. + +## Protobuf Struct Conversion + +Use `openshell_core::proto_struct` when crossing between `serde_json` values and +`prost_types::{Struct, Value}`. Both conversion directions live in this module; +JSON-to-protobuf conversion is fallible so callers cannot silently replace an +unrepresentable number. diff --git a/crates/openshell-core/src/proto_struct.rs b/crates/openshell-core/src/proto_struct.rs index 8871d6f6a7..b90c52ca73 100644 --- a/crates/openshell-core/src/proto_struct.rs +++ b/crates/openshell-core/src/proto_struct.rs @@ -1,10 +1,57 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Helpers for decoding `google.protobuf.Struct` values. +//! Helpers for converting `google.protobuf.Struct` values to and from JSON. use serde::{Deserialize, Deserializer, de::Error as _}; +/// Errors converting JSON values into protobuf well-known types. +#[derive(Debug, thiserror::Error)] +pub enum ProtoStructError { + /// A JSON number cannot be represented by protobuf's double value. + #[error("JSON number {0} cannot be represented as a protobuf double")] + UnrepresentableNumber(serde_json::Number), +} + +/// Convert a JSON object into a protobuf Struct. +pub fn json_object_to_struct( + object: serde_json::Map, +) -> Result { + Ok(prost_types::Struct { + fields: object + .into_iter() + .map(|(key, value)| json_value_to_proto(value).map(|value| (key, value))) + .collect::>()?, + }) +} + +/// Convert a JSON value into a protobuf Value. +pub fn json_value_to_proto( + value: serde_json::Value, +) -> Result { + use prost_types::{ListValue, Value, value::Kind}; + + let kind = match value { + serde_json::Value::Null => Kind::NullValue(0), + serde_json::Value::Bool(value) => Kind::BoolValue(value), + serde_json::Value::Number(value) => Kind::NumberValue( + value + .as_f64() + .ok_or_else(|| ProtoStructError::UnrepresentableNumber(value.clone()))?, + ), + serde_json::Value::String(value) => Kind::StringValue(value), + serde_json::Value::Array(values) => Kind::ListValue(ListValue { + values: values + .into_iter() + .map(json_value_to_proto) + .collect::>()?, + }), + serde_json::Value::Object(object) => Kind::StructValue(json_object_to_struct(object)?), + }; + + Ok(Value { kind: Some(kind) }) +} + /// Convert a protobuf Struct into a JSON object for typed serde decoding. #[must_use] pub fn struct_to_json_object( @@ -72,6 +119,24 @@ where mod tests { use super::*; + #[test] + fn json_and_proto_values_round_trip() { + let json = serde_json::json!({ + "null": null, + "bool": true, + "number": 42.5, + "string": "value", + "list": [1.0, {"nested": "value"}], + }); + let serde_json::Value::Object(object) = json.clone() else { + unreachable!(); + }; + + let proto = json_object_to_struct(object).unwrap(); + + assert_eq!(struct_to_json_value(&proto), json); + } + #[derive(Debug, Default, Deserialize)] #[serde(default)] struct TestConfig { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index d42d41099d..bc37d87be1 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -118,40 +118,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { } fn json_struct(value: serde_json::Value) -> prost_types::Struct { - match json_value(value).kind { - Some(prost_types::value::Kind::StructValue(value)) => value, - _ => panic!("expected JSON object"), - } -} - -fn json_value(value: serde_json::Value) -> prost_types::Value { - match value { - serde_json::Value::Null => prost_types::Value { kind: None }, - serde_json::Value::Bool(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::BoolValue(value)), - }, - serde_json::Value::Number(value) => prost_types::Value { - kind: value.as_f64().map(prost_types::value::Kind::NumberValue), - }, - serde_json::Value::String(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue(value)), - }, - serde_json::Value::Array(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::ListValue( - prost_types::ListValue { - values: values.into_iter().map(json_value).collect(), - }, - )), - }, - serde_json::Value::Object(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::StructValue(prost_types::Struct { - fields: values - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - })), - }, - } + let serde_json::Value::Object(object) = value else { + panic!("expected JSON object"); + }; + openshell_core::proto_struct::json_object_to_struct(object) + .expect("test JSON must convert to a protobuf Struct") } fn inspected_volume(driver: &str, options: HashMap) -> bollard::models::Volume { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 166f18b1cc..eb34905c33 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -2200,38 +2200,11 @@ mod tests { std::sync::LazyLock::new(|| std::sync::Mutex::new(())); fn json_struct(value: serde_json::Value) -> Struct { - match json_value(value).kind { - Some(Kind::StructValue(value)) => value, - _ => panic!("expected JSON object"), - } - } - - fn json_value(value: serde_json::Value) -> Value { - match value { - serde_json::Value::Null => Value { kind: None }, - serde_json::Value::Bool(value) => Value { - kind: Some(Kind::BoolValue(value)), - }, - serde_json::Value::Number(value) => Value { - kind: value.as_f64().map(Kind::NumberValue), - }, - serde_json::Value::String(value) => Value { - kind: Some(Kind::StringValue(value)), - }, - serde_json::Value::Array(values) => Value { - kind: Some(Kind::ListValue(prost_types::ListValue { - values: values.into_iter().map(json_value).collect(), - })), - }, - serde_json::Value::Object(values) => Value { - kind: Some(Kind::StructValue(Struct { - fields: values - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - })), - }, - } + let serde_json::Value::Object(object) = value else { + panic!("expected JSON object"); + }; + openshell_core::proto_struct::json_object_to_struct(object) + .expect("test JSON must convert to a protobuf Struct") } fn kube_api_error(code: u16, message: &str) -> KubeError { diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7c9ab269f1..73bbb59755 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1128,40 +1128,11 @@ mod tests { std::sync::LazyLock::new(|| std::sync::Mutex::new(())); fn json_struct(value: Value) -> prost_types::Struct { - match json_value(value).kind { - Some(prost_types::value::Kind::StructValue(value)) => value, - _ => panic!("expected JSON object"), - } - } - - fn json_value(value: Value) -> prost_types::Value { - match value { - Value::Null => prost_types::Value { kind: None }, - Value::Bool(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::BoolValue(value)), - }, - Value::Number(value) => prost_types::Value { - kind: value.as_f64().map(prost_types::value::Kind::NumberValue), - }, - Value::String(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue(value)), - }, - Value::Array(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::ListValue( - prost_types::ListValue { - values: values.into_iter().map(json_value).collect(), - }, - )), - }, - Value::Object(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::StructValue(prost_types::Struct { - fields: values - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - })), - }, - } + let Value::Object(object) = value else { + panic!("expected JSON object"); + }; + proto_struct::json_object_to_struct(object) + .expect("test JSON must convert to a protobuf Struct") } fn gpu_resources(count: Option) -> ResourceRequirements { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 27d5102811..a76f5485a2 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1274,41 +1274,12 @@ mod tests { PodmanComputeDriver::for_tests(config) } - fn json_value(value: serde_json::Value) -> prost_types::Value { - match value { - serde_json::Value::Null => prost_types::Value { kind: None }, - serde_json::Value::Bool(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::BoolValue(value)), - }, - serde_json::Value::Number(value) => prost_types::Value { - kind: value.as_f64().map(prost_types::value::Kind::NumberValue), - }, - serde_json::Value::String(value) => prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue(value)), - }, - serde_json::Value::Array(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::ListValue( - prost_types::ListValue { - values: values.into_iter().map(json_value).collect(), - }, - )), - }, - serde_json::Value::Object(values) => prost_types::Value { - kind: Some(prost_types::value::Kind::StructValue(prost_types::Struct { - fields: values - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - })), - }, - } - } - fn json_struct(value: serde_json::Value) -> prost_types::Struct { - match json_value(value).kind { - Some(prost_types::value::Kind::StructValue(value)) => value, - _ => panic!("expected JSON object"), - } + let serde_json::Value::Object(object) = value else { + panic!("expected JSON object"); + }; + openshell_core::proto_struct::json_object_to_struct(object) + .expect("test JSON must convert to a protobuf Struct") } fn sandbox_with_volume_mount(volume: &str) -> DriverSandbox { diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index f09006b2e7..d388e28a2f 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -11,16 +11,17 @@ mod compose; mod merge; +mod middleware; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::Path; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ FilesystemPolicy, GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, - LandlockPolicy, McpOptions, MiddlewareEndpointSelector, NetworkBinary, NetworkEndpoint, - NetworkMiddlewareConfig, NetworkPolicyRule, ProcessPolicy, SandboxPolicy, + LandlockPolicy, McpOptions, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, ProcessPolicy, + SandboxPolicy, }; use serde::{Deserialize, Serialize}; @@ -32,6 +33,7 @@ pub use merge::{ PolicyMergeError, PolicyMergeOp, PolicyMergeResult, PolicyMergeWarning, generated_rule_name, merge_policy, policy_covers_rule, }; +pub use middleware::middleware_host_matches; // --------------------------------------------------------------------------- // YAML serde types (canonical — used for both parsing and serialization) @@ -50,7 +52,7 @@ struct PolicyFile { #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] network_policies: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] - network_middlewares: Vec, + network_middlewares: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -91,28 +93,6 @@ struct NetworkPolicyRuleDef { binaries: Vec, } -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct NetworkMiddlewareConfigDef { - name: String, - middleware: String, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - config: BTreeMap, - #[serde(default, skip_serializing_if = "String::is_empty")] - on_error: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - endpoints: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct MiddlewareEndpointSelectorDef { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - include: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - exclude: Vec, -} - #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct NetworkEndpointDef { @@ -695,21 +675,10 @@ fn yaml_mcp_method( method.to_string() } -fn to_proto(raw: PolicyFile) -> SandboxPolicy { - let network_middlewares = raw - .network_middlewares - .into_iter() - .map(|mw| NetworkMiddlewareConfig { - name: mw.name, - middleware: mw.middleware, - config: Some(json_map_to_struct(mw.config)), - on_error: mw.on_error, - endpoints: mw.endpoints.map(|selector| MiddlewareEndpointSelector { - include: selector.include, - exclude: selector.exclude, - }), - }) - .collect(); +fn to_proto(raw: PolicyFile) -> Result { + let network_middlewares = middleware::into_proto(raw.network_middlewares) + .into_diagnostic() + .wrap_err("failed to convert network middleware config")?; let network_policies = raw .network_policies @@ -800,7 +769,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { }) .collect(); - SandboxPolicy { + Ok(SandboxPolicy { version: raw.version, filesystem: raw.filesystem_policy.map(|fs| FilesystemPolicy { include_workdir: fs.include_workdir, @@ -816,7 +785,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { }), network_policies, network_middlewares, - } + }) } // --------------------------------------------------------------------------- @@ -948,27 +917,7 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { }) .collect(); - let network_middlewares = policy - .network_middlewares - .iter() - .map(|mw| NetworkMiddlewareConfigDef { - name: mw.name.clone(), - middleware: mw.middleware.clone(), - config: mw - .config - .as_ref() - .map(struct_to_json_map) - .unwrap_or_default(), - on_error: mw.on_error.clone(), - endpoints: mw - .endpoints - .as_ref() - .map(|selector| MiddlewareEndpointSelectorDef { - include: selector.include.clone(), - exclude: selector.exclude.clone(), - }), - }) - .collect(); + let network_middlewares = middleware::from_proto(&policy.network_middlewares); PolicyFile { version: policy.version, @@ -980,68 +929,6 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { } } -fn json_map_to_struct(map: BTreeMap) -> prost_types::Struct { - prost_types::Struct { - fields: map - .into_iter() - .map(|(key, value)| (key, json_to_protobuf_value(value))) - .collect(), - } -} - -fn json_to_protobuf_value(value: serde_json::Value) -> prost_types::Value { - use prost_types::{ListValue, Struct, Value, value::Kind}; - Value { - kind: Some(match value { - serde_json::Value::Null => Kind::NullValue(0), - serde_json::Value::Bool(value) => Kind::BoolValue(value), - serde_json::Value::Number(value) => { - Kind::NumberValue(value.as_f64().unwrap_or_default()) - } - serde_json::Value::String(value) => Kind::StringValue(value), - serde_json::Value::Array(values) => Kind::ListValue(ListValue { - values: values.into_iter().map(json_to_protobuf_value).collect(), - }), - serde_json::Value::Object(values) => Kind::StructValue(Struct { - fields: values - .into_iter() - .map(|(key, value)| (key, json_to_protobuf_value(value))) - .collect(), - }), - }), - } -} - -fn struct_to_json_map(config: &prost_types::Struct) -> BTreeMap { - config - .fields - .iter() - .map(|(key, value)| (key.clone(), protobuf_value_to_json(value))) - .collect() -} - -fn protobuf_value_to_json(value: &prost_types::Value) -> serde_json::Value { - match value.kind.as_ref() { - Some(prost_types::value::Kind::NullValue(_)) | None => serde_json::Value::Null, - Some(prost_types::value::Kind::BoolValue(value)) => serde_json::Value::Bool(*value), - Some(prost_types::value::Kind::NumberValue(value)) => serde_json::Number::from_f64(*value) - .map_or(serde_json::Value::Null, serde_json::Value::Number), - Some(prost_types::value::Kind::StringValue(value)) => { - serde_json::Value::String(value.clone()) - } - Some(prost_types::value::Kind::ListValue(value)) => { - serde_json::Value::Array(value.values.iter().map(protobuf_value_to_json).collect()) - } - Some(prost_types::value::Kind::StructValue(value)) => serde_json::Value::Object( - value - .fields - .iter() - .map(|(key, value)| (key.clone(), protobuf_value_to_json(value))) - .collect(), - ), - } -} - // --------------------------------------------------------------------------- // Sandbox UID/GID constants // --------------------------------------------------------------------------- @@ -1086,7 +973,7 @@ pub fn parse_sandbox_policy(yaml: &str) -> Result { let raw: PolicyFile = serde_yml::from_str(yaml) .into_diagnostic() .wrap_err("failed to parse sandbox policy YAML")?; - Ok(to_proto(raw)) + to_proto(raw) } /// Serialize a proto sandbox policy to a YAML string. @@ -1343,45 +1230,6 @@ impl fmt::Display for PolicyViolation { } } -/// Match a middleware host selector pattern using the runtime's glob semantics. -/// -/// Invalid or empty patterns return an error instead of silently becoming a -/// non-match. -pub fn middleware_host_matches(pattern: &str, host: &str) -> std::result::Result { - if pattern.is_empty() { - return Err("host pattern must not be empty".to_string()); - } - if pattern.chars().any(char::is_whitespace) { - return Err("host pattern must not contain whitespace".to_string()); - } - - let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) - .map_err(|error| format!("invalid host pattern: {error}"))?; - Ok(pattern.matches(&host.to_ascii_lowercase())) -} - -fn middleware_selector_matches_host( - middleware: &NetworkMiddlewareConfig, - host: &str, -) -> std::result::Result { - let Some(selector) = &middleware.endpoints else { - return Ok(false); - }; - let matches_include = selector - .include - .iter() - .try_fold(false, |matched, pattern| { - middleware_host_matches(pattern, host).map(|matches| matched || matches) - })?; - let matches_exclude = selector - .exclude - .iter() - .try_fold(false, |matched, pattern| { - middleware_host_matches(pattern, host).map(|matches| matched || matches) - })?; - Ok(matches_include && !matches_exclude) -} - /// Validate that a sandbox policy does not contain unsafe content. /// /// Returns `Ok(())` if the policy is safe, or `Err(violations)` listing all @@ -1513,96 +1361,7 @@ pub fn validate_sandbox_policy( } } - let mut middleware_names = HashSet::new(); - for middleware in &policy.network_middlewares { - if middleware.name.is_empty() { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: "name must not be empty".to_string(), - }); - } else if !middleware_names.insert(middleware.name.clone()) { - violations.push(PolicyViolation::DuplicateMiddlewareConfigName { - name: middleware.name.clone(), - }); - } - - if middleware.middleware.is_empty() { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: "implementation must not be empty".to_string(), - }); - } else if middleware.middleware.starts_with("openshell/") - && middleware.middleware != openshell_core::middleware::BUILTIN_SECRETS - { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: format!("unsupported built-in '{}'", middleware.middleware), - }); - } - - if !matches!( - middleware.on_error.as_str(), - "" | "fail_closed" | "fail_open" - ) { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: format!("invalid on_error '{}'", middleware.on_error), - }); - } - - let Some(selector) = &middleware.endpoints else { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: "endpoint selector is required".to_string(), - }); - continue; - }; - if selector.include.is_empty() { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: "endpoint selector must include at least one host pattern".to_string(), - }); - } - for pattern in selector.include.iter().chain(&selector.exclude) { - if let Err(reason) = middleware_host_matches(pattern, "validation.invalid") { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: format!("endpoint selector pattern '{pattern}' is invalid: {reason}"), - }); - } - } - - if middleware.middleware == openshell_core::middleware::BUILTIN_SECRETS { - let config = middleware.config.clone().unwrap_or_default(); - if let Err(error) = - openshell_core::middleware::validate_builtin_config(&middleware.middleware, &config) - { - violations.push(PolicyViolation::InvalidBuiltinMiddlewareConfig { - name: middleware.name.clone(), - reason: error.to_string(), - }); - } - } - - for (key, rule) in &policy.network_policies { - let policy_name = if rule.name.is_empty() { - key - } else { - &rule.name - }; - for endpoint in &rule.endpoints { - if endpoint.tls == "skip" - && middleware_selector_matches_host(middleware, &endpoint.host).unwrap_or(false) - { - violations.push(PolicyViolation::MiddlewareTlsSkipConflict { - middleware_name: middleware.name.clone(), - policy_name: policy_name.clone(), - host: endpoint.host.clone(), - }); - } - } - } - } + violations.extend(middleware::validate(policy)); if violations.is_empty() { Ok(()) @@ -2019,13 +1778,16 @@ network_policies: // ---- Policy validation tests ---- - fn middleware_config(name: &str, implementation: &str) -> NetworkMiddlewareConfig { - NetworkMiddlewareConfig { + fn middleware_config( + name: &str, + implementation: &str, + ) -> openshell_core::proto::NetworkMiddlewareConfig { + openshell_core::proto::NetworkMiddlewareConfig { name: name.into(), middleware: implementation.into(), config: None, on_error: String::new(), - endpoints: Some(MiddlewareEndpointSelector { + endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { include: vec!["api.example.com".into()], exclude: Vec::new(), }), diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs new file mode 100644 index 0000000000..ef10cb70f8 --- /dev/null +++ b/crates/openshell-policy/src/middleware.rs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! YAML schema and protobuf conversion for supervisor middleware policies. + +use std::collections::{BTreeMap, HashSet}; + +use openshell_core::proto::{MiddlewareEndpointSelector, NetworkMiddlewareConfig, SandboxPolicy}; +use openshell_core::proto_struct::{ + ProtoStructError, json_object_to_struct, struct_to_json_object, +}; +use serde::{Deserialize, Serialize}; + +use super::PolicyViolation; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NetworkMiddlewareConfigDef { + name: String, + middleware: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + config: BTreeMap, + #[serde(default, skip_serializing_if = "String::is_empty")] + on_error: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + endpoints: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MiddlewareEndpointSelectorDef { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + include: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + exclude: Vec, +} + +pub fn into_proto( + definitions: Vec, +) -> Result, ProtoStructError> { + definitions + .into_iter() + .map(|definition| { + Ok(NetworkMiddlewareConfig { + name: definition.name, + middleware: definition.middleware, + config: Some(json_object_to_struct( + definition.config.into_iter().collect(), + )?), + on_error: definition.on_error, + endpoints: definition + .endpoints + .map(|selector| MiddlewareEndpointSelector { + include: selector.include, + exclude: selector.exclude, + }), + }) + }) + .collect() +} + +pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec { + middlewares + .iter() + .map(|middleware| NetworkMiddlewareConfigDef { + name: middleware.name.clone(), + middleware: middleware.middleware.clone(), + config: middleware + .config + .as_ref() + .map(struct_to_json_object) + .unwrap_or_default() + .into_iter() + .collect(), + on_error: middleware.on_error.clone(), + endpoints: middleware.endpoints.as_ref().map(|selector| { + MiddlewareEndpointSelectorDef { + include: selector.include.clone(), + exclude: selector.exclude.clone(), + } + }), + }) + .collect() +} + +/// Match a middleware host selector pattern using the runtime's glob semantics. +/// +/// Invalid or empty patterns return an error instead of silently becoming a +/// non-match. +pub fn middleware_host_matches(pattern: &str, host: &str) -> Result { + if pattern.is_empty() { + return Err("host pattern must not be empty".to_string()); + } + if pattern.chars().any(char::is_whitespace) { + return Err("host pattern must not contain whitespace".to_string()); + } + + let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) + .map_err(|error| format!("invalid host pattern: {error}"))?; + Ok(pattern.matches(&host.to_ascii_lowercase())) +} + +fn selector_matches_host(middleware: &NetworkMiddlewareConfig, host: &str) -> Result { + let Some(selector) = &middleware.endpoints else { + return Ok(false); + }; + let matches_include = selector + .include + .iter() + .try_fold(false, |matched, pattern| { + middleware_host_matches(pattern, host).map(|matches| matched || matches) + })?; + let matches_exclude = selector + .exclude + .iter() + .try_fold(false, |matched, pattern| { + middleware_host_matches(pattern, host).map(|matches| matched || matches) + })?; + Ok(matches_include && !matches_exclude) +} + +pub fn validate(policy: &SandboxPolicy) -> Vec { + let mut violations = Vec::new(); + let mut names = HashSet::new(); + + for middleware in &policy.network_middlewares { + if middleware.name.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "name must not be empty".to_string(), + }); + } else if !names.insert(middleware.name.clone()) { + violations.push(PolicyViolation::DuplicateMiddlewareConfigName { + name: middleware.name.clone(), + }); + } + + if middleware.middleware.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "implementation must not be empty".to_string(), + }); + } else if middleware.middleware.starts_with("openshell/") + && middleware.middleware != openshell_core::middleware::BUILTIN_SECRETS + { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("unsupported built-in '{}'", middleware.middleware), + }); + } + + if !matches!( + middleware.on_error.as_str(), + "" | "fail_closed" | "fail_open" + ) { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("invalid on_error '{}'", middleware.on_error), + }); + } + + let Some(selector) = &middleware.endpoints else { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "endpoint selector is required".to_string(), + }); + continue; + }; + if selector.include.is_empty() { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: "endpoint selector must include at least one host pattern".to_string(), + }); + } + for pattern in selector.include.iter().chain(&selector.exclude) { + if let Err(reason) = middleware_host_matches(pattern, "validation.invalid") { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason: format!("endpoint selector pattern '{pattern}' is invalid: {reason}"), + }); + } + } + + if middleware.middleware == openshell_core::middleware::BUILTIN_SECRETS { + let config = middleware.config.clone().unwrap_or_default(); + if let Err(error) = + openshell_core::middleware::validate_builtin_config(&middleware.middleware, &config) + { + violations.push(PolicyViolation::InvalidBuiltinMiddlewareConfig { + name: middleware.name.clone(), + reason: error.to_string(), + }); + } + } + + for (key, rule) in &policy.network_policies { + let policy_name = if rule.name.is_empty() { + key + } else { + &rule.name + }; + for endpoint in &rule.endpoints { + if endpoint.tls == "skip" + && selector_matches_host(middleware, &endpoint.host).unwrap_or(false) + { + violations.push(PolicyViolation::MiddlewareTlsSkipConflict { + middleware_name: middleware.name.clone(), + policy_name: policy_name.clone(), + host: endpoint.host.clone(), + }); + } + } + } + } + + violations +} From 3b5875965266961bfc5b06ea9f5c6bc5575a7355 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 13:14:45 -0700 Subject: [PATCH 26/59] refactor(middleware): move host matching to core Signed-off-by: Piotr Mlocek --- Cargo.lock | 3 +- architecture/sandbox.md | 9 +++--- crates/openshell-core/Cargo.toml | 1 + crates/openshell-core/README.md | 8 ++--- crates/openshell-core/src/middleware.rs | 31 +++++++++++++++++++ crates/openshell-policy/Cargo.toml | 2 -- crates/openshell-policy/src/lib.rs | 16 +++++----- crates/openshell-policy/src/middleware.rs | 19 ++---------- .../openshell-supervisor-network/src/opa.rs | 10 +++--- 9 files changed, 56 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3552ea04ce..46a081025e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3630,6 +3630,7 @@ version = "0.0.0" dependencies = [ "base64 0.22.1", "chrono", + "glob", "ipnet", "miette", "prost", @@ -3769,10 +3770,8 @@ dependencies = [ name = "openshell-policy" version = "0.0.0" dependencies = [ - "glob", "miette", "openshell-core", - "prost-types", "serde", "serde_json", "serde_yml", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index a9f1208e06..9e984774bb 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -73,10 +73,11 @@ operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep the last-known-good service registry when a live config reload fails. Built-in -middleware identifiers and pure config validation live in `openshell-core` so -policy admission does not depend on the supervisor runtime implementation. The -policy and runtime also share the core JSON/protobuf adapter for middleware -configuration, keeping serialization consistent across that boundary. +middleware identifiers, host-selector matching, and pure config validation live +in `openshell-core` so policy admission does not depend on the supervisor +runtime implementation. The policy and runtime also share the core +JSON/protobuf adapter for middleware configuration, keeping serialization +consistent across that boundary. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 0ff6d06d6c..53735a6bb8 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -11,6 +11,7 @@ license.workspace = true repository.workspace = true [dependencies] +glob = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-core/README.md b/crates/openshell-core/README.md index e27ab167f5..b4cdb05f5d 100644 --- a/crates/openshell-core/README.md +++ b/crates/openshell-core/README.md @@ -53,10 +53,10 @@ behavior here, then consume it from the gateway, sandbox, and router. ## Middleware Contracts -Built-in supervisor middleware identifiers and pure configuration validation -live in `openshell_core::middleware`. Policy admission and the supervisor -runtime consume the same contract without introducing a dependency from the -policy crate to the supervisor implementation. +Built-in supervisor middleware identifiers, host-selector matching, and pure +configuration validation live in `openshell_core::middleware`. Policy admission +and the supervisor runtime consume the same contract without introducing a +dependency from the policy crate to the supervisor implementation. ## Protobuf Struct Conversion diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs index 6fb15b1c15..1688cd4f92 100644 --- a/crates/openshell-core/src/middleware.rs +++ b/crates/openshell-core/src/middleware.rs @@ -8,6 +8,23 @@ use miette::{Result, miette}; /// Binding identifier for the built-in secret redaction middleware. pub const BUILTIN_SECRETS: &str = "openshell/secrets"; +/// Match a middleware host selector pattern using the runtime's glob semantics. +/// +/// Matching is case-insensitive. Invalid or empty patterns return an error +/// instead of silently becoming a non-match. +pub fn host_matches(pattern: &str, host: &str) -> std::result::Result { + if pattern.is_empty() { + return Err("host pattern must not be empty".to_string()); + } + if pattern.chars().any(char::is_whitespace) { + return Err("host pattern must not contain whitespace".to_string()); + } + + let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) + .map_err(|error| format!("invalid host pattern: {error}"))?; + Ok(pattern.matches(&host.to_ascii_lowercase())) +} + /// Validate policy-owned configuration for a built-in middleware. pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { match implementation { @@ -39,6 +56,20 @@ fn validate_secrets_config(config: &prost_types::Struct) -> Result<()> { mod tests { use super::*; + #[test] + fn host_matching_is_case_insensitive() { + assert!(host_matches("*.Example.COM", "API.example.com").unwrap()); + assert!(!host_matches("*.example.com", "example.com").unwrap()); + assert!(host_matches("*", "deep.api.example.com").unwrap()); + } + + #[test] + fn host_matching_rejects_invalid_patterns() { + assert!(host_matches("", "api.example.com").is_err()); + assert!(host_matches("api .example.com", "api.example.com").is_err()); + assert!(host_matches("api[.example.com", "api.example.com").is_err()); + } + #[test] fn secrets_config_defaults_to_redact() { validate_builtin_config(BUILTIN_SECRETS, &prost_types::Struct::default()).unwrap(); diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 036964b722..16719de13d 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -11,9 +11,7 @@ license.workspace = true repository.workspace = true [dependencies] -glob = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } -prost-types = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yml = { workspace = true } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index d388e28a2f..d94fd9002e 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1826,15 +1826,13 @@ network_policies: fn validate_rejects_invalid_builtin_middleware_config() { let mut policy = restrictive_default_policy(); let mut middleware = middleware_config("redact-secrets", "openshell/secrets"); - middleware.config = Some(prost_types::Struct { - fields: std::iter::once(( - "secrets".into(), - prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("allow".into())), - }, - )) - .collect(), - }); + middleware.config = Some( + openshell_core::proto_struct::json_object_to_struct( + std::iter::once(("secrets".into(), serde_json::Value::String("allow".into()))) + .collect(), + ) + .unwrap(), + ); policy.network_middlewares.push(middleware); let violations = validate_sandbox_policy(&policy).expect_err("invalid config"); diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index ef10cb70f8..1e795f6009 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -13,6 +13,8 @@ use serde::{Deserialize, Serialize}; use super::PolicyViolation; +pub use openshell_core::middleware::host_matches as middleware_host_matches; + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct NetworkMiddlewareConfigDef { @@ -83,23 +85,6 @@ pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec Result { - if pattern.is_empty() { - return Err("host pattern must not be empty".to_string()); - } - if pattern.chars().any(char::is_whitespace) { - return Err("host pattern must not contain whitespace".to_string()); - } - - let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) - .map_err(|error| format!("invalid host pattern: {error}"))?; - Ok(pattern.matches(&host.to_ascii_lowercase())) -} - fn selector_matches_host(middleware: &NetworkMiddlewareConfig, host: &str) -> Result { let Some(selector) = &middleware.endpoints else { return Ok(false); diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 8702c6e6e2..c34ea9f613 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -741,14 +741,14 @@ fn middleware_selector_matches(config: ®orus::Value, host: &str) -> Result Vec { } for pattern in includes.iter().chain(&excludes) { if let Err(error) = - openshell_policy::middleware_host_matches(pattern, "validation.invalid") + openshell_core::middleware::host_matches(pattern, "validation.invalid") { errors.push(format!( "middleware config '{name}' has invalid endpoint selector pattern '{pattern}': {error}" @@ -1220,10 +1220,10 @@ fn global_selector_matches_any_middleware( let excludes = json_string_array(selector.get("exclude")); !includes.is_empty() && includes.iter().any(|pattern| { - openshell_policy::middleware_host_matches(pattern, host).unwrap_or(false) + openshell_core::middleware::host_matches(pattern, host).unwrap_or(false) }) && !excludes.iter().any(|pattern| { - openshell_policy::middleware_host_matches(pattern, host).unwrap_or(false) + openshell_core::middleware::host_matches(pattern, host).unwrap_or(false) }) }) } From 3b6fc104af78f8a8d2c92958c8816f0f02f18d06 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 3 Jul 2026 14:31:46 -0700 Subject: [PATCH 27/59] feat(middleware): align operations and ordering Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 4 +- crates/openshell-policy/src/lib.rs | 3 + crates/openshell-policy/src/middleware.rs | 8 ++ .../src/builtins/secrets.rs | 7 +- .../src/lib.rs | 83 +++++++++++++++---- .../src/l7/relay.rs | 3 + .../src/l7/rest.rs | 1 + .../openshell-supervisor-network/src/opa.rs | 16 +++- docs/extensibility/supervisor-middleware.mdx | 9 +- docs/reference/policy-schema.mdx | 4 +- docs/sandboxes/policies.mdx | 6 +- proto/middleware.proto | 22 +++-- proto/sandbox.proto | 2 + 13 files changed, 135 insertions(+), 33 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 9e984774bb..bc3982343b 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -68,7 +68,9 @@ are relayed but are not currently parsed for policy enforcement. For admitted HTTP requests, the proxy can run an ordered supervisor middleware chain before credential injection. Host selectors choose the chain independently -of the network rule that admitted the request. Built-ins run in-process; +of the network rule that admitted the request. Policy entries use integer order +values with stable name tie-breaking, and the gRPC contract represents operations +and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index d94fd9002e..b83e07d71e 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1493,6 +1493,7 @@ version: 1 network_middlewares: - name: global-redactor middleware: openshell/secrets + order: 20 on_error: fail_open endpoints: include: ["api.example.com", "*.service.test"] @@ -1520,6 +1521,7 @@ network_policies: assert_eq!(proto.network_middlewares.len(), 2); assert_eq!(proto.network_middlewares[0].name, "global-redactor"); assert_eq!(proto.network_middlewares[0].middleware, "openshell/secrets"); + assert_eq!(proto.network_middlewares[0].order, 20); assert_eq!(proto.network_middlewares[0].on_error, "fail_open"); assert_eq!( proto.network_middlewares[0] @@ -1785,6 +1787,7 @@ network_policies: openshell_core::proto::NetworkMiddlewareConfig { name: name.into(), middleware: implementation.into(), + order: 0, config: None, on_error: String::new(), endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 1e795f6009..71c5ac9a7a 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -20,6 +20,8 @@ pub use openshell_core::middleware::host_matches as middleware_host_matches; pub struct NetworkMiddlewareConfigDef { name: String, middleware: String, + #[serde(default, skip_serializing_if = "is_default")] + order: i32, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] config: BTreeMap, #[serde(default, skip_serializing_if = "String::is_empty")] @@ -28,6 +30,10 @@ pub struct NetworkMiddlewareConfigDef { endpoints: Option, } +fn is_default(value: &T) -> bool { + value == &T::default() +} + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct MiddlewareEndpointSelectorDef { @@ -46,6 +52,7 @@ pub fn into_proto( Ok(NetworkMiddlewareConfig { name: definition.name, middleware: definition.middleware, + order: definition.order, config: Some(json_object_to_struct( definition.config.into_iter().collect(), )?), @@ -67,6 +74,7 @@ pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec MiddlewareBinding { MiddlewareBinding { id: BINDING_ID.into(), - operation: OPERATION.into(), - phase: PHASE.into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: MAX_BODY_BYTES, } } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 9b067b31e1..5fc15a0ddf 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -18,14 +18,16 @@ use openshell_core::proto::middleware::v1::supervisor_middleware_server::Supervi use openshell_core::proto::{ Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, - SupervisorMiddlewareService, ValidateConfigRequest, + SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, SupervisorMiddlewareService, + ValidateConfigRequest, }; use tokio::sync::OnceCell; use tonic::Request; pub const API_VERSION: &str = "openshell.middleware.v1"; -const HTTP_REQUEST_OPERATION: &str = "HttpRequest"; -const PRE_CREDENTIALS_PHASE: &str = "pre_credentials"; +const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation = + SupervisorMiddlewareOperation::HttpRequest; +const PRE_CREDENTIALS_PHASE: SupervisorMiddlewarePhase = SupervisorMiddlewarePhase::PreCredentials; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OnError { FailClosed, @@ -48,6 +50,7 @@ impl OnError { pub struct ChainEntry { pub name: String, pub implementation: String, + pub order: i32, pub config: prost_types::Struct, pub on_error: OnError, } @@ -68,6 +71,7 @@ impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { Ok(Self { name: value.name.clone(), implementation: value.middleware.clone(), + order: value.order, config: value.config.clone().unwrap_or_default(), on_error: OnError::parse(&value.on_error)?, }) @@ -297,10 +301,12 @@ fn validate_external_manifest( binding.id )); } - if binding.operation != HTTP_REQUEST_OPERATION || binding.phase != PRE_CREDENTIALS_PHASE { + if binding.operation != HTTP_REQUEST_OPERATION as i32 + || binding.phase != PRE_CREDENTIALS_PHASE as i32 + { return Err(miette!( - "middleware binding '{}' must support {HTTP_REQUEST_OPERATION}/{PRE_CREDENTIALS_PHASE}", - binding.id + "middleware binding '{}' must support HTTP_REQUEST/PRE_CREDENTIALS", + binding.id, )); } let advertised = usize::try_from(binding.max_body_bytes).map_err(|_| { @@ -523,6 +529,8 @@ impl ChainRunner { pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result> { let manifests = self.manifests().await?; + let mut entries = entries.to_vec(); + sort_chain_entries(&mut entries); entries .iter() .map(|entry| { @@ -810,6 +818,15 @@ impl ChainRunner { } } +/// Sort middleware using the policy-defined priority and a stable name tie-breaker. +pub fn sort_chain_entries(entries: &mut [ChainEntry]) { + entries.sort_by(|left, right| { + left.order + .cmp(&right.order) + .then_with(|| left.name.cmp(&right.name)) + }); +} + fn build_evaluation( entry: &DescribedChainEntry, binding: &MiddlewareBinding, @@ -820,7 +837,7 @@ fn build_evaluation( HttpRequestEvaluation { api_version: API_VERSION.into(), binding_id: binding.id.clone(), - phase: binding.phase.clone(), + phase: binding.phase, context: Some(RequestContext { request_id: input.request_id.clone(), sandbox_id: input.sandbox_id.clone(), @@ -912,6 +929,7 @@ mod tests { ChainEntry { name: name.into(), implementation: BUILTIN_SECRETS.into(), + order: 0, config: prost_types::Struct { fields: std::iter::once(( "secrets".into(), @@ -951,6 +969,10 @@ mod tests { let input = input("payload"); let evaluation = build_evaluation(entry, binding, &input, &BTreeMap::new(), b"payload"); + assert_eq!( + evaluation.phase, + SupervisorMiddlewarePhase::PreCredentials as i32 + ); assert!( evaluation .context @@ -995,11 +1017,32 @@ mod tests { assert_eq!(outcome.applied.len(), 2); } + #[tokio::test] + async fn describe_chain_sorts_by_order_then_name() { + let mut later = entry("later", OnError::FailClosed); + later.order = 20; + let mut beta = entry("beta", OnError::FailClosed); + beta.order = 10; + let mut alpha = entry("alpha", OnError::FailClosed); + alpha.order = 10; + + let described = ChainRunner::default() + .describe_chain(&[later, beta, alpha]) + .await + .expect("describe ordered chain"); + let names: Vec<_> = described + .iter() + .map(|entry| entry.entry.name.as_str()) + .collect(); + assert_eq!(names, vec!["alpha", "beta", "later"]); + } + #[tokio::test] async fn fail_open_allows_unavailable_middleware() { let unavailable = ChainEntry { name: "missing".into(), implementation: "third-party/missing".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; @@ -1016,6 +1059,7 @@ mod tests { let unavailable = ChainEntry { name: "missing".into(), implementation: "third-party/missing".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }; @@ -1036,8 +1080,14 @@ mod tests { .into_inner(); assert_eq!(manifest.api_version, API_VERSION); assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); - assert_eq!(manifest.bindings[0].operation, "HttpRequest"); - assert_eq!(manifest.bindings[0].phase, "pre_credentials"); + assert_eq!( + manifest.bindings[0].operation, + SupervisorMiddlewareOperation::HttpRequest as i32 + ); + assert_eq!( + manifest.bindings[0].phase, + SupervisorMiddlewarePhase::PreCredentials as i32 + ); assert_eq!(manifest.bindings[0].max_body_bytes, 256 * 1024); } @@ -1088,8 +1138,8 @@ mod tests { service_version: "test".into(), bindings: vec![MiddlewareBinding { id: self.binding_id.clone(), - operation: "HttpRequest".into(), - phase: "pre_credentials".into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: self.max_body_bytes, }], })) @@ -1190,6 +1240,7 @@ mod tests { let unresolved = ChainEntry { name: "missing".into(), implementation: "third-party/missing".into(), + order: 10, config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; @@ -1214,6 +1265,7 @@ mod tests { let entry = ChainEntry { name: "external".into(), implementation: "example/redactor".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }; @@ -1229,7 +1281,7 @@ mod tests { .as_ref() .expect("described binding") .phase, - "pre_credentials" + SupervisorMiddlewarePhase::PreCredentials as i32 ); let outcome = runner @@ -1251,6 +1303,7 @@ mod tests { let external_entry = ChainEntry { name: "external".into(), implementation: "example/content-guard".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }; @@ -1284,8 +1337,8 @@ mod tests { service_version: "test".into(), bindings: vec![MiddlewareBinding { id: "example/content-guard".into(), - operation: HTTP_REQUEST_OPERATION.into(), - phase: PRE_CREDENTIALS_PHASE.into(), + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, }], }; @@ -1346,6 +1399,7 @@ mod tests { network_middlewares: vec![NetworkMiddlewareConfig { name: "guard".into(), middleware: "example/content-guard".into(), + order: 0, config: Some(prost_types::Struct::default()), on_error: "fail_closed".into(), endpoints: None, @@ -1367,6 +1421,7 @@ mod tests { &[ChainEntry { name: "guard".into(), implementation: "example/content-guard".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }], diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 87ab2f39f3..96e05f2722 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2801,6 +2801,7 @@ network_policies: let fail_open = ChainEntry { name: "m".into(), implementation: "openshell/secrets".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; @@ -2845,12 +2846,14 @@ network_policies: let resolved = ChainEntry { name: "redact".into(), implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }; let unresolved = ChainEntry { name: "missing".into(), implementation: "third-party/missing".into(), + order: 0, config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 4f2d37f087..5f276a1d49 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -33,6 +33,7 @@ async fn max_middleware_body_bytes() -> usize { .describe_chain(&[openshell_supervisor_middleware::ChainEntry { name: "test".into(), implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, }]) diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index c34ea9f613..6ee3b1daf7 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -729,6 +729,7 @@ fn global_middleware_entries(configs: &[regorus::Value], host: &str) -> Result Result { Ok(ChainEntry { name, implementation, + order: get_field(value, "order") + .and_then(|value| match value { + regorus::Value::Number(number) => number.as_i64(), + _ => None, + }) + .and_then(|value| i32::try_from(value).ok()) + .unwrap_or_default(), config: get_field(value, "config") .map(regorus_value_to_struct) .unwrap_or_default(), @@ -1630,6 +1638,7 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St let mut value = serde_json::json!({ "name": mw.name, "middleware": mw.middleware, + "order": mw.order, }); if let Some(config) = &mw.config { value["config"] = openshell_core::proto_struct::struct_to_json_value(config); @@ -6732,19 +6741,22 @@ network_policies: } #[test] - fn middleware_chain_uses_matching_selector_declaration_order() { + fn middleware_chain_uses_configured_order_and_name_tie_breaker() { let data = r#" network_middlewares: - name: global-redactor middleware: openshell/secrets + order: 20 endpoints: include: ["api.example.com"] - name: policy-redactor middleware: openshell/secrets + order: 10 endpoints: include: ["api.example.com"] - name: endpoint-redactor middleware: openshell/secrets + order: 10 endpoints: include: ["api.example.com"] network_policies: @@ -6775,7 +6787,7 @@ network_policies: let names: Vec<_> = chain.iter().map(|entry| entry.name.as_str()).collect(); assert_eq!( names, - vec!["global-redactor", "policy-redactor", "endpoint-redactor"] + vec!["endpoint-redactor", "policy-redactor", "global-redactor"] ); } diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index f2ea0c98a9..80b7b9d436 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -18,7 +18,7 @@ For each inspected HTTP request, the supervisor: 1. Evaluates network and L7 policy. 2. Selects middleware whose host selectors match the admitted destination. 3. Buffers the request body using the smallest body limit in the selected chain. -4. Runs matching middleware in policy declaration order. +4. Runs matching middleware by ascending `order`, using the policy-local name to break ties. 5. Applies allowed transformations, injects provider credentials, and forwards the request. Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. @@ -63,6 +63,7 @@ Add middleware configs to the top-level `network_middlewares` list: network_middlewares: - name: redact-secrets middleware: openshell/secrets + order: 10 config: secrets: redact on_error: fail_closed @@ -71,11 +72,11 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, implementation-owned `config`, failure behavior, and host selectors. +Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. `include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints. -Matching configs run once each in top-level declaration order. Different config names may reference the same binding and run as separate stages. Config names must be unique. +Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may reference the same binding and run as separate stages. Config names must be unique. See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. @@ -131,7 +132,7 @@ See [Logging](/observability/logging) for log access and [OCSF JSON Export](/obs ## Current Limitations - Middleware applies only to HTTP requests parsed by the supervisor. -- The supported operation and phase are `HttpRequest/pre_credentials`. +- The typed operation and phase are `HTTP_REQUEST/PRE_CREDENTIALS`. - Selection uses destination host include and exclude patterns. - Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. - Operator-run services support plaintext `http://` and TLS `https://` endpoints. HTTPS certificates must chain to a CA in the platform trust store. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index e36c535be9..266f5d8ca8 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -478,12 +478,13 @@ Identifies an executable that is permitted to use the associated endpoints. **Category:** Dynamic -An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once in list order before provider credential injection. +An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once by ascending `order`, with the policy-local name breaking ties, before provider credential injection. ```yaml showLineNumbers={false} network_middlewares: - name: redact-secrets middleware: openshell/secrets + order: 10 config: secrets: redact on_error: fail_closed @@ -496,6 +497,7 @@ network_middlewares: |---|---|---|---| | `name` | string | Yes | Policy-local config name. Names must be unique within the list. | | `middleware` | string | Yes | Built-in or operator-registered binding ID. `openshell/` is reserved for built-ins. | +| `order` | integer | No | Execution priority. Lower values run first; names break ties. Defaults to `0`. | | `config` | object | No | Implementation-owned configuration validated by the selected middleware. | | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | | `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists. Exclusions take precedence. | diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 1353fd640c..a49e89c19b 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -48,6 +48,7 @@ network_policies: network_middlewares: - name: redact-secrets middleware: openshell/secrets + order: 10 config: secrets: redact on_error: fail_closed @@ -68,7 +69,7 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in | `landlock` | Static | Configures Landlock LSM enforcement behavior. Set `compatibility` to `best_effort` (skip individual inaccessible paths while applying remaining rules) or `hard_requirement` (fail if any path is inaccessible or the required kernel ABI is unavailable). Refer to the [Policy Schema Reference](/reference/policy-schema#landlock) for the full behavior table. | | `process` | Static | Sets the OS-level identity for the agent process. `run_as_user` and `run_as_group` default to `sandbox`. Root (`root` or `0`) is rejected. The agent also runs with seccomp filters that block dangerous system calls. | | `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` goes through the proxy, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints without `protocol` allow the TCP stream through without inspecting payloads.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | -| `network_middlewares` | Dynamic | Declares ordered HTTP request middleware configs. After network and L7 policy admit a request, OpenShell matches each config's host selectors independently and runs matching entries in declaration order before credential injection. | +| `network_middlewares` | Dynamic | Declares ordered HTTP request middleware configs. After network and L7 policy admit a request, OpenShell matches each config's host selectors independently and runs matching entries by ascending `order`, using the policy-local name to break ties, before credential injection. | ## Supervisor Middleware @@ -78,6 +79,7 @@ Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies network_middlewares: - name: redact-secrets middleware: openshell/secrets + order: 10 config: secrets: redact on_error: fail_closed @@ -86,7 +88,7 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Matching entries run once each in top-level declaration order. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. +Matching entries run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. `openshell/secrets` is built into the supervisor. Operator-provided binding IDs must be registered before a policy can reference them. The gateway validates implementation-owned config before accepting the policy. diff --git a/proto/middleware.proto b/proto/middleware.proto index 7a4eb28dfe..421acd3d5c 100644 --- a/proto/middleware.proto +++ b/proto/middleware.proto @@ -38,10 +38,10 @@ message MiddlewareManifest { message MiddlewareBinding { // Stable binding id used by policy configuration and audit logs. string id = 1; - // Supported operation name. V1 supports "HttpRequest". - string operation = 2; - // Supported evaluation phase. V1 supports "pre_credentials". - string phase = 3; + // Supported operation. V1 supports HTTP_REQUEST. + SupervisorMiddlewareOperation operation = 2; + // Supported evaluation phase. V1 supports PRE_CREDENTIALS. + SupervisorMiddlewarePhase phase = 3; // Maximum request or replacement body this binding can process. uint64 max_body_bytes = 4; } @@ -71,7 +71,7 @@ message HttpRequestEvaluation { // Manifest binding id selected for this evaluation. string binding_id = 2; // Evaluation phase selected for this request. - string phase = 3; + SupervisorMiddlewarePhase phase = 3; // Sandbox and request identity available to the supervisor. RequestContext context = 4; // Validated service-specific policy configuration. @@ -84,6 +84,18 @@ message HttpRequestEvaluation { bytes body = 8; } +// Supervisor operation selected for middleware evaluation. +enum SupervisorMiddlewareOperation { + SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED = 0; + SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST = 1; +} + +// Ordered phase within a supervisor operation. +enum SupervisorMiddlewarePhase { + SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED = 0; + SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS = 1; +} + // RequestContext identifies the sandbox request being evaluated. message RequestContext { // Request id used to correlate middleware and supervisor logs. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index c4018573d6..a8a27349c9 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -77,6 +77,8 @@ message NetworkMiddlewareConfig { string on_error = 4; // Host selector controlling which admitted destinations use this config. MiddlewareEndpointSelector endpoints = 5; + // Deterministic execution order. Lower values run first; names break ties. + int32 order = 6; } // Host selector controlling which admitted destinations use a middleware config. From f4c0206404c1cb885767b7ff4b85715f56ee7862 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 8 Jul 2026 17:16:41 -0700 Subject: [PATCH 28/59] fix(middleware): tighten request handling Signed-off-by: Piotr Mlocek --- crates/openshell-core/src/proto_struct.rs | 54 ++++++++++++++++--- .../src/l7/middleware.rs | 30 ++++++++++- .../src/l7/rest.rs | 21 +++++++- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/crates/openshell-core/src/proto_struct.rs b/crates/openshell-core/src/proto_struct.rs index b90c52ca73..ce95c65f0c 100644 --- a/crates/openshell-core/src/proto_struct.rs +++ b/crates/openshell-core/src/proto_struct.rs @@ -8,8 +8,8 @@ use serde::{Deserialize, Deserializer, de::Error as _}; /// Errors converting JSON values into protobuf well-known types. #[derive(Debug, thiserror::Error)] pub enum ProtoStructError { - /// A JSON number cannot be represented by protobuf's double value. - #[error("JSON number {0} cannot be represented as a protobuf double")] + /// A JSON number cannot be represented exactly by protobuf's double value. + #[error("JSON number {0} cannot be represented exactly as a protobuf double")] UnrepresentableNumber(serde_json::Number), } @@ -34,11 +34,7 @@ pub fn json_value_to_proto( let kind = match value { serde_json::Value::Null => Kind::NullValue(0), serde_json::Value::Bool(value) => Kind::BoolValue(value), - serde_json::Value::Number(value) => Kind::NumberValue( - value - .as_f64() - .ok_or_else(|| ProtoStructError::UnrepresentableNumber(value.clone()))?, - ), + serde_json::Value::Number(value) => Kind::NumberValue(number_to_f64_exact(&value)?), serde_json::Value::String(value) => Kind::StringValue(value), serde_json::Value::Array(values) => Kind::ListValue(ListValue { values: values @@ -52,6 +48,27 @@ pub fn json_value_to_proto( Ok(Value { kind: Some(kind) }) } +fn number_to_f64_exact(value: &serde_json::Number) -> Result { + let number = value + .as_f64() + .ok_or_else(|| ProtoStructError::UnrepresentableNumber(value.clone()))?; + + let exact = value.as_i64().map_or_else( + || value.as_u64().is_none_or(integer_is_exact_in_f64), + |integer| integer_is_exact_in_f64(integer.unsigned_abs()), + ); + + exact + .then_some(number) + .ok_or_else(|| ProtoStructError::UnrepresentableNumber(value.clone())) +} + +fn integer_is_exact_in_f64(integer: u64) -> bool { + integer == 0 + || (u64::BITS - integer.leading_zeros()).saturating_sub(integer.trailing_zeros()) + <= f64::MANTISSA_DIGITS +} + /// Convert a protobuf Struct into a JSON object for typed serde decoding. #[must_use] pub fn struct_to_json_object( @@ -137,6 +154,29 @@ mod tests { assert_eq!(struct_to_json_value(&proto), json); } + #[test] + fn rejects_integer_that_cannot_round_trip_through_protobuf_double() { + let value = serde_json::json!(9_007_199_254_740_993_u64); + + let err = json_value_to_proto(value).expect_err("lossy integer must be rejected"); + + assert!(err.to_string().contains("9007199254740993")); + assert!(err.to_string().contains("exactly")); + } + + #[test] + fn accepts_integer_that_round_trips_through_protobuf_double() { + let value = serde_json::json!(9_007_199_254_740_992_u64); + + let proto = json_value_to_proto(value.clone()).expect("integer should be exact"); + + assert_eq!( + value_to_json(&proto).as_f64(), + value.as_f64(), + "protobuf Struct stores all numbers as doubles" + ); + } + #[derive(Debug, Default, Deserialize)] #[serde(default)] struct TestConfig { diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index a33d06e377..048baa3a29 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -205,7 +205,12 @@ fn safe_middleware_headers(headers: &[u8]) -> Result> { if name.is_empty() || matches!( name.as_str(), - "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" + "authorization" + | "proxy-authorization" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" ) || name.starts_with("x-amz-") || name.starts_with("x-openshell-credential") @@ -351,3 +356,26 @@ fn emit_middleware_events( ocsf_emit!(event); } } + +#[cfg(test)] +mod tests { + use super::safe_middleware_headers; + + #[test] + fn middleware_headers_exclude_origin_and_proxy_credentials() { + let headers = safe_middleware_headers( + b"GET http://api.example.test/v1 HTTP/1.1\r\n\ + Authorization: Bearer origin-secret\r\n\ + Proxy-Authorization: Basic proxy-secret\r\n\ + X-Request-ID: request-123\r\n\r\n", + ) + .expect("headers should parse"); + + assert_eq!( + headers.get("x-request-id").map(String::as_str), + Some("request-123") + ); + assert!(!headers.contains_key("authorization")); + assert!(!headers.contains_key("proxy-authorization")); + } +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 5f276a1d49..72b0e96f90 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -1170,6 +1170,7 @@ async fn collect_chunked_body( generation_guard: Option<&PolicyGenerationGuard>, max_wire_bytes: Option, ) -> Result> { + let max_decoded_bytes = max_wire_bytes.unwrap_or(MAX_REWRITE_BODY_BYTES); let mut read_state = ChunkedReadState { buffered_pos: 0, wire_bytes: 0, @@ -1207,9 +1208,9 @@ async fn collect_chunked_body( } } - if body.len().saturating_add(chunk_size) > MAX_REWRITE_BODY_BYTES { + if body.len().saturating_add(chunk_size) > max_decoded_bytes { return Err(miette!( - "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" + "decoded chunked body exceeds {max_decoded_bytes} byte buffer limit" )); } read_buffered_exact( @@ -3371,6 +3372,22 @@ mod tests { assert!(error.to_string().contains("wire representation")); } + #[tokio::test] + async fn middleware_chunked_body_can_exceed_credential_rewrite_limit() { + let max_body_bytes = 1024 * 1024; + let payload_len = 300 * 1024; + assert!(payload_len > MAX_REWRITE_BODY_BYTES); + let mut wire = format!("{payload_len:x}\r\n").into_bytes(); + wire.extend(std::iter::repeat_n(b'x', payload_len)); + wire.extend_from_slice(b"\r\n0\r\n\r\n"); + + let body = collect_chunked_body(&mut tokio::io::empty(), &wire, None, Some(max_body_bytes)) + .await + .expect("middleware cap should control chunked body collection"); + + assert_eq!(body.len(), payload_len); + } + /// SEC-009: Bare LF in headers enables header injection. #[tokio::test] async fn reject_bare_lf_in_headers() { From faea5e0f7002a226592f3e836846292ee4bfe948 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 10:51:37 -0700 Subject: [PATCH 29/59] feat(middleware): match hosts with DNS-label-aware patterns Replace whole-string glob host matching with a shared, compiled host-pattern module in openshell-core. `*` matches exactly one DNS label and `**` matches one or more labels, mirroring the endpoint glob semantics in sandbox-policy.rego; parity is pinned by a table test that runs the same pattern/host pairs through both the Rust matcher and regorus. Brace alternates are rejected at validation because the two glob dialects disagree on them. Middleware selectors compile once into HostSelector, the tls-skip conflict check detects conservative pattern overlap so wildcard endpoints no longer evade it, and the supervisor's local-file JSON validation routes through the same typed validator as protobuf policies instead of an OPA-specific copy. Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 14 +- crates/openshell-core/src/host_pattern.rs | 348 ++++++++++++++++++ crates/openshell-core/src/lib.rs | 1 + crates/openshell-core/src/middleware.rs | 31 -- crates/openshell-policy/src/lib.rs | 36 +- crates/openshell-policy/src/middleware.rs | 102 +++-- .../data/sandbox-policy.rego | 8 +- .../src/l7/token_grant_injection.rs | 57 ++- .../openshell-supervisor-network/src/opa.rs | 306 ++++++++------- docs/extensibility/supervisor-middleware.mdx | 2 +- docs/reference/policy-schema.mdx | 2 +- 11 files changed, 650 insertions(+), 257 deletions(-) create mode 100644 crates/openshell-core/src/host_pattern.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index bc3982343b..489cf20f06 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -75,11 +75,15 @@ operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and policy-owned config before delivery. Supervisors keep the last-known-good service registry when a live config reload fails. Built-in -middleware identifiers, host-selector matching, and pure config validation live -in `openshell-core` so policy admission does not depend on the supervisor -runtime implementation. The policy and runtime also share the core -JSON/protobuf adapter for middleware configuration, keeping serialization -consistent across that boundary. +middleware identifiers live in `openshell-core::middleware`; reusable compiled +DNS host patterns and selectors live in the neutral +`openshell-core::host_pattern` module. Structural policy validation lives in +`openshell-policy`; the supervisor's local-file path projects its JSON into the +same typed validator instead of maintaining an OPA-specific copy. Rego exposes +the middleware list as policy data, but Rust performs selector validation, +overlap detection, matching, and chain ordering. The policy and runtime also +share the core JSON/protobuf adapter for middleware configuration, keeping +serialization consistent across that boundary. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-core/src/host_pattern.rs b/crates/openshell-core/src/host_pattern.rs new file mode 100644 index 0000000000..f5d9a099a0 --- /dev/null +++ b/crates/openshell-core/src/host_pattern.rs @@ -0,0 +1,348 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! DNS-label-aware host patterns and selectors. + +use std::collections::{HashSet, VecDeque}; + +/// A validated, compiled DNS host pattern. +/// +/// Matching is case-insensitive. A `*` wildcard stays within one DNS label, +/// while a label consisting only of `**` consumes one or more labels. These +/// semantics mirror the Rego endpoint glob matching used for network policy +/// admission (`glob.match` with a `.` delimiter), so a pattern copied from a +/// network endpoint selects exactly the hosts that endpoint admits. +#[derive(Clone)] +pub struct HostPattern { + source: String, + labels: Vec, +} + +#[derive(Clone)] +enum HostLabelPattern { + Recursive, + Label { + source: String, + pattern: glob::Pattern, + literal: bool, + }, +} + +/// A validated destination host selector. +/// +/// Includes are evaluated first and exclusions take precedence. Constructing +/// the selector compiles every pattern once, so request-time selection cannot +/// fail due to malformed input. +#[derive(Clone)] +pub struct HostSelector { + includes: Vec, + excludes: Vec, +} + +impl HostSelector { + pub fn new(include: &[String], exclude: &[String]) -> Result { + if include.is_empty() { + return Err("endpoint selector must include at least one host pattern".to_string()); + } + Ok(Self { + includes: include + .iter() + .map(|pattern| HostPattern::new(pattern)) + .collect::>()?, + excludes: exclude + .iter() + .map(|pattern| HostPattern::new(pattern)) + .collect::>()?, + }) + } + + #[must_use] + pub fn matches(&self, host: &str) -> bool { + self.includes.iter().any(|pattern| pattern.matches(host)) + && !self.excludes.iter().any(|pattern| pattern.matches(host)) + } + + /// Conservatively determine whether this selector can match a concrete + /// host admitted by `candidate`. + #[must_use] + pub fn may_match_pattern(&self, candidate: &HostPattern) -> bool { + self.includes.iter().any(|include| { + if !include.overlaps(candidate) { + return false; + } + + let concrete_intersection = candidate.literal().or_else(|| include.literal()); + let excluded = concrete_intersection.map_or_else( + || { + self.excludes.iter().any(|excluded| { + excluded.is_universal() + || excluded.source == include.source + || excluded.source == candidate.source + }) + }, + |host| self.excludes.iter().any(|excluded| excluded.matches(host)), + ); + !excluded + }) + } +} + +impl HostPattern { + pub fn new(pattern: &str) -> Result { + if pattern.is_empty() { + return Err("host pattern must not be empty".to_string()); + } + if pattern.chars().any(char::is_whitespace) { + return Err("host pattern must not contain whitespace".to_string()); + } + // The glob crate treats braces as literal characters while endpoint + // glob matching expands them, so a brace pattern would silently never + // match a real host. Fail loud instead. + if pattern.chars().any(|ch| matches!(ch, '{' | '}')) { + return Err( + "host pattern must not contain brace alternates; list each host pattern separately" + .to_string(), + ); + } + + let source = pattern.to_ascii_lowercase(); + if source.split('.').any(str::is_empty) { + return Err("host pattern must not contain empty DNS labels".to_string()); + } + let labels = source + .split('.') + .map(|label| { + if label == "**" { + return Ok(HostLabelPattern::Recursive); + } + let pattern = glob::Pattern::new(label) + .map_err(|error| format!("invalid host pattern: {error}"))?; + Ok(HostLabelPattern::Label { + source: label.to_string(), + pattern, + literal: !label.chars().any(|ch| matches!(ch, '*' | '?' | '[')), + }) + }) + .collect::, String>>()?; + Ok(Self { source, labels }) + } + + #[must_use] + pub fn matches(&self, host: &str) -> bool { + let host = host.to_ascii_lowercase(); + if host.split('.').any(str::is_empty) { + return false; + } + self.matches_labels(&host.split('.').collect::>()) + } + + fn matches_labels(&self, host: &[&str]) -> bool { + let mut pending = vec![(0, 0)]; + let mut visited = HashSet::new(); + while let Some((pattern_idx, host_idx)) = pending.pop() { + if !visited.insert((pattern_idx, host_idx)) { + continue; + } + if pattern_idx == self.labels.len() && host_idx == host.len() { + return true; + } + match self.labels.get(pattern_idx) { + Some(HostLabelPattern::Recursive) if host_idx < host.len() => { + pending.push((pattern_idx + 1, host_idx + 1)); + pending.push((pattern_idx, host_idx + 1)); + } + Some(HostLabelPattern::Label { pattern, .. }) if host_idx < host.len() => { + if pattern.matches(host[host_idx]) { + pending.push((pattern_idx + 1, host_idx + 1)); + } + } + Some(_) | None => {} + } + } + false + } + + fn literal(&self) -> Option<&str> { + self.labels + .iter() + .all(|label| matches!(label, HostLabelPattern::Label { literal: true, .. })) + .then_some(self.source.as_str()) + } + + fn is_universal(&self) -> bool { + matches!(self.labels.as_slice(), [HostLabelPattern::Recursive]) + } + + fn overlaps(&self, other: &Self) -> bool { + let mut pending = VecDeque::from([(0, 0)]); + let mut visited = HashSet::new(); + while let Some((left_idx, right_idx)) = pending.pop_front() { + if !visited.insert((left_idx, right_idx)) { + continue; + } + if left_idx == self.labels.len() && right_idx == other.labels.len() { + return true; + } + + let (Some(left), Some(right)) = + (self.labels.get(left_idx), other.labels.get(right_idx)) + else { + continue; + }; + + // Every transition consumes exactly one shared host label, so a + // `**` on either side must consume at least one label before it + // can complete — the same minimum the matcher enforces. + match (left, right) { + (HostLabelPattern::Recursive, HostLabelPattern::Recursive) => { + pending.push_back((left_idx + 1, right_idx + 1)); + pending.push_back((left_idx, right_idx + 1)); + pending.push_back((left_idx + 1, right_idx)); + } + (HostLabelPattern::Recursive, HostLabelPattern::Label { .. }) => { + pending.push_back((left_idx + 1, right_idx + 1)); + pending.push_back((left_idx, right_idx + 1)); + } + (HostLabelPattern::Label { .. }, HostLabelPattern::Recursive) => { + pending.push_back((left_idx + 1, right_idx + 1)); + pending.push_back((left_idx + 1, right_idx)); + } + (left, right) if label_patterns_may_overlap(left, right) => { + pending.push_back((left_idx + 1, right_idx + 1)); + } + _ => {} + } + } + false + } +} + +/// Match a host using DNS-label-aware glob semantics. +/// +/// Matching is case-insensitive. `*` cannot cross a `.` label boundary, while +/// a label consisting of `**` consumes one or more labels. Invalid or empty +/// patterns return an error instead of silently becoming a non-match. +pub fn host_matches(pattern: &str, host: &str) -> Result { + Ok(HostPattern::new(pattern)?.matches(host)) +} + +/// Conservatively determine whether two DNS host patterns can match the same +/// concrete host. +pub fn host_patterns_overlap(left: &str, right: &str) -> Result { + let left = HostPattern::new(left)?; + let right = HostPattern::new(right)?; + Ok(left.overlaps(&right)) +} + +/// Determine whether a selector can match any concrete host admitted by a host +/// pattern. +/// +/// This is conservative for intersections between two non-literal globs: +/// exclusions suppress a conflict only when they cover a known concrete +/// intersection or exactly cover one of the intersecting patterns. +pub fn selector_may_match_pattern( + include: &[String], + exclude: &[String], + candidate: &str, +) -> Result { + let selector = HostSelector::new(include, exclude)?; + let candidate = HostPattern::new(candidate)?; + Ok(selector.may_match_pattern(&candidate)) +} + +fn label_patterns_may_overlap(left: &HostLabelPattern, right: &HostLabelPattern) -> bool { + let ( + HostLabelPattern::Label { + source: left_source, + pattern: left_pattern, + literal: left_literal, + }, + HostLabelPattern::Label { + source: right_source, + pattern: right_pattern, + literal: right_literal, + }, + ) = (left, right) + else { + return true; + }; + + match (*left_literal, *right_literal) { + (true, true) => left_source == right_source, + (true, false) => right_pattern.matches(left_source), + (false, true) => left_pattern.matches(right_source), + (false, false) => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_matching_is_case_insensitive() { + assert!(host_matches("*.Example.COM", "API.example.com").unwrap()); + assert!(!host_matches("*.example.com", "example.com").unwrap()); + assert!(!host_matches("*.example.com", "deep.api.example.com").unwrap()); + assert!(host_matches("**.example.com", "deep.api.example.com").unwrap()); + assert!(host_matches("*-api.example.com", "tenant-api.example.com").unwrap()); + assert!(!host_matches("*", "deep.api.example.com").unwrap()); + } + + #[test] + fn host_matching_rejects_invalid_patterns() { + assert!(host_matches("", "api.example.com").is_err()); + assert!(host_matches("api .example.com", "api.example.com").is_err()); + assert!(host_matches("api[.example.com", "api.example.com").is_err()); + assert!(host_matches("*.{prod,staging}.example.com", "api.prod.example.com").is_err()); + assert!(host_matches("api**.example.com", "apix.example.com").is_err()); + } + + #[test] + fn recursive_wildcard_requires_at_least_one_label() { + assert!(host_matches("**.example.com", "api.example.com").unwrap()); + assert!(!host_matches("**.example.com", "example.com").unwrap()); + assert!(host_matches("api.**.com", "api.x.y.com").unwrap()); + assert!(!host_matches("api.**.com", "api.com").unwrap()); + } + + #[test] + fn universal_wildcard_matches_any_host() { + for host in [ + "com", + "localhost", + "example.com", + "deep.api.example.com", + "xn--bcher-kva.example", + "tenant-api.internal", + "192.168.1.1", + "[::1]", + ] { + assert!(host_matches("**", host).unwrap(), "** must match {host}"); + } + + // Hosts with empty labels are invalid and never match, even for `**`. + assert!(!host_matches("**", "").unwrap()); + assert!(!host_matches("**", "api.example.com.").unwrap()); + assert!(!host_matches("**", "api..example.com").unwrap()); + } + + #[test] + fn host_pattern_overlap_handles_concrete_and_wildcard_hosts() { + assert!(host_patterns_overlap("*.example.com", "api.example.com").unwrap()); + assert!(host_patterns_overlap("**.example.com", "deep.api.example.com").unwrap()); + assert!(!host_patterns_overlap("*.example.com", "deep.api.example.com").unwrap()); + assert!(!host_patterns_overlap("*.example.com", "*.other.com").unwrap()); + assert!(host_patterns_overlap("**.example.com", "*.api.example.com").unwrap()); + assert!(!host_patterns_overlap("**.example.com", "example.com").unwrap()); + } + + #[test] + fn selector_pattern_overlap_honors_concrete_exclusions() { + let include = vec!["*.example.com".to_string()]; + let exclude = vec!["api.example.com".to_string()]; + + assert!(!selector_may_match_pattern(&include, &exclude, "api.example.com").unwrap()); + assert!(selector_may_match_pattern(&include, &exclude, "*.example.com").unwrap()); + } +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 9ba041dd85..4595f8965a 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -20,6 +20,7 @@ pub mod forward; pub mod google_cloud; pub mod gpu; pub mod grpc_client; +pub mod host_pattern; pub mod image; pub mod inference; pub mod metadata; diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs index 1688cd4f92..6fb15b1c15 100644 --- a/crates/openshell-core/src/middleware.rs +++ b/crates/openshell-core/src/middleware.rs @@ -8,23 +8,6 @@ use miette::{Result, miette}; /// Binding identifier for the built-in secret redaction middleware. pub const BUILTIN_SECRETS: &str = "openshell/secrets"; -/// Match a middleware host selector pattern using the runtime's glob semantics. -/// -/// Matching is case-insensitive. Invalid or empty patterns return an error -/// instead of silently becoming a non-match. -pub fn host_matches(pattern: &str, host: &str) -> std::result::Result { - if pattern.is_empty() { - return Err("host pattern must not be empty".to_string()); - } - if pattern.chars().any(char::is_whitespace) { - return Err("host pattern must not contain whitespace".to_string()); - } - - let pattern = glob::Pattern::new(&pattern.to_ascii_lowercase()) - .map_err(|error| format!("invalid host pattern: {error}"))?; - Ok(pattern.matches(&host.to_ascii_lowercase())) -} - /// Validate policy-owned configuration for a built-in middleware. pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { match implementation { @@ -56,20 +39,6 @@ fn validate_secrets_config(config: &prost_types::Struct) -> Result<()> { mod tests { use super::*; - #[test] - fn host_matching_is_case_insensitive() { - assert!(host_matches("*.Example.COM", "API.example.com").unwrap()); - assert!(!host_matches("*.example.com", "example.com").unwrap()); - assert!(host_matches("*", "deep.api.example.com").unwrap()); - } - - #[test] - fn host_matching_rejects_invalid_patterns() { - assert!(host_matches("", "api.example.com").is_err()); - assert!(host_matches("api .example.com", "api.example.com").is_err()); - assert!(host_matches("api[.example.com", "api.example.com").is_err()); - } - #[test] fn secrets_config_defaults_to_redact() { validate_builtin_config(BUILTIN_SECRETS, &prost_types::Struct::default()).unwrap(); diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index b83e07d71e..2cc4d98a0a 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -34,6 +34,7 @@ pub use merge::{ merge_policy, policy_covers_rule, }; pub use middleware::middleware_host_matches; +pub use middleware::validate_json as validate_network_middleware_json; // --------------------------------------------------------------------------- // YAML serde types (canonical — used for both parsing and serialization) @@ -1940,7 +1941,9 @@ network_policies: fn middleware_host_selector_matching_is_case_insensitive() { assert!(middleware_host_matches("*.Example.COM", "API.example.com").unwrap()); assert!(!middleware_host_matches("*.example.com", "example.com").unwrap()); - assert!(middleware_host_matches("*", "deep.api.example.com").unwrap()); + assert!(!middleware_host_matches("*.example.com", "deep.api.example.com").unwrap()); + assert!(middleware_host_matches("**.example.com", "deep.api.example.com").unwrap()); + assert!(!middleware_host_matches("**.example.com", "example.com").unwrap()); } #[test] @@ -1974,6 +1977,37 @@ network_policies: ))); } + #[test] + fn validate_rejects_concrete_selector_overlapping_tls_skip_wildcard() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/secrets"); + middleware.endpoints.as_mut().unwrap().include = vec!["api.example.com".into()]; + policy.network_middlewares.push(middleware); + policy.network_policies.insert( + "api".into(), + NetworkPolicyRule { + name: "api".into(), + endpoints: vec![NetworkEndpoint { + host: "*.example.com".into(), + port: 443, + tls: "skip".into(), + ..Default::default() + }], + binaries: Vec::new(), + }, + ); + + let violations = validate_sandbox_policy(&policy).expect_err("tls skip conflict"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::MiddlewareTlsSkipConflict { + middleware_name, + policy_name, + host, + } if middleware_name == "redactor" && policy_name == "api" && host == "*.example.com" + ))); + } + #[test] fn validate_rejects_non_sandbox_user() { let mut policy = restrictive_default_policy(); diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 71c5ac9a7a..52b70596dc 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -5,7 +5,10 @@ use std::collections::{BTreeMap, HashSet}; -use openshell_core::proto::{MiddlewareEndpointSelector, NetworkMiddlewareConfig, SandboxPolicy}; +use openshell_core::proto::{ + MiddlewareEndpointSelector, NetworkEndpoint, NetworkMiddlewareConfig, NetworkPolicyRule, + SandboxPolicy, +}; use openshell_core::proto_struct::{ ProtoStructError, json_object_to_struct, struct_to_json_object, }; @@ -13,7 +16,8 @@ use serde::{Deserialize, Serialize}; use super::PolicyViolation; -pub use openshell_core::middleware::host_matches as middleware_host_matches; +pub use openshell_core::host_pattern::host_matches as middleware_host_matches; +use openshell_core::host_pattern::{HostPattern, HostSelector}; #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -43,6 +47,33 @@ struct MiddlewareEndpointSelectorDef { exclude: Vec, } +/// Middleware-relevant projection of the runtime policy JSON accepted by the +/// supervisor's local-file mode. Unrelated network and L7 fields are ignored; +/// middleware entries retain their strict canonical schema. +#[derive(Debug, Default, Deserialize)] +struct MiddlewareValidationPolicyDef { + #[serde(default)] + network_middlewares: Vec, + #[serde(default)] + network_policies: BTreeMap, +} + +#[derive(Debug, Default, Deserialize)] +struct MiddlewareValidationNetworkPolicyDef { + #[serde(default)] + name: String, + #[serde(default)] + endpoints: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct MiddlewareValidationEndpointDef { + #[serde(default)] + host: String, + #[serde(default)] + tls: String, +} + pub fn into_proto( definitions: Vec, ) -> Result, ProtoStructError> { @@ -93,23 +124,38 @@ pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec Result { - let Some(selector) = &middleware.endpoints else { - return Ok(false); - }; - let matches_include = selector - .include - .iter() - .try_fold(false, |matched, pattern| { - middleware_host_matches(pattern, host).map(|matches| matched || matches) - })?; - let matches_exclude = selector - .exclude - .iter() - .try_fold(false, |matched, pattern| { - middleware_host_matches(pattern, host).map(|matches| matched || matches) - })?; - Ok(matches_include && !matches_exclude) +/// Validate middleware configuration from the supervisor's runtime policy +/// JSON through the same typed validator used for protobuf policies. +pub fn validate_json(data: &serde_json::Value) -> Result, String> { + let definition: MiddlewareValidationPolicyDef = serde_json::from_value(data.clone()) + .map_err(|error| format!("failed to parse network middleware policy: {error}"))?; + let network_middlewares = into_proto(definition.network_middlewares) + .map_err(|error| format!("failed to convert network middleware config: {error}"))?; + let network_policies = definition + .network_policies + .into_iter() + .map(|(key, rule)| { + let rule = NetworkPolicyRule { + name: rule.name, + endpoints: rule + .endpoints + .into_iter() + .map(|endpoint| NetworkEndpoint { + host: endpoint.host, + tls: endpoint.tls, + ..Default::default() + }) + .collect(), + ..Default::default() + }; + (key, rule) + }) + .collect(); + Ok(validate(&SandboxPolicy { + network_middlewares, + network_policies, + ..Default::default() + })) } pub fn validate(policy: &SandboxPolicy) -> Vec { @@ -165,14 +211,21 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { reason: "endpoint selector must include at least one host pattern".to_string(), }); } + let mut selector_valid = !selector.include.is_empty(); for pattern in selector.include.iter().chain(&selector.exclude) { - if let Err(reason) = middleware_host_matches(pattern, "validation.invalid") { + if let Err(reason) = HostPattern::new(pattern) { + selector_valid = false; violations.push(PolicyViolation::InvalidMiddlewareConfig { name: middleware.name.clone(), reason: format!("endpoint selector pattern '{pattern}' is invalid: {reason}"), }); } } + let compiled_selector = if selector_valid { + HostSelector::new(&selector.include, &selector.exclude).ok() + } else { + None + }; if middleware.middleware == openshell_core::middleware::BUILTIN_SECRETS { let config = middleware.config.clone().unwrap_or_default(); @@ -193,9 +246,12 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { &rule.name }; for endpoint in &rule.endpoints { - if endpoint.tls == "skip" - && selector_matches_host(middleware, &endpoint.host).unwrap_or(false) - { + let overlaps_tls_skip = endpoint.tls == "skip" + && compiled_selector.as_ref().is_some_and(|selector| { + HostPattern::new(&endpoint.host) + .is_ok_and(|endpoint| selector.may_match_pattern(&endpoint)) + }); + if overlaps_tls_skip { violations.push(PolicyViolation::MiddlewareTlsSkipConflict { middleware_name: middleware.name.clone(), policy_name: policy_name.clone(), diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index fcc5838e1f..255961af04 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -854,6 +854,8 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { count(_matching_endpoint_configs) > 0 } +# Expose middleware policy data to Rust. Selection and validation stay in Rust; +# Rego does not evaluate middleware selectors. network_middlewares := object.get(data, "network_middlewares", []) _policy_has_exact_declared_endpoint(policy) if { @@ -909,7 +911,7 @@ endpoint_path_matches_request(ep, request) if { } # An endpoint has extended config if it specifies L7 protocol, allowed_ips, -# middleware, or an explicit tls mode (e.g. tls: skip). +# or an explicit tls mode (e.g. tls: skip). endpoint_has_extended_config(ep) if { ep.protocol } @@ -918,10 +920,6 @@ endpoint_has_extended_config(ep) if { count(object.get(ep, "allowed_ips", [])) > 0 } -endpoint_has_extended_config(ep) if { - count(object.get(ep, "middleware", [])) > 0 -} - endpoint_has_extended_config(ep) if { ep.tls } diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 0d7c18e996..07a8c641f1 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -199,45 +199,16 @@ fn dynamic_credential_key_match_score( return None; } - let host_lc = host.to_ascii_lowercase(); - let endpoint_host_lc = endpoint_host.to_ascii_lowercase(); - if !host_pattern_matches(&endpoint_host_lc, &host_lc) + if !openshell_core::host_pattern::host_matches(endpoint_host, host).unwrap_or(false) || !crate::l7::endpoint_path_matches(endpoint_path, request_path) { return None; } - Some(host_pattern_specificity(&endpoint_host_lc) + endpoint_path_specificity(endpoint_path)) -} - -fn host_pattern_matches(pattern: &str, host: &str) -> bool { - if pattern == host { - return true; - } - if !pattern.contains('*') { - return false; - } - - let pattern_labels: Vec<&str> = pattern.split('.').collect(); - let host_labels: Vec<&str> = host.split('.').collect(); - host_pattern_labels_match(&pattern_labels, &host_labels) -} - -fn host_pattern_labels_match(pattern: &[&str], host: &[&str]) -> bool { - match pattern.split_first() { - None => host.is_empty(), - Some((label, rest)) if *label == "**" => { - host_pattern_labels_match(rest, host) - || (!host.is_empty() && host_pattern_labels_match(pattern, &host[1..])) - } - Some((label, rest)) if *label == "*" => { - !host.is_empty() && host_pattern_labels_match(rest, &host[1..]) - } - Some((literal, rest)) => { - host.first().is_some_and(|label| label == literal) - && host_pattern_labels_match(rest, &host[1..]) - } - } + Some( + host_pattern_specificity(&endpoint_host.to_ascii_lowercase()) + + endpoint_path_specificity(endpoint_path), + ) } fn host_pattern_specificity(pattern: &str) -> u32 { @@ -575,6 +546,24 @@ mod tests { )); } + #[test] + fn dynamic_credential_key_matches_case_insensitive_intra_label_wildcard() { + let key = "*-API.Example.COM\t443\t\tprovider:access_token"; + + assert!(dynamic_credential_key_matches( + key, + "tenant-api.example.com", + 443, + "/anything" + )); + assert!(!dynamic_credential_key_matches( + key, + "api.deep.example.com", + 443, + "/anything" + )); + } + #[test] fn dynamic_credential_key_matches_double_wildcard_hosts() { let key = "**.example.com\t443\t\tprovider:access_token"; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 6ee3b1daf7..324d09785e 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -8,13 +8,13 @@ //! on every proxy CONNECT request. use miette::Result; +use openshell_core::host_pattern::HostSelector; use openshell_core::policy::{ FilesystemPolicy, LandlockCompatibility, LandlockPolicy, ProcessPolicy, }; use openshell_core::proto::SandboxPolicy as ProtoSandboxPolicy; use openshell_policy::L7ConfigStanza; use openshell_supervisor_middleware::{ChainEntry, ChainRunner, MiddlewareRegistry}; -use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{ Arc, Mutex, RwLock, @@ -698,10 +698,6 @@ fn query_middleware_chain_locked( engine: &mut regorus::Engine, input: &NetworkInput, ) -> Result> { - engine - .set_input_json(&network_input_json(input).to_string()) - .map_err(|e| miette::miette!("{e}"))?; - let configs_val = engine .eval_rule("data.openshell.sandbox.network_middlewares".into()) .map_err(|e| miette::miette!("{e}"))?; @@ -737,23 +733,10 @@ fn middleware_selector_matches(config: ®orus::Value, host: &str) -> Result Result { @@ -877,11 +860,16 @@ fn preprocess_yaml_data(yaml_str: &str) -> Result { } // Validate BEFORE expanding presets (catches user errors like rules+access) - let middleware_errors = validate_middleware_policies(&data); + let middleware_errors = openshell_policy::validate_network_middleware_json(&data) + .map_err(|error| miette::miette!(error))?; if !middleware_errors.is_empty() { return Err(miette::miette!( "middleware policy validation failed:\n{}", - middleware_errors.join("\n") + middleware_errors + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") )); } @@ -1105,137 +1093,6 @@ fn normalize_l7_rule_aliases( } } -fn validate_middleware_policies(data: &serde_json::Value) -> Vec { - let mut errors = Vec::new(); - let middlewares = data - .get("network_middlewares") - .and_then(serde_json::Value::as_array) - .map_or(&[][..], Vec::as_slice); - let mut names = HashSet::new(); - for mw in middlewares { - let name = mw - .get("name") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(); - let implementation = mw - .get("middleware") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(); - if name.is_empty() { - errors.push("network_middlewares entry has empty name".to_string()); - } else if !names.insert(name.to_string()) { - errors.push(format!("duplicate middleware config '{name}'")); - } - if implementation.is_empty() { - errors.push(format!( - "middleware config '{name}' has empty implementation" - )); - } - if implementation.starts_with("openshell/") - && implementation != openshell_supervisor_middleware::BUILTIN_SECRETS - { - errors.push(format!( - "middleware config '{name}' references unsupported built-in '{implementation}'" - )); - } - let on_error = mw - .get("on_error") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(); - if !matches!(on_error, "" | "fail_closed" | "fail_open") { - errors.push(format!( - "middleware config '{name}' has invalid on_error '{on_error}'" - )); - } - - let Some(selector) = mw.get("endpoints") else { - errors.push(format!( - "middleware config '{name}' requires an endpoint selector" - )); - continue; - }; - let includes = json_string_array(selector.get("include")); - let excludes = json_string_array(selector.get("exclude")); - if includes.is_empty() { - errors.push(format!( - "middleware config '{name}' endpoint selector must include at least one host pattern" - )); - } - for pattern in includes.iter().chain(&excludes) { - if let Err(error) = - openshell_core::middleware::host_matches(pattern, "validation.invalid") - { - errors.push(format!( - "middleware config '{name}' has invalid endpoint selector pattern '{pattern}': {error}" - )); - } - } - } - - let Some(policies) = data - .get("network_policies") - .and_then(serde_json::Value::as_object) - else { - return errors; - }; - - for (policy_name, policy) in policies { - for endpoint in policy - .get("endpoints") - .and_then(serde_json::Value::as_array) - .map_or(&[][..], Vec::as_slice) - { - let tls_skip = endpoint - .get("tls") - .and_then(serde_json::Value::as_str) - .is_some_and(|tls| tls == "skip"); - if tls_skip && global_selector_matches_any_middleware(middlewares, endpoint) { - errors.push(format!( - "network policy '{policy_name}' tls: skip endpoint matches a global middleware selector" - )); - } - } - } - errors -} - -fn json_string_array(value: Option<&serde_json::Value>) -> Vec { - value - .and_then(serde_json::Value::as_array) - .map(|values| { - values - .iter() - .filter_map(serde_json::Value::as_str) - .map(ToString::to_string) - .collect() - }) - .unwrap_or_default() -} - -fn global_selector_matches_any_middleware( - middlewares: &[serde_json::Value], - endpoint: &serde_json::Value, -) -> bool { - let host = endpoint - .get("host") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(); - middlewares.iter().any(|mw| { - let Some(selector) = mw.get("endpoints") else { - return false; - }; - let includes = json_string_array(selector.get("include")); - let excludes = json_string_array(selector.get("exclude")); - !includes.is_empty() - && includes.iter().any(|pattern| { - openshell_core::middleware::host_matches(pattern, host).unwrap_or(false) - }) - && !excludes.iter().any(|pattern| { - openshell_core::middleware::host_matches(pattern, host).unwrap_or(false) - }) - }) -} - /// Resolve a policy binary path through the container's root filesystem. /// /// On Linux, `/proc//root/` provides access to the container's mount @@ -6791,6 +6648,124 @@ network_policies: ); } + #[test] + fn middleware_chain_uses_dns_label_glob_semantics() { + let data = r#" +network_middlewares: + - name: single-label + middleware: openshell/secrets + order: 10 + endpoints: + include: ["*.Example.COM"] + exclude: ["trusted.example.com"] + - name: recursive + middleware: openshell/secrets + order: 20 + endpoints: + include: ["**.example.com"] + - name: intra-label + middleware: openshell/secrets + order: 30 + endpoints: + include: ["*-api.example.com"] +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let names_for = |host: &str| { + let input = NetworkInput { + host: host.into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + engine + .query_middleware_chain_with_generation(&input) + .unwrap() + .0 + .into_iter() + .map(|entry| entry.name) + .collect::>() + }; + + assert_eq!( + names_for("api.example.com"), + vec!["single-label", "recursive"] + ); + assert_eq!(names_for("deep.api.example.com"), vec!["recursive"]); + assert_eq!(names_for("trusted.example.com"), vec!["recursive"]); + assert_eq!( + names_for("tenant-api.example.com"), + vec!["single-label", "recursive", "intra-label"] + ); + } + + #[test] + fn host_pattern_matches_rego_endpoint_host_semantics() { + // Middleware selectors and the tls-skip overlap validation promise the + // same host semantics as endpoint admission, which is decided by the + // endpoint_allowed branches in sandbox-policy.rego. Pin parity by + // running one table through openshell_core::host_pattern and through + // regorus with those branches verbatim. + let policy = r#" +package test + +default host_match = false + +host_match if { + not contains(input.pattern, "*") + lower(input.pattern) == lower(input.host) +} + +host_match if { + contains(input.pattern, "*") + glob.match(lower(input.pattern), ["."], lower(input.host)) +} +"#; + let mut engine = regorus::Engine::new(); + engine + .add_policy("test.rego".into(), policy.into()) + .unwrap(); + + let cases = [ + ("api.example.com", "api.example.com"), + ("api.example.com", "API.EXAMPLE.COM"), + ("api.example.com", "api.example.org"), + ("*.example.com", "api.example.com"), + ("*.example.com", "example.com"), + ("*.example.com", "deep.api.example.com"), + ("*-api.example.com", "tenant-api.example.com"), + ("*-api.example.com", "api.example.com"), + ("*.a?i.example.com", "x.abi.example.com"), + ("**.example.com", "example.com"), + ("**.example.com", "api.example.com"), + ("**.example.com", "deep.api.example.com"), + ("api.**.com", "api.com"), + ("api.**.com", "api.x.com"), + ("api.**.com", "api.x.y.com"), + ("api.**", "api"), + ("api.**", "api.com"), + ("*", "com"), + ("*", "example.com"), + ("**", "com"), + ("**", "deep.api.example.com"), + ]; + for (pattern, host) in cases { + let rust = openshell_core::host_pattern::host_matches(pattern, host).unwrap(); + engine + .set_input_json( + &serde_json::json!({ "pattern": pattern, "host": host }).to_string(), + ) + .unwrap(); + let rego = engine.eval_rule("data.test.host_match".into()).unwrap() + == regorus::Value::from(true); + assert_eq!( + rust, rego, + "host pattern parity mismatch: pattern={pattern} host={host} rust={rust} rego={rego}" + ); + } + } + #[test] fn middleware_policy_validation_rejects_bad_configs() { let cases = [ @@ -6839,7 +6814,7 @@ network_middlewares: - name: redactor middleware: openshell/secrets "#, - "requires an endpoint selector", + "endpoint selector is required", ), ( "malformed selector", @@ -6868,6 +6843,25 @@ network_policies: tls: skip binaries: - { path: /usr/bin/curl } +"#, + "tls: skip", + ), + ( + "tls skip wildcard overlap", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets + endpoints: + include: ["api.example.com"] +network_policies: + api: + endpoints: + - host: "*.example.com" + port: 443 + tls: skip + binaries: + - { path: /usr/bin/curl } "#, "tls: skip", ), diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 80b7b9d436..d89b4207c6 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -74,7 +74,7 @@ network_middlewares: Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. -`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints. +`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may reference the same binding and run as separate stages. Config names must be unique. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 266f5d8ca8..ea3cb68cf6 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -502,7 +502,7 @@ network_middlewares: | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | | `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists. Exclusions take precedence. | -Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. +Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints: `*` matches exactly one DNS label and `**` matches one or more labels, so `**.example.com` covers subdomains but not `example.com` itself. Brace alternates are rejected at validation. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, failure behavior, body limits, and operational guidance. From 10f979d9378cad8f945fca1c6f10598dbcc0afde Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 11:34:11 -0700 Subject: [PATCH 30/59] refactor(middleware): simplify supervisor middleware proto contract Address proto-level review feedback on the supervisor middleware contract while nothing is deployed against it: - Drop the api_version fields from MiddlewareManifest, ValidateConfigRequest, and HttpRequestEvaluation. The openshell.middleware.v1 package already versions the protocol, so the handshake check and per-request version stamping were redundant. - Rename middleware.proto to supervisor_middleware.proto to match the service it defines. - Renumber SupervisorMiddlewareService.max_body_bytes from 4 to 3 in sandbox.proto to close the field-number gap before a future field silently reuses it. - Clarify in the manifest comments that the described service is the operator-run gRPC server implementing SupervisorMiddleware. Signed-off-by: Piotr Mlocek --- .../src/lib.rs | 13 ------ .../src/service.rs | 3 +- proto/sandbox.proto | 2 +- ...ware.proto => supervisor_middleware.proto} | 41 +++++++++---------- 4 files changed, 21 insertions(+), 38 deletions(-) rename proto/{middleware.proto => supervisor_middleware.proto} (87%) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 5fc15a0ddf..c4a4927396 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -24,7 +24,6 @@ use openshell_core::proto::{ use tokio::sync::OnceCell; use tonic::Request; -pub const API_VERSION: &str = "openshell.middleware.v1"; const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation = SupervisorMiddlewareOperation::HttpRequest; const PRE_CREDENTIALS_PHASE: SupervisorMiddlewarePhase = SupervisorMiddlewarePhase::PreCredentials; @@ -272,13 +271,6 @@ fn validate_external_manifest( operator_max_body_bytes: usize, known_binding_ids: &mut HashSet, ) -> Result> { - if manifest.api_version != API_VERSION { - return Err(miette!( - "middleware registration '{}' reports unsupported API version '{}'", - registration.name, - manifest.api_version - )); - } if manifest.bindings.is_empty() { return Err(miette!( "middleware registration '{}' describes no bindings", @@ -590,7 +582,6 @@ impl ChainRunner { let response = state .service .validate_config(Request::new(ValidateConfigRequest { - api_version: API_VERSION.into(), binding_id: implementation.into(), config: Some(config), })) @@ -835,7 +826,6 @@ fn build_evaluation( body: &[u8], ) -> HttpRequestEvaluation { HttpRequestEvaluation { - api_version: API_VERSION.into(), binding_id: binding.id.clone(), phase: binding.phase, context: Some(RequestContext { @@ -1078,7 +1068,6 @@ mod tests { .await .expect("describe") .into_inner(); - assert_eq!(manifest.api_version, API_VERSION); assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); assert_eq!( manifest.bindings[0].operation, @@ -1133,7 +1122,6 @@ mod tests { _request: Request<()>, ) -> std::result::Result, tonic::Status> { Ok(tonic::Response::new(MiddlewareManifest { - api_version: API_VERSION.into(), name: "test/middleware".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { @@ -1332,7 +1320,6 @@ mod tests { fn external_manifest_rejects_operator_limit_above_capability() { let registration = external_registration(4097); let manifest = MiddlewareManifest { - api_version: API_VERSION.into(), name: "example/service".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { diff --git a/crates/openshell-supervisor-middleware/src/service.rs b/crates/openshell-supervisor-middleware/src/service.rs index 51df8d0700..1cb713f389 100644 --- a/crates/openshell-supervisor-middleware/src/service.rs +++ b/crates/openshell-supervisor-middleware/src/service.rs @@ -8,7 +8,7 @@ use openshell_core::proto::{ }; use tonic::{Request, Response, Status}; -use crate::{API_VERSION, builtins, safe_reason, validate_builtin_config}; +use crate::{builtins, safe_reason, validate_builtin_config}; #[derive(Debug, Default)] pub struct InProcessMiddlewareService; @@ -20,7 +20,6 @@ impl SupervisorMiddleware for InProcessMiddlewareService { _request: Request<()>, ) -> Result, Status> { Ok(Response::new(MiddlewareManifest { - api_version: API_VERSION.into(), name: "openshell/in-process".into(), service_version: env!("CARGO_PKG_VERSION").into(), bindings: builtins::describe(), diff --git a/proto/sandbox.proto b/proto/sandbox.proto index a8a27349c9..4e4d4e053b 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -371,5 +371,5 @@ message SupervisorMiddlewareService { // gRPC endpoint reachable from the sandbox supervisor. string grpc_endpoint = 2; // Operator-owned body limit applied to every binding exposed by the service. - uint64 max_body_bytes = 4; + uint64 max_body_bytes = 3; } diff --git a/proto/middleware.proto b/proto/supervisor_middleware.proto similarity index 87% rename from proto/middleware.proto rename to proto/supervisor_middleware.proto index 421acd3d5c..e5444f755b 100644 --- a/proto/middleware.proto +++ b/proto/supervisor_middleware.proto @@ -22,16 +22,17 @@ service SupervisorMiddleware { rpc EvaluateHttpRequest(HttpRequestEvaluation) returns (HttpRequestResult); } -// MiddlewareManifest describes one service and the bindings it exposes. +// MiddlewareManifest describes one middleware service and the bindings it +// exposes. The service is the operator-run gRPC server implementing +// SupervisorMiddleware. message MiddlewareManifest { - // Middleware protocol version implemented by the service. - string api_version = 1; - // Human-readable service name used for diagnostics. - string name = 2; - // Service-defined version string used for diagnostics. - string service_version = 3; - // Bindings exposed by this service. - repeated MiddlewareBinding bindings = 4; + // Human-readable middleware service name used for diagnostics. + string name = 1; + // Release version of the middleware service implementation, used for + // diagnostics. + string service_version = 2; + // Bindings exposed by this middleware service. + repeated MiddlewareBinding bindings = 3; } // MiddlewareBinding declares one operation and phase supported by a service. @@ -48,12 +49,10 @@ message MiddlewareBinding { // ValidateConfigRequest contains one policy configuration to validate. message ValidateConfigRequest { - // Middleware protocol version selected by OpenShell. - string api_version = 1; // Manifest binding id associated with this configuration. - string binding_id = 2; + string binding_id = 1; // Service-specific policy configuration. - google.protobuf.Struct config = 3; + google.protobuf.Struct config = 2; } // ValidateConfigResponse reports whether a policy configuration is accepted. @@ -66,22 +65,20 @@ message ValidateConfigResponse { // HttpRequestEvaluation contains one buffered HTTP request to evaluate. message HttpRequestEvaluation { - // Middleware protocol version selected by OpenShell. - string api_version = 1; // Manifest binding id selected for this evaluation. - string binding_id = 2; + string binding_id = 1; // Evaluation phase selected for this request. - SupervisorMiddlewarePhase phase = 3; + SupervisorMiddlewarePhase phase = 2; // Sandbox and request identity available to the supervisor. - RequestContext context = 4; + RequestContext context = 3; // Validated service-specific policy configuration. - google.protobuf.Struct config = 5; + google.protobuf.Struct config = 4; // Destination and HTTP request target. - HttpRequestTarget target = 6; + HttpRequestTarget target = 5; // HTTP request headers before OpenShell injects credentials. - map headers = 7; + map headers = 6; // Buffered request body. Empty for a bodyless request. - bytes body = 8; + bytes body = 7; } // Supervisor operation selected for middleware evaluation. From 104ab6c1b484c7509b8970d77fe659234a87698c Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 13:02:26 -0700 Subject: [PATCH 31/59] refactor(middleware): type the secrets config and name unsafe headers Address reviewer feedback on built-in config validation and header mutation diagnostics: - Replace field-fishing validation of the openshell/secrets config with a typed SecretsConfig (deny_unknown_fields, default redact). Unknown fields and wrong-typed values now fail policy validation instead of being silently accepted, and the redact default is encoded in the type rather than an unwrap_or buried in parsing code. - Stop discarding the header-mutation validation error: the on_error reason now names the offending header and the required x-openshell-middleware- prefix, matching how service errors are already surfaced. - Document the add_headers constraints (prefix, no rewrites, protected headers, value safety) in the proto contract and the extensibility docs, along with the secrets config default and strictness. Signed-off-by: Piotr Mlocek --- crates/openshell-core/src/middleware.rs | 89 ++++++++++++++----- .../src/lib.rs | 23 +++-- docs/extensibility/supervisor-middleware.mdx | 8 +- proto/supervisor_middleware.proto | 7 +- 4 files changed, 99 insertions(+), 28 deletions(-) diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs index 6fb15b1c15..07a3b08657 100644 --- a/crates/openshell-core/src/middleware.rs +++ b/crates/openshell-core/src/middleware.rs @@ -4,53 +4,100 @@ //! Shared supervisor middleware identifiers and policy validation contracts. use miette::{Result, miette}; +use serde::Deserialize; /// Binding identifier for the built-in secret redaction middleware. pub const BUILTIN_SECRETS: &str = "openshell/secrets"; +/// Policy-owned configuration for the built-in `openshell/secrets` middleware. +/// +/// Unknown fields and wrong-typed values are rejected so a config typo fails +/// policy validation instead of silently running with defaults. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SecretsConfig { + /// Redaction mode. Omitting the field selects [`SecretsMode::Redact`]. + pub secrets: SecretsMode, +} + +/// Supported `openshell/secrets` modes. Phase 1 supports only `redact`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SecretsMode { + #[default] + Redact, +} + +impl SecretsConfig { + /// Parse and validate a policy-owned protobuf config. + pub fn from_struct(config: &prost_types::Struct) -> Result { + serde_json::from_value(crate::proto_struct::struct_to_json_value(config)).map_err(|error| { + miette!( + "invalid {BUILTIN_SECRETS} config: {error}; phase 1 supports only secrets: redact" + ) + }) + } +} + /// Validate policy-owned configuration for a built-in middleware. pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { match implementation { - BUILTIN_SECRETS => validate_secrets_config(config), + BUILTIN_SECRETS => SecretsConfig::from_struct(config).map(|_| ()), other => Err(miette!( "middleware implementation '{other}' is not available in phase 1" )), } } -fn validate_secrets_config(config: &prost_types::Struct) -> Result<()> { - let mode = config - .fields - .get("secrets") - .and_then(|value| match value.kind.as_ref() { - Some(prost_types::value::Kind::StringValue(value)) => Some(value.as_str()), - _ => None, - }) - .unwrap_or("redact"); - if mode != "redact" { - return Err(miette!( - "{BUILTIN_SECRETS} only supports config.secrets: redact in phase 1" - )); - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; + fn string_config(key: &str, value: &str) -> prost_types::Struct { + prost_types::Struct { + fields: std::iter::once(( + key.to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue(value.into())), + }, + )) + .collect(), + } + } + #[test] fn secrets_config_defaults_to_redact() { - validate_builtin_config(BUILTIN_SECRETS, &prost_types::Struct::default()).unwrap(); + let config = SecretsConfig::from_struct(&prost_types::Struct::default()).unwrap(); + assert_eq!(config.secrets, SecretsMode::Redact); + } + + #[test] + fn secrets_config_accepts_explicit_redact() { + let config = SecretsConfig::from_struct(&string_config("secrets", "redact")).unwrap(); + assert_eq!(config.secrets, SecretsMode::Redact); } #[test] fn secrets_config_rejects_unsupported_mode() { + let error = validate_builtin_config(BUILTIN_SECRETS, &string_config("secrets", "allow")) + .unwrap_err(); + assert!(error.to_string().contains("supports only secrets: redact")); + } + + #[test] + fn secrets_config_rejects_unknown_field() { + let error = validate_builtin_config(BUILTIN_SECRETS, &string_config("secret", "redact")) + .unwrap_err(); + assert!(error.to_string().contains("unknown field")); + } + + #[test] + fn secrets_config_rejects_non_string_mode() { let config = prost_types::Struct { fields: std::iter::once(( "secrets".to_string(), prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("allow".into())), + kind: Some(prost_types::value::Kind::NumberValue(42.0)), }, )) .collect(), @@ -60,7 +107,7 @@ mod tests { assert!( error .to_string() - .contains("only supports config.secrets: redact") + .contains("invalid openshell/secrets config") ); } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index c4a4927396..33443e9ed1 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -751,9 +751,11 @@ impl ChainRunner { } // A result proposing unsafe header mutations is a malformed response: - // route it through `on_error` instead of applying any of it. - if validate_header_mutations(&headers, &result.add_headers).is_err() { - match apply_on_error(entry, "unsafe_response_headers", &mut applied) { + // route it through `on_error` instead of applying any of it. The + // validation error names the offending header and the required + // x-openshell-middleware- prefix so operators can fix the service. + if let Err(error) = validate_header_mutations(&headers, &result.add_headers) { + match apply_on_error(entry, &safe_reason(&error.to_string()), &mut applied) { OnErrorAction::FailOpen => continue, OnErrorAction::FailClosed(reason) => { return Ok(ChainOutcome { @@ -860,7 +862,10 @@ fn validate_header_mutations( )); } if !is_safe_append_header(&lower) { - return Err(miette!("middleware cannot append unsafe header '{name}'")); + return Err(miette!( + "middleware can only append new request headers prefixed with \ + x-openshell-middleware- and cannot append '{name}'" + )); } // Reject CR/LF and other control characters in the value: writing them // verbatim into the upstream header block would enable header injection @@ -1087,7 +1092,8 @@ mod tests { &std::iter::once(("Authorization".into(), "Bearer nope".into())).collect(), ) .expect_err("unsafe header"); - assert!(err.to_string().contains("unsafe header")); + assert!(err.to_string().contains("x-openshell-middleware-")); + assert!(err.to_string().contains("'Authorization'")); } #[test] @@ -1567,6 +1573,13 @@ mod tests { .expect("evaluate"); assert!(!outcome.allowed); assert!(outcome.reason.starts_with("middleware_failed:")); + // The deny reason names the offending header so operators can fix the + // service without reading supervisor source. + assert!( + outcome.reason.contains("x-openshell-middleware-inject"), + "reason should name the offending header: {}", + outcome.reason + ); assert!(outcome.applied.iter().any(|inv| inv.failed)); // The unsafe header is never forwarded. assert!(outcome.added_headers.is_empty()); diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index d89b4207c6..7705de9098 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -30,7 +30,7 @@ Middleware receives the request before credential injection. Operator-run servic | Built-in | None | Defined by OpenShell | Runs inside the supervisor | | Operator-run service | Required in gateway TOML | Set by the operator, up to the service capability | Runs as a separate service reachable by the gateway and supervisors | -`openshell/secrets` is the built-in middleware currently available. It identifies common secret patterns in UTF-8 request bodies and replaces matched values before the request leaves the sandbox. +`openshell/secrets` is the built-in middleware currently available. It identifies common secret patterns in UTF-8 request bodies and replaces matched values before the request leaves the sandbox. Its `config` accepts one field, `secrets: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Operator-run services expose one or more binding IDs. Policies reference a binding ID, such as `example/content-guard`, rather than the gateway registration name. @@ -104,6 +104,12 @@ Every middleware binding declares the largest request or replacement body it sup The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure and follows that config's `on_error` behavior. +## Add Request Headers + +A middleware result can add new request headers before OpenShell injects credentials. Header names are case-insensitive and must start with `x-openshell-middleware-`. Values must not contain control characters. + +A result that rewrites an existing request header, uses a name outside the required prefix, or carries an unsafe value is treated as a middleware failure and follows that config's `on_error` behavior. The failure reason names the offending header. Protected headers, including `authorization`, `cookie`, `host`, `content-length`, `transfer-encoding`, and OpenShell credential and signature headers, are always rejected. + ## Operate Middleware Services Plan startup and updates around these boundaries: diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index e5444f755b..f54927e8a9 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -165,7 +165,12 @@ message HttpRequestResult { bytes body = 3; // True when body should replace the request body, including with an empty body. bool has_body = 4; - // Request headers to add before forwarding. Protected headers are rejected. + // Request headers to add before forwarding. Only new headers whose + // case-insensitive names start with "x-openshell-middleware-" are accepted. + // Existing request headers cannot be rewritten, protected headers such as + // authorization or transfer-encoding are always rejected, and values must + // not contain control characters. A violating result is a middleware + // failure handled according to the policy failure mode. map add_headers = 5; // Audit-safe findings produced during evaluation. repeated Finding findings = 6; From 2f23c38b42cb316493e254491cc4cb041cf40247 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 15:10:43 -0700 Subject: [PATCH 32/59] fix(middleware): buffer request bodies at the chain's largest stage limit Buffering at the smallest stage limit forced the whole chain onto the unbuffered over-capacity path whenever the body exceeded any single stage, losing per-stage on_error behavior: a small fail-open stage could deny a request that a larger fail-closed stage was able to inspect. Buffer for the most capable stage instead, so stages whose own limit is smaller fail individually with request_body_over_capacity through their own on_error while the rest of the chain still runs. A body over every stage limit keeps the existing aggregate handling, which now matches the per-stage semantics by construction. Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 + .../src/lib.rs | 83 +++++++++++ .../openshell-supervisor-network/Cargo.toml | 1 + .../src/l7/middleware.rs | 24 +-- .../src/l7/relay.rs | 140 ++++++++++++++++++ docs/extensibility/supervisor-middleware.mdx | 6 +- 6 files changed, 242 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46a081025e..c0882410bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3983,6 +3983,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-tungstenite 0.26.2", + "tonic", "tower-mcp-types", "tracing", "tracing-subscriber", diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 33443e9ed1..38b90d9516 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -1322,6 +1322,89 @@ mod tests { ); } + #[tokio::test] + async fn undersized_stage_fails_open_while_later_stage_runs() { + // A body over one stage's limit must fail only that stage through its + // own `on_error`, not the whole chain: the 1 KiB fail-open guard is + // skipped while the 256 KiB fail-closed redactor still runs. + let external = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: allow_result(), + }); + let registry = registry_with_external(external, external_registration(1024)).await; + let runner = ChainRunner::from_registry(registry); + let guard_entry = ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }; + let mut redact_entry = entry("redact", OnError::FailClosed); + redact_entry.order = 10; + let entries = [guard_entry, redact_entry]; + + let body = format!("{}password=\"top-secret\"", "x".repeat(1500)); + let outcome = runner + .evaluate(&entries, input(&body)) + .await + .expect("evaluate mixed-limit chain"); + + assert!(outcome.allowed); + assert_eq!(outcome.applied.len(), 2); + assert!( + outcome.applied[0].failed, + "undersized guard must be skipped" + ); + assert_eq!(outcome.applied[0].decision, Decision::Allow); + assert!(!outcome.applied[1].failed); + assert!(outcome.applied[1].transformed); + let body = String::from_utf8(outcome.body).expect("utf8"); + assert!(body.contains("[REDACTED]")); + assert!(!body.contains("top-secret")); + } + + #[tokio::test] + async fn transformed_body_still_over_later_stage_capacity_honors_on_error() { + // Per-stage capacity applies to the current body: the redactor's + // replacement is still over the 1 KiB guard limit, so the fail-closed + // guard denies through its own `on_error` after the redactor ran. + let external = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: allow_result(), + }); + let registry = registry_with_external(external, external_registration(1024)).await; + let runner = ChainRunner::from_registry(registry); + let guard_entry = ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let entries = [entry("redact", OnError::FailClosed), guard_entry]; + + let body = format!("{}password=\"top-secret\"", "x".repeat(1500)); + let outcome = runner + .evaluate(&entries, input(&body)) + .await + .expect("evaluate mixed-limit chain"); + + assert!(!outcome.allowed); + assert_eq!( + outcome.reason, + "middleware_failed: request_body_over_capacity" + ); + assert_eq!(outcome.applied.len(), 2); + assert!( + outcome.applied[0].transformed, + "redactor ran before the deny" + ); + assert!(outcome.applied[1].failed); + } + #[test] fn external_manifest_rejects_operator_limit_above_capability() { let registration = external_registration(4097); diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index b8cae51133..46430af58a 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -52,6 +52,7 @@ webpki-roots = { workspace = true } [dev-dependencies] openshell-core = { path = "../openshell-core", features = ["test-helpers"] } tempfile = "3" +tonic = { workspace = true } temp-env = "0.3" tokio-tungstenite = { workspace = true } futures = { workspace = true } diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 048baa3a29..6c9d80037d 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -19,11 +19,14 @@ pub enum MiddlewareApplyResult { Denied(String), } -/// Smallest body-buffering limit across the entries that actually resolved to a -/// registered binding. Unresolved entries (`is_resolved() == false`) report a -/// zero limit and are excluded here: they are handled by their `on_error` policy -/// in `evaluate_described` without inspecting the body, so letting a zero drag -/// the chain limit to zero would spuriously fail the whole chain over capacity. +/// Largest body-buffering limit across the entries that actually resolved to a +/// registered binding. Buffering for the most capable stage lets every stage +/// that can handle the body run; stages whose own limit is smaller are failed +/// individually with `request_body_over_capacity` through their `on_error` +/// policy in `evaluate_described`, instead of one undersized stage forcing the +/// whole chain onto the unbuffered path. Unresolved entries +/// (`is_resolved() == false`) report a zero limit and are excluded here: they +/// are handled by their `on_error` policy without inspecting the body. /// Returns `None` when no entry resolved, so the caller can skip buffering. pub(super) fn middleware_chain_body_limit( chain: &[openshell_supervisor_middleware::DescribedChainEntry], @@ -32,7 +35,7 @@ pub(super) fn middleware_chain_body_limit( .iter() .filter(|entry| entry.is_resolved()) .map(openshell_supervisor_middleware::DescribedChainEntry::max_body_bytes) - .min() + .max() } pub async fn apply_middleware_chain( @@ -147,10 +150,11 @@ pub(super) fn raw_query_from_request_headers(headers: &[u8]) -> Result { .map_or_else(String::new, |(_, query)| query.to_string())) } -/// Apply the chain's `on_error` policy when the request body cannot be buffered -/// for inspection because it exceeds the size cap. The RFC treats an unbufferable -/// body as an `on_error` event: it is denied unless every attached middleware is -/// `fail_open`, and passing it through is only safe when no bytes were consumed. +/// Apply the chain's `on_error` policy when the request body exceeds every +/// stage's buffering limit. No stage can inspect such a body, so each stage +/// would individually fail with `request_body_over_capacity`; the aggregate is +/// a deny unless every attached middleware is `fail_open`, and passing the +/// body through is only safe when no bytes were consumed. pub(super) fn resolve_unbuffered_body( ctx: &L7EvalContext, req: crate::l7::provider::L7Request, diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 96e05f2722..2efeb2a2f2 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2875,6 +2875,146 @@ network_policies: assert_eq!(middleware_chain_body_limit(&none), None); } + /// A middleware service advertising two bindings with different body + /// limits, for exercising mixed-limit chain buffering at the relay level. + /// The redactor binding replaces the body; the guard binding allows as-is. + struct TwoLimitService; + + #[tonic::async_trait] + impl openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware + for TwoLimitService + { + async fn describe( + &self, + _request: tonic::Request<()>, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + use openshell_core::proto::{ + MiddlewareBinding, MiddlewareManifest, SupervisorMiddlewareOperation, + SupervisorMiddlewarePhase, + }; + let binding = |id: &str, max_body_bytes: u64| MiddlewareBinding { + id: id.into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes, + }; + Ok(tonic::Response::new(MiddlewareManifest { + name: "test/two-limits".into(), + service_version: "test".into(), + bindings: vec![binding("test/redactor", 8192), binding("test/guard", 16)], + })) + } + + async fn validate_config( + &self, + _request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + let evaluation = request.into_inner(); + let mut result = openshell_core::proto::HttpRequestResult { + decision: openshell_core::proto::Decision::Allow as i32, + ..Default::default() + }; + if evaluation.binding_id == "test/redactor" { + result.body = b"[SCRUBBED BY TEST REDACTOR]".to_vec(); + result.has_body = true; + } + Ok(tonic::Response::new(result)) + } + } + + #[tokio::test] + async fn body_over_smallest_stage_limit_is_buffered_and_evaluated() { + use openshell_supervisor_middleware::{ChainEntry, ChainRunner, OnError}; + + // A 64-byte body exceeds the 16-byte guard limit but fits the 8 KiB + // redactor. The chain must buffer for its largest stage so the + // redactor runs and replaces the body, while the undersized fail-open + // guard is skipped through its own on_error, instead of the whole + // chain taking the unbuffered over-capacity path. + let (_config, tunnel_engine, ctx) = + middleware_relay_context("openshell/secrets", "fail_closed"); + let runner = ChainRunner::new(Arc::new(TwoLimitService)); + let chain = vec![ + ChainEntry { + name: "redact".into(), + implementation: "test/redactor".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ChainEntry { + name: "guard".into(), + implementation: "test/guard".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailOpen, + }, + ]; + let described = runner.describe_chain(&chain).await.expect("describe chain"); + assert_eq!(middleware_chain_body_limit(&described), Some(8192)); + + let body = [b'a'; 64]; + let raw_header = format!( + "POST /v1/messages HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let req = crate::l7::provider::L7Request { + action: "POST".into(), + target: "/v1/messages".into(), + query_params: std::collections::HashMap::new(), + raw_header: raw_header.into_bytes(), + body_length: crate::l7::provider::BodyLength::ContentLength(body.len() as u64), + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + app.write_all(&body).await.unwrap(); + + let result = crate::l7::middleware::apply_middleware_chain_for_scheme( + req, + &mut relay_client, + &ctx, + "https", + chain, + &runner, + tunnel_engine.generation_guard(), + ) + .await + .expect("apply middleware chain"); + + match result { + MiddlewareApplyResult::Allowed(rebuilt) => { + let raw = String::from_utf8(rebuilt.raw_header).expect("utf8 request"); + assert!( + raw.ends_with("[SCRUBBED BY TEST REDACTOR]"), + "redactor must replace the body: {raw}" + ); + } + MiddlewareApplyResult::Denied(reason) => { + panic!("body within the largest stage limit must not fail the chain: {reason}") + } + } + } + #[tokio::test] async fn all_unresolved_fail_open_forwards_body_unbuffered() { // A chain whose only entry is an unregistered binding has no resolvable diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 7705de9098..75515abfb6 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -17,7 +17,7 @@ For each inspected HTTP request, the supervisor: 1. Evaluates network and L7 policy. 2. Selects middleware whose host selectors match the admitted destination. -3. Buffers the request body using the smallest body limit in the selected chain. +3. Buffers the request body using the largest body limit in the selected chain. 4. Runs matching middleware by ascending `order`, using the policy-local name to break ties. 5. Applies allowed transformations, injects provider credentials, and forwards the request. @@ -99,10 +99,10 @@ Every middleware binding declares the largest request or replacement body it sup - Built-in middleware uses its OpenShell-defined limit. - Each operator-run registration sets `max_body_bytes` no higher than the service capability. -- A selected chain buffers using its smallest stage limit. +- A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. - The same per-stage limit applies to request bodies and replacement bodies. -The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure and follows that config's `on_error` behavior. +The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. A body larger than every stage limit cannot be inspected at all and is denied unless every selected stage is `fail_open`. ## Add Request Headers From eb7e2ca7728705ce5fd376020a9d0e38606f4fb0 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 15:32:14 -0700 Subject: [PATCH 33/59] test(middleware): pin audit endpoints still enforcing middleware denies enforcement: audit applies to an endpoint's network and L7 policy rules; middleware is an independently configured control with its own failure semantics, and a service that needs to observe without blocking returns an allow decision with findings. Document that middleware denies and fail_closed failures block even on audit endpoints, and cover both directions with relay tests: an audited policy-denied request forwards through a healthy chain, and a fail-closed middleware failure still returns 403 under audit. Signed-off-by: Piotr Mlocek --- .../src/l7/relay.rs | 125 +++++++++++++++++- docs/extensibility/supervisor-middleware.mdx | 2 + 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 2efeb2a2f2..a499baede9 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2097,6 +2097,14 @@ network_policies: fn middleware_relay_context( middleware_impl: &str, on_error: &str, + ) -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + middleware_relay_context_with_enforcement(middleware_impl, on_error, "enforce") + } + + fn middleware_relay_context_with_enforcement( + middleware_impl: &str, + on_error: &str, + enforcement: &str, ) -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { let data = format!( r#" @@ -2113,7 +2121,7 @@ network_policies: - host: api.example.test port: 8080 protocol: rest - enforcement: enforce + enforcement: {enforcement} rules: - allow: method: POST @@ -2623,6 +2631,121 @@ network_policies: .unwrap(); } + #[tokio::test] + async fn audit_endpoint_forwards_policy_denied_request_through_healthy_chain() { + // Baseline for audit semantics: a request the L7 policy denies is + // still forwarded on an `enforcement: audit` endpoint when the + // middleware chain is healthy and allows it. + let (config, tunnel_engine, ctx) = + middleware_relay_context_with_enforcement("openshell/secrets", "fail_closed", "audit"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all( + b"GET /other HTTP/1.1\r\nHost: api.example.test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + + let mut upstream_request = [0u8; 512]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_request), + ) + .await + .expect("audited request should reach upstream") + .unwrap(); + assert!(String::from_utf8_lossy(&upstream_request[..n]).starts_with("GET /other")); + + upstream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut client_response = [0u8; 512]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + app.read(&mut client_response), + ) + .await + .expect("response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("204 No Content")); + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn audit_endpoint_still_enforces_middleware_deny() { + // `enforcement: audit` applies to the endpoint's L7 policy rules, not + // to middleware: a middleware deny (here a fail-closed failure) must + // block with 403 even though the same request would be forwarded + // under audit with a healthy chain. + let (config, tunnel_engine, ctx) = middleware_relay_context_with_enforcement( + "example/unavailable", + "fail_closed", + "audit", + ); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all( + b"GET /other HTTP/1.1\r\nHost: api.example.test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden")); + assert!(response.contains("middleware_failed")); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive request bytes" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[tokio::test] async fn jsonrpc_middleware_fail_closed_does_not_reach_upstream() { let data = r#" diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 75515abfb6..fac13c146b 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -93,6 +93,8 @@ Use `fail_open` only when bypassing the middleware preserves the intended securi An explicit deny decision always stops the chain and denies the request, regardless of `on_error`. +Middleware decisions are enforced regardless of the endpoint's `enforcement` mode. `enforcement: audit` applies to an endpoint's network and L7 policy rules and does not bypass middleware: a middleware deny, or a failed `fail_closed` stage, blocks the request even on an audit endpoint. A middleware service that needs to observe traffic without blocking should return an allow decision with findings, which OpenShell emits as detection findings. + ## Set Body Limits Every middleware binding declares the largest request or replacement body it supports. From eac2cd3e12ab8176c6dda0380d58f826d67f16c1 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 15:55:14 -0700 Subject: [PATCH 34/59] fix(middleware): preserve repeated request headers in the contract A map cannot represent repeated HTTP headers: the parser kept only the last value for middleware while request rebuilding forwarded every original value upstream, so a duplicated header could smuggle a first-positioned value past inspection. HttpRequestEvaluation now carries repeated HttpHeader entries in wire order, the parser and HttpRequestInput preserve duplicates, and append validation rejects a mutation matching any occurrence of an existing name. A recording service test pins that middleware receives every value the upstream will see. Signed-off-by: Piotr Mlocek --- .../src/lib.rs | 149 ++++++++++++++++-- .../src/l7/middleware.rs | 52 ++++-- .../src/l7/relay.rs | 10 +- docs/extensibility/supervisor-middleware.mdx | 2 +- proto/supervisor_middleware.proto | 15 +- 5 files changed, 189 insertions(+), 39 deletions(-) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 38b90d9516..21107f892c 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -16,7 +16,7 @@ pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - Decision, Finding, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, + Decision, Finding, HttpHeader, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, SupervisorMiddlewareService, ValidateConfigRequest, @@ -116,7 +116,10 @@ pub struct HttpRequestInput { pub method: String, pub path: String, pub query: String, - pub headers: BTreeMap, + /// Lowercased request headers in wire order. Repeated header names are + /// preserved as separate entries so middleware inspects every value the + /// upstream will receive. + pub headers: Vec<(String, String)>, pub body: Vec, } @@ -771,7 +774,7 @@ impl ChainRunner { } } for (name, value) in &result.add_headers { - headers.insert(name.to_ascii_lowercase(), value.clone()); + headers.push((name.to_ascii_lowercase(), value.clone())); added_headers.insert(name.to_ascii_lowercase(), value.clone()); } let transformed = result.has_body; @@ -824,7 +827,7 @@ fn build_evaluation( entry: &DescribedChainEntry, binding: &MiddlewareBinding, input: &HttpRequestInput, - headers: &BTreeMap, + headers: &[(String, String)], body: &[u8], ) -> HttpRequestEvaluation { HttpRequestEvaluation { @@ -844,19 +847,25 @@ fn build_evaluation( path: input.path.clone(), query: input.query.clone(), }), - headers: headers.clone().into_iter().collect(), + headers: headers + .iter() + .map(|(name, value)| HttpHeader { + name: name.clone(), + value: value.clone(), + }) + .collect(), body: body.to_vec(), } } fn validate_header_mutations( - existing_headers: &BTreeMap, + existing_headers: &[(String, String)], mutations: &HashMap, ) -> Result<()> { let mut seen = HashSet::new(); for (name, value) in mutations { let lower = name.to_ascii_lowercase(); - if !seen.insert(lower.clone()) || existing_headers.contains_key(&lower) { + if !seen.insert(lower.clone()) || existing_headers.iter().any(|(name, _)| *name == lower) { return Err(miette!( "middleware cannot rewrite existing header '{name}'" )); @@ -948,7 +957,7 @@ mod tests { method: "POST".into(), path: "/v1".into(), query: String::new(), - headers: BTreeMap::new(), + headers: Vec::new(), body: body.as_bytes().to_vec(), } } @@ -962,7 +971,7 @@ mod tests { let entry = &entries[0]; let binding = entry.binding.as_ref().expect("described binding"); let input = input("payload"); - let evaluation = build_evaluation(entry, binding, &input, &BTreeMap::new(), b"payload"); + let evaluation = build_evaluation(entry, binding, &input, &[], b"payload"); assert_eq!( evaluation.phase, @@ -1088,7 +1097,7 @@ mod tests { #[test] fn unsafe_header_mutation_is_rejected() { let err = validate_header_mutations( - &BTreeMap::new(), + &[], &std::iter::once(("Authorization".into(), "Bearer nope".into())).collect(), ) .expect_err("unsafe header"); @@ -1101,7 +1110,7 @@ mod tests { // A safe header *name* with a CRLF-bearing value must still be rejected, // otherwise it would inject extra headers into the upstream request. let err = validate_header_mutations( - &BTreeMap::new(), + &[], &std::iter::once(( "x-openshell-middleware-inject".into(), "ok\r\nAuthorization: Bearer evil".into(), @@ -1185,6 +1194,124 @@ mod tests { } } + /// A middleware that records every evaluation it receives and allows the + /// request, for asserting what the supervisor actually sends to services. + struct RecordingService { + received: std::sync::Mutex>, + } + + #[tonic::async_trait] + impl SupervisorMiddleware for RecordingService { + async fn describe( + &self, + _request: Request<()>, + ) -> std::result::Result, tonic::Status> { + Ok(tonic::Response::new(MiddlewareManifest { + name: "test/recorder".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/recorder".into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: 4096, + }], + })) + } + + async fn validate_config( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.received + .lock() + .expect("recording lock") + .push(request.into_inner()); + Ok(tonic::Response::new(allow_result())) + } + } + + #[tokio::test] + async fn repeated_request_headers_reach_middleware_in_wire_order() { + // A map contract would collapse repeated header names to one value + // while the upstream still receives every original value, creating an + // inspection differential. The service must see each entry in wire + // order. + let service = Arc::new(RecordingService { + received: std::sync::Mutex::new(Vec::new()), + }); + let recorder: Arc = service.clone(); + let runner = ChainRunner::new(recorder); + let recorder_entry = ChainEntry { + name: "recorder".into(), + implementation: "example/recorder".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let mut request = input("payload"); + request.headers = vec![ + ("x-api-key".into(), "first-value".into()), + ("accept".into(), "application/json".into()), + ("x-api-key".into(), "second-value".into()), + ]; + + let outcome = runner + .evaluate(&[recorder_entry], request) + .await + .expect("evaluate recording chain"); + assert!(outcome.allowed); + + let received = service.received.lock().expect("recorded evaluations"); + assert_eq!(received.len(), 1); + let headers: Vec<(&str, &str)> = received[0] + .headers + .iter() + .map(|header| (header.name.as_str(), header.value.as_str())) + .collect(); + assert_eq!( + headers, + vec![ + ("x-api-key", "first-value"), + ("accept", "application/json"), + ("x-api-key", "second-value"), + ] + ); + } + + #[test] + fn existing_header_rewrite_is_rejected() { + // Any occurrence of an existing name blocks the mutation, including + // repeated names where only one entry matches. + let existing = [ + ("x-openshell-middleware-tag".to_string(), "one".to_string()), + ("accept".to_string(), "application/json".to_string()), + ]; + let err = validate_header_mutations( + &existing, + &std::iter::once(("X-OpenShell-Middleware-Tag".into(), "two".into())).collect(), + ) + .expect_err("rewrite of existing header"); + assert!(err.to_string().contains("cannot rewrite existing header")); + } + fn external_registration(max_body_bytes: u64) -> SupervisorMiddlewareService { SupervisorMiddlewareService { name: "local-guard-service".into(), diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 6c9d80037d..d88e04c49c 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -10,7 +10,6 @@ use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; -use std::collections::BTreeMap; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncWrite}; @@ -68,14 +67,8 @@ pub async fn apply_middleware_chain_for_scheme, + headers: Vec<(String, String)>, query: String, body: Vec, ) -> openshell_supervisor_middleware::HttpRequestInput { @@ -197,10 +190,13 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { ocsf_emit!(event); } -fn safe_middleware_headers(headers: &[u8]) -> Result> { +/// Parse the raw header block into middleware-visible headers, preserving +/// wire order and repeated names so middleware inspects every value the +/// upstream will receive. Credential-bearing headers are omitted. +fn safe_middleware_headers(headers: &[u8]) -> Result> { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - let mut out = BTreeMap::new(); + let mut out = Vec::new(); for line in header_str.lines().skip(1) { let Some((name, value)) = line.split_once(':') else { continue; @@ -221,7 +217,7 @@ fn safe_middleware_headers(headers: &[u8]) -> Result> { { continue; } - out.insert(name, value.trim().to_string()); + out.push((name, value.trim().to_string())); } Ok(out) } @@ -376,10 +372,32 @@ mod tests { .expect("headers should parse"); assert_eq!( - headers.get("x-request-id").map(String::as_str), - Some("request-123") + headers, + vec![("x-request-id".to_string(), "request-123".to_string())] + ); + } + + #[test] + fn middleware_headers_preserve_repeated_names_in_wire_order() { + // Repeated header names must reach middleware as separate entries in + // wire order: keeping only one value would let a request smuggle a + // differently-positioned duplicate past inspection while the upstream + // still receives every original value. + let headers = safe_middleware_headers( + b"POST /v1 HTTP/1.1\r\n\ + X-Api-Key: first-value\r\n\ + Accept: application/json\r\n\ + X-Api-Key: second-value\r\n\r\n", + ) + .expect("headers should parse"); + + assert_eq!( + headers, + vec![ + ("x-api-key".to_string(), "first-value".to_string()), + ("accept".to_string(), "application/json".to_string()), + ("x-api-key".to_string(), "second-value".to_string()), + ] ); - assert!(!headers.contains_key("authorization")); - assert!(!headers.contains_key("proxy-authorization")); } } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index a499baede9..0fce744f4a 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -3231,14 +3231,8 @@ network_policies: token_grant_resolver: None, }; - let input = middleware_request_input( - "http", - &req, - &ctx, - BTreeMap::new(), - String::new(), - Vec::new(), - ); + let input = + middleware_request_input("http", &req, &ctx, Vec::new(), String::new(), Vec::new()); assert_eq!(input.scheme, "http"); } diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index fac13c146b..667a14610b 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -21,7 +21,7 @@ For each inspected HTTP request, the supervisor: 4. Runs matching middleware by ascending `order`, using the policy-local name to break ties. 5. Applies allowed transformations, injects provider credentials, and forwards the request. -Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. +Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Request headers are delivered in wire order and repeated header names are preserved as separate entries, so a service inspects every value the upstream will receive. ## Choose a Middleware Type diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index f54927e8a9..fdc7f03c4b 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -75,12 +75,23 @@ message HttpRequestEvaluation { google.protobuf.Struct config = 4; // Destination and HTTP request target. HttpRequestTarget target = 5; - // HTTP request headers before OpenShell injects credentials. - map headers = 6; + // HTTP request headers before OpenShell injects credentials, in wire + // order. Repeated header names are preserved as separate entries so a + // service inspects every value the upstream will receive. Protected + // headers such as authorization or cookie are omitted. + repeated HttpHeader headers = 6; // Buffered request body. Empty for a bodyless request. bytes body = 7; } +// HttpHeader is one request header line. +message HttpHeader { + // Lowercased header name. + string name = 1; + // Header value with surrounding whitespace trimmed. + string value = 2; +} + // Supervisor operation selected for middleware evaluation. enum SupervisorMiddlewareOperation { SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED = 0; From 11dbb44e42ed05a893fb3be98898684342c2a54b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 16:38:02 -0700 Subject: [PATCH 35/59] fix(middleware): re-evaluate transformed bodies and gate uninspectable traffic Policy ran before the middleware chain, so an allowed JSON-RPC, MCP, or GraphQL request whose body a middleware replaced was forwarded without re-checking it, letting a transformation smuggle a locally denied operation upstream - and a later stage could receive a payload an earlier stage had already transformed out of policy. The chain runner now takes a TransformedBodyValidator and re-checks the body after every stage that replaces it, stopping the chain on a policy deny before the next stage runs. Every middleware hook therefore receives a payload the sandbox policy admits, and nothing non-compliant reaches the upstream. The relay and forward proxy build the validator from the shared reevaluate_transformed_body helper, which is exhaustive over L7Protocol so a new protocol must define its transformed-body semantics before it compiles. An unparseable replacement denies even under audit, mirroring force_deny for original parse errors; a policy deny respects the endpoint's enforcement mode. Inspection was also optional for L4-only endpoints: h2c prefaces, unknown TCP protocols, TLS without termination state, tls: skip tunnels, and the unimplemented SQL relay all raw-copied without running a matching chain. These paths now share one gate: traffic no middleware can inspect is denied when any matching entry is fail_closed and relayed with a bypass detection finding when every entry is fail_open. A future per-config on_uninspectable knob could let operators opt uninspectable protocols through explicitly. Signed-off-by: Piotr Mlocek --- .../src/lib.rs | 197 +++++ .../src/l7/middleware.rs | 88 ++- .../src/l7/relay.rs | 696 +++++++++++++++++- .../openshell-supervisor-network/src/opa.rs | 11 + .../openshell-supervisor-network/src/proxy.rs | 181 ++++- docs/extensibility/supervisor-middleware.mdx | 7 +- 6 files changed, 1155 insertions(+), 25 deletions(-) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 21107f892c..fc17857690 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -106,6 +106,14 @@ impl DescribedChainEntry { } } +/// Re-checks a middleware-transformed request body against sandbox policy. +/// +/// Returns `Some(reason)` to deny the chain, `None` to proceed. Invoked after +/// each stage that replaces the body so neither a later stage nor the upstream +/// sees a payload the policy would reject. Protocols with no body-aware policy +/// pass `None` for the validator instead. +pub type TransformedBodyValidator<'a> = dyn Fn(&[u8]) -> Result> + Send + Sync + 'a; + #[derive(Debug, Clone)] pub struct HttpRequestInput { pub request_id: String, @@ -616,6 +624,23 @@ impl ChainRunner { &self, entries: &[DescribedChainEntry], input: HttpRequestInput, + ) -> Result { + self.evaluate_described_with_validator(entries, input, None) + .await + } + + /// Evaluate a described chain, re-checking the request body against sandbox + /// policy after every stage that replaces it. Policy runs on the original + /// body before the chain, so without this a stage could hand the next stage + /// (or the upstream) a payload the policy rejects. When `validator` returns + /// a deny reason the chain stops with that reason, so no later stage ever + /// sees a non-compliant body. `None` skips re-checks for protocols with no + /// body-aware policy. + pub async fn evaluate_described_with_validator( + &self, + entries: &[DescribedChainEntry], + input: HttpRequestInput, + validator: Option<&TransformedBodyValidator<'_>>, ) -> Result { let mut headers = input.headers.clone(); let mut body = input.body.clone(); @@ -800,6 +825,25 @@ impl ChainRunner { transformed, failed: false, }); + + // The stage ran successfully but its output must still satisfy the + // sandbox policy the original body was admitted under. Re-check now, + // before the next stage or the upstream sees the replaced body. A + // policy deny here is a hard deny, independent of `on_error`. + if transformed + && let Some(validate) = validator + && let Some(reason) = validate(&body)? + { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } } Ok(ChainOutcome { @@ -1174,6 +1218,159 @@ mod tests { } } + /// A two-binding service for exercising per-stage validation: `test/transform` + /// replaces the body, `test/second` records that it ran and allows. + struct TwoStageService { + second_ran: Arc, + } + + #[tonic::async_trait] + impl SupervisorMiddleware for TwoStageService { + async fn describe( + &self, + _request: Request<()>, + ) -> std::result::Result, tonic::Status> { + let binding = |id: &str| MiddlewareBinding { + id: id.into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: 256 * 1024, + }; + Ok(tonic::Response::new(MiddlewareManifest { + name: "test/two-stage".into(), + service_version: "test".into(), + bindings: vec![binding("test/transform"), binding("test/second")], + })) + } + + async fn validate_config( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + let evaluation = request.into_inner(); + let mut result = allow_result(); + if evaluation.binding_id == "test/transform" { + result.body = b"TRANSFORMED".to_vec(); + result.has_body = true; + } else { + self.second_ran + .store(true, std::sync::atomic::Ordering::SeqCst); + } + Ok(tonic::Response::new(result)) + } + } + + #[tokio::test] + async fn per_stage_validation_denies_before_the_next_stage_runs() { + // The validator rejects the first stage's transformed body. The chain + // must stop there: the second stage never runs, so it never sees a + // payload the policy would reject. + let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let service: Arc = Arc::new(TwoStageService { + second_ran: Arc::clone(&second_ran), + }); + let runner = ChainRunner::new(service); + let transform = ChainEntry { + name: "transform".into(), + implementation: "test/transform".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let second = ChainEntry { + name: "second".into(), + implementation: "test/second".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let described = runner + .describe_chain(&[transform, second]) + .await + .expect("describe two-stage chain"); + + let validator: Box> = Box::new(|body: &[u8]| { + if body == b"TRANSFORMED" { + Ok(Some("transformed body denied by policy".to_string())) + } else { + Ok(None) + } + }); + let outcome = runner + .evaluate_described_with_validator(&described, input("original"), Some(&*validator)) + .await + .expect("evaluate two-stage chain"); + + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "transformed body denied by policy"); + assert_eq!(outcome.applied.len(), 1, "only the first stage should run"); + assert_eq!(outcome.applied[0].name, "transform"); + assert!( + !second_ran.load(std::sync::atomic::Ordering::SeqCst), + "second stage must not run after a policy deny" + ); + } + + #[tokio::test] + async fn per_stage_validator_allows_compliant_transformations() { + // A validator that accepts every body lets both stages run; the second + // stage sees the first stage's output. + let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let service: Arc = Arc::new(TwoStageService { + second_ran: Arc::clone(&second_ran), + }); + let runner = ChainRunner::new(service); + let transform = ChainEntry { + name: "transform".into(), + implementation: "test/transform".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let second = ChainEntry { + name: "second".into(), + implementation: "test/second".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + let described = runner + .describe_chain(&[transform, second]) + .await + .expect("describe two-stage chain"); + + let validator: Box> = Box::new(|_body: &[u8]| Ok(None)); + let outcome = runner + .evaluate_described_with_validator(&described, input("original"), Some(&*validator)) + .await + .expect("evaluate two-stage chain"); + + assert!(outcome.allowed); + assert_eq!(outcome.applied.len(), 2); + assert!( + second_ran.load(std::sync::atomic::Ordering::SeqCst), + "second stage should run when the transformation is compliant" + ); + } + fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService { ScriptedService { binding_id: BUILTIN_SECRETS.into(), diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index d88e04c49c..a6f7f2987f 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -18,6 +18,69 @@ pub enum MiddlewareApplyResult { Denied(String), } +/// How traffic a middleware chain can never inspect (h2c, non-HTTP TCP, +/// protocols without an L7 relay) must be handled for a matching chain. +/// +/// This is derived from each entry's `on_error` today. A future per-config +/// `on_uninspectable` knob could let an operator keep `fail_closed` error +/// handling for HTTP traffic while allowing uninspectable protocols through +/// without maintaining host excludes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UninspectableTrafficGate { + /// No middleware matches this destination; raw relay is unaffected. + Unrestricted, + /// Every matching entry is `fail_open`: relay raw bytes but emit a bypass + /// detection finding. + BypassWithFinding, + /// At least one matching entry is `fail_closed`: deny, the middleware + /// must be able to see the traffic for it to flow. + Deny, +} + +pub fn uninspectable_traffic_gate( + chain: &[openshell_supervisor_middleware::ChainEntry], +) -> UninspectableTrafficGate { + if chain.is_empty() { + return UninspectableTrafficGate::Unrestricted; + } + if chain + .iter() + .all(|entry| entry.on_error == openshell_supervisor_middleware::OnError::FailOpen) + { + UninspectableTrafficGate::BypassWithFinding + } else { + UninspectableTrafficGate::Deny + } +} + +/// Emit the detection finding for traffic a matching middleware chain cannot +/// inspect: denied under a fail-closed chain, bypassed under fail-open. +pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied: bool) { + let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if denied { + SeverityId::High + } else { + SeverityId::Medium + }) + .finding_info(FindingInfo::new( + "openshell.middleware.traffic_uninspectable", + "Supervisor middleware cannot inspect this traffic", + )) + .evidence_pairs(&[ + ("policy", ctx.policy_name.as_str()), + ("host", ctx.host.as_str()), + ("protocol", detail), + ("disposition", if denied { "denied" } else { "fail_open" }), + ]) + .message(if denied { + "Uninspectable traffic to host with required middleware; denied" + } else { + "Uninspectable traffic bypassed middleware (fail_open)" + }) + .build(); + ocsf_emit!(event); +} + /// Largest body-buffering limit across the entries that actually resolved to a /// registered binding. Buffering for the most capable stage lets every stage /// that can handle the body run; stages whose own limit is smaller are failed @@ -44,11 +107,22 @@ pub async fn apply_middleware_chain( chain: Vec, runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, + validator: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>>, ) -> Result { - apply_middleware_chain_for_scheme(req, client, ctx, "https", chain, runner, generation_guard) - .await + apply_middleware_chain_for_scheme( + req, + client, + ctx, + "https", + chain, + runner, + generation_guard, + validator, + ) + .await } +#[allow(clippy::too_many_arguments)] pub async fn apply_middleware_chain_for_scheme( req: crate::l7::provider::L7Request, client: &mut C, @@ -57,6 +131,7 @@ pub async fn apply_middleware_chain_for_scheme, runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, + validator: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>>, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); @@ -93,7 +168,12 @@ pub async fn apply_middleware_chain_for_scheme Result> { Ok(out) } -pub(super) fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { +pub fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { crate::opa::NetworkInput { host: ctx.host.clone(), port: ctx.port, diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 0fce744f4a..cde4bf3329 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -8,7 +8,8 @@ //! and either forwards or denies the request. use crate::l7::middleware::{ - MiddlewareApplyResult, apply_middleware_chain, middleware_network_input, + MiddlewareApplyResult, UninspectableTrafficGate, apply_middleware_chain, + emit_middleware_uninspectable, middleware_network_input, uninspectable_traffic_gate, }; #[cfg(test)] use crate::l7::middleware::{ @@ -212,6 +213,20 @@ where if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); } + // The SQL relay is not implemented, so a matching middleware + // chain can never inspect this stream: gate it like any other + // uninspectable protocol. + let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + match uninspectable_traffic_gate(&chain) { + UninspectableTrafficGate::Deny => { + emit_middleware_uninspectable(ctx, "sql passthrough", true); + return Ok(()); + } + UninspectableTrafficGate::BypassWithFinding => { + emit_middleware_uninspectable(ctx, "sql passthrough", false); + } + UninspectableTrafficGate::Unrestricted => {} + } // SQL provider is Phase 3 — fall through to passthrough with warning { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -461,6 +476,11 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + // Route selection resolved `config` per request, so re-check the + // body against that protocol's policy after every transforming + // stage (a no-op for REST and websocket, whose policy inputs the + // chain cannot mutate). + let validate = transformed_body_validator(config, &engine, ctx, &request_info); let req = match apply_middleware_chain( req, client, @@ -468,10 +488,11 @@ where chain, engine.middleware_runner(), engine.generation_guard(), + Some(&validate), ) .await? { - MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { crate::l7::rest::RestProvider::default() .deny_with_redacted_target( @@ -950,6 +971,9 @@ where if allowed || config.enforcement == EnforcementMode::Audit { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + // REST and websocket-upgrade policy evaluates only the method, + // path, and query, which a middleware result cannot mutate, so no + // per-stage body re-check is needed. let req = match apply_middleware_chain( req, client, @@ -957,10 +981,11 @@ where chain, engine.middleware_runner(), engine.generation_guard(), + None, ) .await? { - MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { provider .deny_with_redacted_target( @@ -1225,6 +1250,11 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + // Policy admitted the original body above; re-check the body + // against the same body-aware policy after every transforming + // stage so a middleware cannot smuggle a denied operation to the + // upstream or the next stage. + let validate = transformed_body_validator(config, engine, ctx, &request_info); let req = match apply_middleware_chain( req, client, @@ -1232,10 +1262,11 @@ where chain, engine.middleware_runner(), engine.generation_guard(), + Some(&validate), ) .await? { - MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { crate::l7::rest::RestProvider::default() .deny_with_redacted_target( @@ -1455,6 +1486,11 @@ where if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + // Policy admitted the original body above; re-check the body + // against the same body-aware policy after every transforming + // stage so a middleware cannot smuggle a denied operation to the + // upstream or the next stage. + let validate = transformed_body_validator(config, engine, ctx, &request_info); let req = match apply_middleware_chain( req, client, @@ -1462,10 +1498,11 @@ where chain, engine.middleware_runner(), engine.generation_guard(), + Some(&validate), ) .await? { - MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { crate::l7::rest::RestProvider::default() .deny_with_redacted_target( @@ -1746,6 +1783,143 @@ fn jsonrpc_request_for_call( item_request } +/// Re-evaluate body-aware policy against a middleware-transformed body. Policy +/// admits the original body before the chain runs, so each replaced body must +/// be checked again before the next stage or the upstream sees it: a +/// transformation cannot smuggle a denied or unparseable operation past the +/// policy. Returns the deny reason, or `None` when the transformed body is +/// admissible. An unparseable replacement or a response frame denies even +/// under audit, mirroring `force_deny` for the original body; a policy deny +/// respects the endpoint's enforcement mode. Method, path, and query come from +/// `request_info` because a middleware result cannot mutate them. +/// +/// The match is exhaustive over `L7Protocol` on purpose: adding a protocol +/// does not compile until its transformed-body re-evaluation is defined here, +/// either by re-deriving the body-dependent policy inputs or by documenting +/// why none exist. Build the per-request validator with +/// [`transformed_body_validator`]. +fn reevaluate_transformed_body( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + ctx: &L7EvalContext, + request_info: &L7RequestInfo, + body: &[u8], +) -> Result> { + let (engine_type, transformed_info, hard_deny_reason) = match config.protocol { + // REST and websocket-upgrade policy evaluates only the method, path, + // and query, which a middleware result cannot mutate; the body is not + // a policy input. SQL has no body-aware L7 policy either; the + // uninspectable-traffic gate keeps required middleware ahead of the + // unimplemented SQL relay. + L7Protocol::Rest | L7Protocol::Websocket | L7Protocol::Sql => return Ok(None), + L7Protocol::JsonRpc | L7Protocol::Mcp => { + let info = crate::l7::jsonrpc::parse_jsonrpc_body_with_options( + body, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + ); + let hard_deny_reason = info + .error + .as_deref() + .map(|error| format!("JSON-RPC request rejected: {error}")) + .or_else(|| jsonrpc_response_frame_hard_deny_reason(config.protocol, &info)); + let mut transformed_info = request_info.clone(); + transformed_info.jsonrpc = Some(info); + ( + jsonrpc_engine_type(config.protocol), + transformed_info, + hard_deny_reason, + ) + } + L7Protocol::Graphql => { + // GraphQL classification needs the request method and query + // params; only the body was replaced, so rebuild from + // `request_info` and the new body. + let request = crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: request_info.target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + let info = crate::l7::graphql::classify_request(&request, body); + let hard_deny_reason = info + .error + .as_deref() + .map(|error| format!("GraphQL request rejected: {error}")); + let mut transformed_info = request_info.clone(); + transformed_info.graphql = Some(info); + ("l7-graphql", transformed_info, hard_deny_reason) + } + }; + + if let Some(reason) = hard_deny_reason { + let reason = format!("middleware transformation rejected: {reason}"); + emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + return Ok(Some(reason)); + } + + let (allowed, reason) = evaluate_l7_request(engine, ctx, &transformed_info)?; + if allowed { + return Ok(None); + } + let reason = format!("middleware transformation denied by policy: {reason}"); + if config.enforcement == EnforcementMode::Audit { + emit_transformed_body_decision(ctx, request_info, engine_type, "audit", &reason); + return Ok(None); + } + emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + Ok(Some(reason)) +} + +/// Build the per-stage transformed-body validator the middleware chain calls +/// after every stage that replaces the body. Borrows the policy inputs, so it +/// lives only as long as this request's evaluation. +pub(crate) fn transformed_body_validator<'a>( + config: &'a L7EndpointConfig, + engine: &'a TunnelPolicyEngine, + ctx: &'a L7EvalContext, + request_info: &'a L7RequestInfo, +) -> impl Fn(&[u8]) -> Result> + Send + Sync + 'a { + move |body: &[u8]| reevaluate_transformed_body(config, engine, ctx, request_info, body) +} + +/// Log the post-transformation policy decision as an OCSF HTTP Activity +/// event, mirroring the pre-middleware decision logs. `request_info.target` +/// is already redacted by the callers. +fn emit_transformed_body_decision( + ctx: &L7EvalContext, + request_info: &L7RequestInfo, + engine_type: &str, + decision_str: &str, + reason: &str, +) { + let (action_id, disposition_id, severity) = match decision_str { + "deny" => (ActionId::Denied, DispositionId::Blocked, SeverityId::Medium), + _ => ( + ActionId::Allowed, + DispositionId::Allowed, + SeverityId::Informational, + ), + }; + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(action_id) + .disposition(disposition_id) + .severity(severity) + .http_request(HttpRequest::new( + &request_info.action, + OcsfUrl::new("http", &ctx.host, &request_info.target, ctx.port), + )) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .firewall_rule(&ctx.policy_name, engine_type) + .message(format!( + "L7_REQUEST_TRANSFORMED {decision_str} {} {}:{}{} reason={}", + request_info.action, ctx.host, ctx.port, request_info.target, reason + )) + .build(); + ocsf_emit!(event); +} + fn evaluate_l7_request_once( engine: &TunnelPolicyEngine, ctx: &L7EvalContext, @@ -1918,9 +2092,12 @@ where return Ok(()); } let runner = engine.middleware_runner()?; - match apply_middleware_chain(req, client, ctx, chain, &runner, generation_guard).await? + // The passthrough path enforces no L7 policy, so there is no + // body-aware decision to re-check after a transformation. + match apply_middleware_chain(req, client, ctx, chain, &runner, generation_guard, None) + .await? { - MiddlewareApplyResult::Allowed(req) => req, + MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { crate::l7::rest::RestProvider::default() .deny_with_redacted_target( @@ -2998,6 +3175,510 @@ network_policies: assert_eq!(middleware_chain_body_limit(&none), None); } + /// A middleware service whose single binding replaces every request body + /// with a fixed payload, for exercising post-transformation policy + /// re-evaluation. + struct BodyReplacingService { + replacement: &'static [u8], + } + + #[tonic::async_trait] + impl openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware + for BodyReplacingService + { + async fn describe( + &self, + _request: tonic::Request<()>, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::MiddlewareManifest { + name: "test/rewriter".into(), + service_version: "test".into(), + bindings: vec![openshell_core::proto::MiddlewareBinding { + id: "example/rewriter".into(), + operation: openshell_core::proto::SupervisorMiddlewareOperation::HttpRequest + as i32, + phase: openshell_core::proto::SupervisorMiddlewarePhase::PreCredentials + as i32, + max_body_bytes: 8192, + }], + }, + )) + } + + async fn validate_config( + &self, + _request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + _request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::HttpRequestResult { + decision: openshell_core::proto::Decision::Allow as i32, + body: self.replacement.to_vec(), + has_body: true, + ..Default::default() + }, + )) + } + } + + fn jsonrpc_transforming_relay_parts( + enforcement: &str, + replacement: &'static [u8], + ) -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + let data = format!( + r#" +network_middlewares: + - name: rewriter + middleware: example/rewriter + on_error: fail_closed + endpoints: + include: ["api.example.test"] +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: api.example.test + port: 443 + protocol: json-rpc + enforcement: {enforcement} + rules: + - allow: + method: reports.list + binaries: + - {{ path: /usr/bin/node }} +"# + ); + let engine = OpaEngine::from_strings(TEST_POLICY, &data).unwrap(); + engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( + Arc::new(BodyReplacingService { replacement }), + )); + let input = NetworkInput { + host: "api.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let config = crate::l7::parse_l7_config(&endpoint_config.expect("json-rpc config")) + .expect("parse JSON-RPC config"); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + (config, tunnel_engine, ctx) + } + + async fn run_jsonrpc_transform_case( + enforcement: &str, + replacement: &'static [u8], + ) -> (String, Option) { + let (config, tunnel_engine, ctx) = + jsonrpc_transforming_relay_parts(enforcement, replacement); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_jsonrpc( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.list"}"#; + let request = format!( + "POST /rpc HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + // Give the relay a moment to either deny (client sees a response) or + // forward (upstream sees the request). + let mut upstream_request = [0u8; 1024]; + let upstream_read = tokio::time::timeout( + std::time::Duration::from_millis(300), + upstream.read(&mut upstream_request), + ) + .await; + let upstream_seen = match upstream_read { + Ok(Ok(n)) if n > 0 => { + let seen = String::from_utf8_lossy(&upstream_request[..n]).to_string(); + upstream + .write_all( + b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + Some(seen) + } + _ => None, + }; + + let mut response = [0u8; 1024]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("client should receive a response") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]).to_string(); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + (response, upstream_seen) + } + + #[tokio::test] + async fn transformed_jsonrpc_body_is_reevaluated_and_denied() { + // Policy allows reports.list; the middleware replaces the body with a + // method the policy denies. The transformed body must be re-evaluated + // and the request denied before anything reaches the upstream. + let (response, upstream_seen) = run_jsonrpc_transform_case( + "enforce", + br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, + ) + .await; + assert!(response.contains("403 Forbidden"), "{response}"); + assert!( + response.contains("middleware transformation denied by policy"), + "{response}" + ); + assert!(upstream_seen.is_none(), "upstream must not see the request"); + } + + #[tokio::test] + async fn transformed_jsonrpc_body_policy_deny_forwards_under_audit() { + // Under enforcement: audit a policy deny of the transformed body is + // logged but forwarded, mirroring audit semantics for original + // bodies. + let (response, upstream_seen) = run_jsonrpc_transform_case( + "audit", + br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, + ) + .await; + assert!(response.contains("204 No Content"), "{response}"); + let upstream_seen = upstream_seen.expect("audited request reaches upstream"); + assert!(upstream_seen.contains("admin.delete"), "{upstream_seen}"); + } + + #[tokio::test] + async fn unparseable_transformation_denies_even_under_audit() { + // An unparseable replacement mirrors force_deny for original parse + // errors: denied even on an audit endpoint. + let (response, upstream_seen) = run_jsonrpc_transform_case("audit", b"not json").await; + assert!(response.contains("403 Forbidden"), "{response}"); + assert!( + response.contains("middleware transformation rejected"), + "{response}" + ); + assert!(upstream_seen.is_none(), "upstream must not see the request"); + } + + #[tokio::test] + async fn transformed_graphql_body_is_reevaluated_and_denied() { + // GraphQL counterpart: policy allows query { viewer }; the middleware + // rewrites the body into a denied mutation. + let data = r#" +network_middlewares: + - name: rewriter + middleware: example/rewriter + on_error: fail_closed + endpoints: + include: ["api.example.test"] +network_policies: + graphql_api: + name: graphql_api + endpoints: + - host: api.example.test + port: 443 + protocol: graphql + enforcement: enforce + rules: + - allow: + operation_type: query + fields: [viewer] + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( + Arc::new(BodyReplacingService { + replacement: br#"{"query":"mutation { deleteRepository }"}"#, + }), + )); + let input = NetworkInput { + host: "api.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let config = crate::l7::parse_l7_config(&endpoint_config.expect("graphql config")) + .expect("parse GraphQL config"); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "graphql_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_graphql( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"query":"query { viewer }"}"#; + let request = format!( + "POST /graphql HTTP/1.1\r\nHost: api.example.test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ); + app.write_all(request.as_bytes()).await.unwrap(); + + let mut response = [0u8; 1024]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden"), "{response}"); + assert!( + response.contains("middleware transformation denied by policy"), + "{response}" + ); + + let mut upstream_request = [0u8; 32]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_request), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive request bytes" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + fn sql_middleware_relay_context( + on_error: &str, + ) -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + let data = format!( + r#" +network_middlewares: + - name: guard + middleware: example/unavailable + on_error: {on_error} + endpoints: + include: ["db.example.test"] +network_policies: + sql_db: + name: sql_db + endpoints: + - host: db.example.test + port: 5432 + protocol: sql + enforcement: audit + rules: + - allow: + command: SELECT + binaries: + - {{ path: /usr/bin/psql }} +"# + ); + let engine = OpaEngine::from_strings(TEST_POLICY, &data).unwrap(); + let input = NetworkInput { + host: "db.example.test".into(), + port: 5432, + binary_path: PathBuf::from("/usr/bin/psql"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let config = crate::l7::parse_l7_config(&endpoint_config.expect("sql config")) + .expect("parse SQL config"); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "db.example.test".into(), + port: 5432, + policy_name: "sql_db".into(), + binary_path: "/usr/bin/psql".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + (config, tunnel_engine, ctx) + } + + #[tokio::test] + async fn sql_passthrough_denies_with_fail_closed_middleware() { + // The SQL relay is unimplemented, so a fail-closed chain can never + // inspect the stream: the connection must be closed instead of + // silently bypassing the middleware. + let (config, tunnel_engine, ctx) = sql_middleware_relay_context("fail_closed"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all(b"\x00\x00\x00\x08\x04\xd2\x16\x2f") + .await + .ok(); + + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should close the connection") + .unwrap() + .unwrap(); + + let mut upstream_bytes = [0u8; 16]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_bytes), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "upstream should not receive SQL bytes" + ); + } + + #[tokio::test] + async fn sql_passthrough_relays_with_fail_open_middleware() { + // An all-fail-open chain accepts the bypass (with a detection + // finding) and the raw stream flows. + let (config, tunnel_engine, ctx) = sql_middleware_relay_context("fail_open"); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let _relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + app.write_all(b"\x00\x00\x00\x08\x04\xd2\x16\x2f") + .await + .unwrap(); + + let mut upstream_bytes = [0u8; 16]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_bytes), + ) + .await + .expect("fail-open chain must relay SQL bytes") + .unwrap(); + assert_eq!(&upstream_bytes[..n], b"\x00\x00\x00\x08\x04\xd2\x16\x2f"); + } + + #[test] + fn uninspectable_gate_reflects_chain_on_error() { + use openshell_supervisor_middleware::{ChainEntry, OnError}; + + let entry = |on_error| ChainEntry { + name: "m".into(), + implementation: "example/guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error, + }; + + assert_eq!( + uninspectable_traffic_gate(&[]), + UninspectableTrafficGate::Unrestricted + ); + assert_eq!( + uninspectable_traffic_gate(&[entry(OnError::FailOpen), entry(OnError::FailOpen)]), + UninspectableTrafficGate::BypassWithFinding + ); + assert_eq!( + uninspectable_traffic_gate(&[entry(OnError::FailOpen), entry(OnError::FailClosed)]), + UninspectableTrafficGate::Deny + ); + } + /// A middleware service advertising two bindings with different body /// limits, for exercising mixed-limit chain buffering at the relay level. /// The redactor binding replaces the body; the guard binding allows as-is. @@ -3120,6 +3801,7 @@ network_policies: chain, &runner, tunnel_engine.generation_guard(), + None, ) .await .expect("apply middleware chain"); diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 324d09785e..a6a73482fd 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -440,6 +440,17 @@ impl OpaEngine { .map_err(|_| miette::miette!("middleware runner lock poisoned")) } + /// Test-only: swap the middleware runner without a connected registry, so + /// relay tests can inject scripted middleware services. Does not bump the + /// policy generation; call before capturing tunnel engines. + #[cfg(test)] + pub(crate) fn set_middleware_runner_for_tests(&self, runner: ChainRunner) { + *self + .middleware_runner + .write() + .expect("middleware runner lock") = runner; + } + /// Return a guard for a previously captured policy generation. pub fn generation_guard(&self, expected_generation: u64) -> Result { let generation = self.current_generation(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 8a10ea3fdb..29363a3859 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -384,25 +384,64 @@ fn could_be_supported_tunnel_protocol_prefix(peek: &[u8]) -> bool { || crate::l7::rest::could_be_http2_prior_knowledge_prefix(peek) } -fn unsupported_l7_tunnel_protocol_detail( - tunnel_protocol: TunnelProtocol, +/// Why tunnel payload inspection is mandatory for a connection, in message +/// precedence order: an L7-configured endpoint owns the wording even when a +/// fail-closed middleware chain also matches. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum InspectionRequirement { + None, + L7Route, + RequiredMiddleware, +} + +fn inspection_requirement( should_inspect_l7: bool, -) -> Option<&'static str> { - if !should_inspect_l7 { - return None; + middleware_gate: crate::l7::middleware::UninspectableTrafficGate, +) -> InspectionRequirement { + if should_inspect_l7 { + InspectionRequirement::L7Route + } else if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::Deny { + InspectionRequirement::RequiredMiddleware + } else { + InspectionRequirement::None } +} - match tunnel_protocol { - TunnelProtocol::H2cPriorKnowledge => { +fn unsupported_l7_tunnel_protocol_detail( + tunnel_protocol: TunnelProtocol, + requirement: InspectionRequirement, +) -> Option<&'static str> { + match (tunnel_protocol, requirement) { + (_, InspectionRequirement::None) | (TunnelProtocol::Tls | TunnelProtocol::Http1, _) => None, + (TunnelProtocol::H2cPriorKnowledge, InspectionRequirement::L7Route) => { Some("HTTP/2 prior-knowledge (h2c) is not supported for L7-inspected endpoints") } - TunnelProtocol::Unsupported => { + (TunnelProtocol::H2cPriorKnowledge, InspectionRequirement::RequiredMiddleware) => { + Some("HTTP/2 prior-knowledge (h2c) cannot be inspected by required middleware") + } + (TunnelProtocol::Unsupported, InspectionRequirement::L7Route) => { Some("Unsupported tunnel protocol for L7-inspected endpoint") } - TunnelProtocol::Tls | TunnelProtocol::Http1 => None, + (TunnelProtocol::Unsupported, InspectionRequirement::RequiredMiddleware) => { + Some("Unsupported tunnel protocol cannot be inspected by required middleware") + } } } +/// Gate for traffic that would bypass L7 inspection entirely: query the +/// middleware chain matching this destination and process identity, and +/// decide whether raw relay is allowed. Uninspectable traffic is denied when +/// any matching entry is `fail_closed`; an all-`fail_open` chain passes it +/// through with a bypass detection finding. +fn middleware_uninspectable_gate( + opa_engine: &OpaEngine, + ctx: &crate::l7::relay::L7EvalContext, +) -> Result { + let input = crate::l7::middleware::middleware_network_input(ctx); + let (chain, _generation) = opa_engine.query_middleware_chain_with_generation(&input)?; + Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) +} + async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; @@ -1104,6 +1143,32 @@ async fn handle_tcp_connection( }; if effective_tls_skip { + // Policy validation rejects fail-closed middleware overlapping + // `tls: skip` endpoints; this runtime gate is defense in depth. + match middleware_uninspectable_gate(&opa_engine, &ctx)? { + crate::l7::middleware::UninspectableTrafficGate::Deny => { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, "tls-skip tunnel", true); + respond( + &mut client, + &build_json_error_response( + 403, + "Forbidden", + "middleware_required", + "tls: skip tunnel cannot be inspected by required middleware", + ), + ) + .await?; + return Ok(()); + } + crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding => { + crate::l7::middleware::emit_middleware_uninspectable( + &ctx, + "tls-skip tunnel", + false, + ); + } + crate::l7::middleware::UninspectableTrafficGate::Unrestricted => {} + } // tls: skip — raw tunnel, no termination, no credential injection. debug!( host = %host_lc, @@ -1208,6 +1273,29 @@ async fn handle_tcp_connection( } } } else { + // Without TLS state the stream cannot be terminated, so a + // matching middleware chain can never inspect it. Close instead + // of raw-copying when any matching entry is fail_closed; a 403 + // is useless mid-handshake, so the deny is the closed socket + // plus the finding. + match middleware_uninspectable_gate(&opa_engine, &ctx)? { + crate::l7::middleware::UninspectableTrafficGate::Deny => { + crate::l7::middleware::emit_middleware_uninspectable( + &ctx, + "tls without termination state", + true, + ); + return Ok(()); + } + crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding => { + crate::l7::middleware::emit_middleware_uninspectable( + &ctx, + "tls without termination state", + false, + ); + } + crate::l7::middleware::UninspectableTrafficGate::Unrestricted => {} + } { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) @@ -1308,9 +1396,14 @@ async fn handle_tcp_connection( } } } else { + let middleware_gate = middleware_uninspectable_gate(&opa_engine, &ctx)?; + let requirement = inspection_requirement(should_inspect_l7, middleware_gate); if let Some(protocol_detail) = - unsupported_l7_tunnel_protocol_detail(tunnel_protocol, should_inspect_l7) + unsupported_l7_tunnel_protocol_detail(tunnel_protocol, requirement) { + if requirement == InspectionRequirement::RequiredMiddleware { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, protocol_detail, true); + } let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -1353,6 +1446,9 @@ async fn handle_tcp_connection( return Ok(()); } + if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, "non-http tcp", false); + } // Neither TLS nor HTTP — raw binary relay. debug!( host = %host_lc, @@ -3362,6 +3458,11 @@ async fn handle_forward_proxy( let mut upstream_target = path.clone(); let mut websocket_extensions = crate::l7::rest::WebSocketExtensionMode::Preserve; let mut forward_tunnel_engine: Option = None; + // L7 endpoint config and evaluated request info, carried past the L7 + // block so a middleware-transformed body can be re-evaluated against the + // same policy inputs before it is forwarded. + let mut forward_l7_reeval: Option<(crate::l7::L7EndpointConfig, crate::l7::L7RequestInfo)> = + None; let mut forward_upgrade_config: Option = None; let mut forward_upgrade_target = String::new(); let mut forward_upgrade_query_params = std::collections::HashMap::new(); @@ -3807,6 +3908,7 @@ async fn handle_forward_proxy( } l7_activity_pending = true; forward_tunnel_engine = Some(tunnel_engine); + forward_l7_reeval = Some((l7_config.config.clone(), request_info)); } // 5. DNS resolution + SSRF defence (mirrors the CONNECT path logic). @@ -4225,6 +4327,24 @@ async fn handle_forward_proxy( &upstream_target, forward_request_bytes, )?; + // On a body-aware forward L7 route, re-check the body against the same + // policy after every transforming stage so a middleware cannot smuggle + // a denied operation past it. Non-L7 forwards enforce no body-aware + // policy, so they carry no validator. + let validate; + let validator_ref: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>> = + match (forward_l7_reeval.as_ref(), forward_tunnel_engine.as_ref()) { + (Some((config, request_info)), Some(engine)) => { + validate = crate::l7::relay::transformed_body_validator( + config, + engine, + &l7_ctx, + request_info, + ); + Some(&validate) + } + _ => None, + }; forward_request_bytes = match crate::l7::middleware::apply_middleware_chain_for_scheme( request, client, @@ -4233,6 +4353,7 @@ async fn handle_forward_proxy( chain, &middleware_runner, &forward_generation_guard, + validator_ref, ) .await? { @@ -4503,6 +4624,8 @@ mod tests { #[tokio::test] async fn h2c_prior_knowledge_is_blocked_for_l7_tunnel() { + use crate::l7::middleware::UninspectableTrafficGate; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); @@ -4519,10 +4642,44 @@ mod tests { .expect("client sent bytes"); assert_eq!(protocol, TunnelProtocol::H2cPriorKnowledge); assert_eq!( - unsupported_l7_tunnel_protocol_detail(protocol, true), + unsupported_l7_tunnel_protocol_detail(protocol, InspectionRequirement::L7Route), Some("HTTP/2 prior-knowledge (h2c) is not supported for L7-inspected endpoints") ); - assert_eq!(unsupported_l7_tunnel_protocol_detail(protocol, false), None); + // A fail-closed middleware chain makes inspection mandatory even for + // an L4-only endpoint. + assert_eq!( + unsupported_l7_tunnel_protocol_detail( + protocol, + InspectionRequirement::RequiredMiddleware + ), + Some("HTTP/2 prior-knowledge (h2c) cannot be inspected by required middleware") + ); + assert_eq!( + unsupported_l7_tunnel_protocol_detail( + TunnelProtocol::Unsupported, + InspectionRequirement::RequiredMiddleware + ), + Some("Unsupported tunnel protocol cannot be inspected by required middleware") + ); + assert_eq!( + unsupported_l7_tunnel_protocol_detail(protocol, InspectionRequirement::None), + None + ); + + // The L7 route owns the wording even when middleware also requires + // inspection; the middleware requirement applies only without a route. + assert_eq!( + inspection_requirement(true, UninspectableTrafficGate::Deny), + InspectionRequirement::L7Route + ); + assert_eq!( + inspection_requirement(false, UninspectableTrafficGate::Deny), + InspectionRequirement::RequiredMiddleware + ); + assert_eq!( + inspection_requirement(false, UninspectableTrafficGate::BypassWithFinding), + InspectionRequirement::None + ); } #[tokio::test] diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 667a14610b..3a4ad3c806 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -19,7 +19,10 @@ For each inspected HTTP request, the supervisor: 2. Selects middleware whose host selectors match the admitted destination. 3. Buffers the request body using the largest body limit in the selected chain. 4. Runs matching middleware by ascending `order`, using the policy-local name to break ties. -5. Applies allowed transformations, injects provider credentials, and forwards the request. +5. Re-checks body-aware protocol policy (GraphQL, JSON-RPC, MCP) after each stage that replaces the body. Every middleware receives a payload the policy admits, and a transformation cannot smuggle a denied or unparseable operation to a later stage or the upstream. +6. Applies allowed transformations, injects provider credentials, and forwards the request. + +Because each transformed body is re-checked before the next stage runs, a middleware hook always receives a request that satisfies the sandbox policy. A stage whose output the policy rejects stops the chain; under `enforcement: audit` the rejection is logged and the request proceeds. Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Request headers are delivered in wire order and repeated header names are preserved as separate entries, so a service inspects every value the upstream will receive. @@ -139,7 +142,7 @@ See [Logging](/observability/logging) for log access and [OCSF JSON Export](/obs ## Current Limitations -- Middleware applies only to HTTP requests parsed by the supervisor. +- Middleware applies only to HTTP requests parsed by the supervisor. Traffic the supervisor cannot inspect, such as HTTP/2 prior knowledge or non-HTTP TCP protocols, is denied on hosts matched by a fail-closed middleware, and relayed with a detection finding when every matching middleware is `fail_open`. - The typed operation and phase are `HTTP_REQUEST/PRE_CREDENTIALS`. - Selection uses destination host include and exclude patterns. - Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. From 5c0fc9f0cc53c6e3b26b7d78af24d8a73ae924b7 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 9 Jul 2026 17:33:59 -0700 Subject: [PATCH 36/59] refactor(middleware): centralize transformed body policy evaluation Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 8 + .../src/lib.rs | 127 +++++++-- .../src/l7/middleware.rs | 42 ++- .../src/l7/relay.rs | 201 ++++++++++---- .../openshell-supervisor-network/src/proxy.rs | 258 ++++++++++++++---- docs/extensibility/supervisor-middleware.mdx | 2 + 6 files changed, 493 insertions(+), 145 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 653f3d1e89..1226f9a541 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -85,6 +85,14 @@ overlap detection, matching, and chain ordering. The policy and runtime also share the core JSON/protobuf adapter for middleware configuration, keeping serialization consistent across that boundary. +The Rust HTTP pipeline makes transformed-body handling explicit for every +middleware invocation: body-independent protocols select a no-recheck mode, +while GraphQL, JSON-RPC, and MCP carry a policy evaluator bound to the captured +policy generation. Each replacement is reclassified and evaluated before the +next stage runs. Hard-deny classification is shared by CONNECT relays, +route-selected relays, and forward proxying; evaluator failures become +structured fail-closed outcomes instead of escaping the middleware pipeline. + `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index fc17857690..3444816732 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -111,9 +111,23 @@ impl DescribedChainEntry { /// Returns `Some(reason)` to deny the chain, `None` to proceed. Invoked after /// each stage that replaces the body so neither a later stage nor the upstream /// sees a payload the policy would reject. Protocols with no body-aware policy -/// pass `None` for the validator instead. +/// select [`TransformedBodyPolicy::NotPolicyRelevant`] instead. pub type TransformedBodyValidator<'a> = dyn Fn(&[u8]) -> Result> + Send + Sync + 'a; +/// Whether middleware body replacements affect the selected request policy. +/// +/// The network pipeline must choose a mode explicitly. This avoids representing +/// a security-relevant re-evaluation requirement as an optional callback where +/// an omitted value is indistinguishable from an intentionally body-independent +/// protocol. +#[derive(Clone, Copy)] +pub enum TransformedBodyPolicy<'a> { + /// The selected policy does not inspect the request body. + NotPolicyRelevant, + /// Re-evaluate every body replacement before the next stage runs. + Reevaluate(&'a TransformedBodyValidator<'a>), +} + #[derive(Debug, Clone)] pub struct HttpRequestInput { pub request_id: String, @@ -625,22 +639,26 @@ impl ChainRunner { entries: &[DescribedChainEntry], input: HttpRequestInput, ) -> Result { - self.evaluate_described_with_validator(entries, input, None) - .await + self.evaluate_described_with_policy( + entries, + input, + TransformedBodyPolicy::NotPolicyRelevant, + ) + .await } /// Evaluate a described chain, re-checking the request body against sandbox /// policy after every stage that replaces it. Policy runs on the original /// body before the chain, so without this a stage could hand the next stage - /// (or the upstream) a payload the policy rejects. When `validator` returns + /// (or the upstream) a payload the policy rejects. When the evaluator returns /// a deny reason the chain stops with that reason, so no later stage ever - /// sees a non-compliant body. `None` skips re-checks for protocols with no - /// body-aware policy. - pub async fn evaluate_described_with_validator( + /// sees a non-compliant body. Body-independent protocols must select + /// [`TransformedBodyPolicy::NotPolicyRelevant`] explicitly. + pub async fn evaluate_described_with_policy( &self, entries: &[DescribedChainEntry], input: HttpRequestInput, - validator: Option<&TransformedBodyValidator<'_>>, + transformed_body_policy: TransformedBodyPolicy<'_>, ) -> Result { let mut headers = input.headers.clone(); let mut body = input.body.clone(); @@ -831,18 +849,26 @@ impl ChainRunner { // before the next stage or the upstream sees the replaced body. A // policy deny here is a hard deny, independent of `on_error`. if transformed - && let Some(validate) = validator - && let Some(reason) = validate(&body)? + && let TransformedBodyPolicy::Reevaluate(validate) = transformed_body_policy { - return Ok(ChainOutcome { - allowed: false, - reason, - body, - added_headers, - findings, - metadata, - applied, - }); + let denied = match validate(&body) { + Ok(reason) => reason, + Err(error) => Some(format!( + "transformed_body_policy_evaluation_failed: {}", + safe_reason(&error.to_string()) + )), + }; + if let Some(reason) = denied { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + added_headers, + findings, + metadata, + applied, + }); + } } } @@ -1315,7 +1341,11 @@ mod tests { } }); let outcome = runner - .evaluate_described_with_validator(&described, input("original"), Some(&*validator)) + .evaluate_described_with_policy( + &described, + input("original"), + TransformedBodyPolicy::Reevaluate(&*validator), + ) .await .expect("evaluate two-stage chain"); @@ -1359,7 +1389,11 @@ mod tests { let validator: Box> = Box::new(|_body: &[u8]| Ok(None)); let outcome = runner - .evaluate_described_with_validator(&described, input("original"), Some(&*validator)) + .evaluate_described_with_policy( + &described, + input("original"), + TransformedBodyPolicy::Reevaluate(&*validator), + ) .await .expect("evaluate two-stage chain"); @@ -1371,6 +1405,57 @@ mod tests { ); } + #[tokio::test] + async fn per_stage_validator_error_becomes_structured_denial() { + let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let service: Arc = Arc::new(TwoStageService { + second_ran: Arc::clone(&second_ran), + }); + let runner = ChainRunner::new(service); + let entries = [ + ChainEntry { + name: "transform".into(), + implementation: "test/transform".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ChainEntry { + name: "second".into(), + implementation: "test/second".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ]; + let described = runner + .describe_chain(&entries) + .await + .expect("describe two-stage chain"); + let validator: Box> = + Box::new(|_body: &[u8]| Err(miette!("OPA engine unavailable"))); + + let outcome = runner + .evaluate_described_with_policy( + &described, + input("original"), + TransformedBodyPolicy::Reevaluate(&*validator), + ) + .await + .expect("policy evaluator failure should be a chain outcome"); + + assert!(!outcome.allowed); + assert!( + outcome + .reason + .starts_with("transformed_body_policy_evaluation_failed:"), + "{}", + outcome.reason + ); + assert_eq!(outcome.applied.len(), 1); + assert!(!second_ran.load(std::sync::atomic::Ordering::SeqCst)); + } + fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService { ScriptedService { binding_id: BUILTIN_SECRETS.into(), diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index a6f7f2987f..b0e236fd1e 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -107,7 +107,7 @@ pub async fn apply_middleware_chain( chain: Vec, runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, - validator: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>>, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, ) -> Result { apply_middleware_chain_for_scheme( req, @@ -117,7 +117,7 @@ pub async fn apply_middleware_chain( chain, runner, generation_guard, - validator, + transformed_body_policy, ) .await } @@ -131,7 +131,7 @@ pub async fn apply_middleware_chain_for_scheme, runner: &openshell_supervisor_middleware::ChainRunner, generation_guard: &PolicyGenerationGuard, - validator: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>>, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); @@ -168,24 +168,23 @@ pub async fn apply_middleware_chain_for_scheme Option { + request + .graphql + .as_ref() + .and_then(|info| info.error.as_deref()) + .map(|error| format!("GraphQL request rejected: {error}")) + .or_else(|| { + request.jsonrpc.as_ref().and_then(|info| { + info.error + .as_deref() + .map(|error| format!("JSON-RPC request rejected: {error}")) + .or_else(|| jsonrpc_response_frame_hard_deny_reason(protocol, info)) + }) + }) +} + /// Check if a miette error represents a benign connection close. /// /// TLS handshake EOF, missing `close_notify`, connection resets, and broken @@ -1805,7 +1811,7 @@ fn reevaluate_transformed_body( request_info: &L7RequestInfo, body: &[u8], ) -> Result> { - let (engine_type, transformed_info, hard_deny_reason) = match config.protocol { + let (engine_type, transformed_info) = match config.protocol { // REST and websocket-upgrade policy evaluates only the method, path, // and query, which a middleware result cannot mutate; the body is not // a policy input. SQL has no body-aware L7 policy either; the @@ -1817,18 +1823,9 @@ fn reevaluate_transformed_body( body, crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), ); - let hard_deny_reason = info - .error - .as_deref() - .map(|error| format!("JSON-RPC request rejected: {error}")) - .or_else(|| jsonrpc_response_frame_hard_deny_reason(config.protocol, &info)); let mut transformed_info = request_info.clone(); transformed_info.jsonrpc = Some(info); - ( - jsonrpc_engine_type(config.protocol), - transformed_info, - hard_deny_reason, - ) + (jsonrpc_engine_type(config.protocol), transformed_info) } L7Protocol::Graphql => { // GraphQL classification needs the request method and query @@ -1842,17 +1839,13 @@ fn reevaluate_transformed_body( body_length: crate::l7::provider::BodyLength::None, }; let info = crate::l7::graphql::classify_request(&request, body); - let hard_deny_reason = info - .error - .as_deref() - .map(|error| format!("GraphQL request rejected: {error}")); let mut transformed_info = request_info.clone(); transformed_info.graphql = Some(info); - ("l7-graphql", transformed_info, hard_deny_reason) + ("l7-graphql", transformed_info) } }; - if let Some(reason) = hard_deny_reason { + if let Some(reason) = l7_request_hard_deny_reason(config.protocol, &transformed_info) { let reason = format!("middleware transformation rejected: {reason}"); emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); return Ok(Some(reason)); @@ -2094,8 +2087,16 @@ where let runner = engine.middleware_runner()?; // The passthrough path enforces no L7 policy, so there is no // body-aware decision to re-check after a transformation. - match apply_middleware_chain(req, client, ctx, chain, &runner, generation_guard, None) - .await? + match apply_middleware_chain( + req, + client, + ctx, + chain, + &runner, + generation_guard, + openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, + ) + .await? { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied(reason) => { @@ -3801,7 +3802,7 @@ network_policies: chain, &runner, tunnel_engine.generation_guard(), - None, + openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, ) .await .expect("apply middleware chain"); @@ -4731,6 +4732,102 @@ network_policies: assert!(no_params_message.contains("rule_methods=initialize")); } + #[tokio::test] + async fn route_selected_jsonrpc_response_frame_hard_denies_under_audit() { + let data = r" +network_policies: + route_api: + name: route_api + endpoints: + - host: gateway.example.test + port: 443 + path: /rpc + protocol: json-rpc + enforcement: audit + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/node } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "gateway.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let configs = vec![ + crate::l7::parse_l7_config(&endpoint.expect("JSON-RPC endpoint")) + .expect("parse JSON-RPC config"), + ]; + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "gateway.example.test".into(), + port: 443, + policy_name: "route_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_route_selection( + &configs, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"jsonrpc":"2.0","id":7,"result":{"ok":true}}"#; + let request = format!( + "POST /rpc HTTP/1.1\r\nHost: gateway.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + app.write_all(request.as_bytes()).await.unwrap(); + app.write_all(body).await.unwrap(); + + let mut response = [0u8; 1024]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("hard denial should reach client") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!(response.contains("403 Forbidden"), "{response}"); + assert!(response.contains("response frames"), "{response}"); + + let mut upstream_bytes = [0u8; 16]; + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_bytes), + ) + .await; + assert!( + matches!(result, Err(_) | Ok(Ok(0))), + "hard-denied response frame must not reach upstream" + ); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + #[tokio::test] async fn route_selected_websocket_upgrade_rejects_invalid_accept_without_forwarding_101() { let data = r#" diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 29363a3859..0b9dbd9fd5 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -493,25 +493,65 @@ fn emit_forward_success_activity(tx: Option<&ActivitySender>, l7_activity_pendin ); } -fn forward_l7_hard_deny_reason( - protocol: crate::l7::L7Protocol, - request_info: &crate::l7::L7RequestInfo, -) -> Option { - request_info - .graphql - .as_ref() - .and_then(|info| info.error.as_deref()) - .map(|error| format!("GraphQL request rejected: {error}")) - .or_else(|| { - request_info.jsonrpc.as_ref().and_then(|info| { - info.error - .as_deref() - .map(|error| format!("JSON-RPC request rejected: {error}")) - .or_else(|| { - crate::l7::relay::jsonrpc_response_frame_hard_deny_reason(protocol, info) - }) - }) - }) +/// Body-aware policy state carried from forward L7 admission to middleware +/// execution. The selected route and its captured policy engine stay paired so +/// middleware cannot be invoked with only half of the re-evaluation context. +struct ForwardL7Reevaluation<'a> { + config: &'a crate::l7::L7EndpointConfig, + engine: &'a crate::opa::TunnelPolicyEngine, + request_info: &'a crate::l7::L7RequestInfo, +} + +/// Executes the middleware portion of the forward HTTP pipeline with an +/// explicit transformed-body policy. +struct ForwardMiddlewarePipeline<'a> { + ctx: &'a crate::l7::relay::L7EvalContext, + scheme: &'a str, + runner: &'a openshell_supervisor_middleware::ChainRunner, + generation_guard: &'a PolicyGenerationGuard, + l7_reevaluation: Option>, +} + +impl ForwardMiddlewarePipeline<'_> { + #[allow( + clippy::option_if_let_else, + reason = "the Some branch must keep a borrowed evaluator alive across the async call" + )] + async fn apply( + &self, + request: crate::l7::provider::L7Request, + client: &mut C, + chain: Vec, + ) -> Result + where + C: TokioAsyncRead + TokioAsyncWrite + Unpin + Send, + { + let validate; + let transformed_body_policy = match &self.l7_reevaluation { + Some(l7) => { + validate = crate::l7::relay::transformed_body_validator( + l7.config, + l7.engine, + self.ctx, + l7.request_info, + ); + openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate) + } + None => openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, + }; + + crate::l7::middleware::apply_middleware_chain_for_scheme( + request, + client, + self.ctx, + self.scheme, + chain, + self.runner, + self.generation_guard, + transformed_body_policy, + ) + .await + } } /// Emit a denial event to the aggregator channel (if configured). @@ -3787,10 +3827,10 @@ async fn handle_forward_proxy( jsonrpc, }; - let parse_error_reason = - forward_l7_hard_deny_reason(l7_config.config.protocol, &request_info); - let force_deny = parse_error_reason.is_some(); - let (allowed, reason) = parse_error_reason.map_or_else( + let hard_deny_reason = + crate::l7::relay::l7_request_hard_deny_reason(l7_config.config.protocol, &request_info); + let force_deny = hard_deny_reason.is_some(); + let (allowed, reason) = hard_deny_reason.map_or_else( || { crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) .unwrap_or_else(|e| { @@ -4327,36 +4367,22 @@ async fn handle_forward_proxy( &upstream_target, forward_request_bytes, )?; - // On a body-aware forward L7 route, re-check the body against the same - // policy after every transforming stage so a middleware cannot smuggle - // a denied operation past it. Non-L7 forwards enforce no body-aware - // policy, so they carry no validator. - let validate; - let validator_ref: Option<&openshell_supervisor_middleware::TransformedBodyValidator<'_>> = - match (forward_l7_reeval.as_ref(), forward_tunnel_engine.as_ref()) { - (Some((config, request_info)), Some(engine)) => { - validate = crate::l7::relay::transformed_body_validator( - config, - engine, - &l7_ctx, - request_info, - ); - Some(&validate) - } - _ => None, - }; - forward_request_bytes = match crate::l7::middleware::apply_middleware_chain_for_scheme( - request, - client, - &l7_ctx, - &scheme, - chain, - &middleware_runner, - &forward_generation_guard, - validator_ref, - ) - .await? - { + let l7_reevaluation = match (forward_l7_reeval.as_ref(), forward_tunnel_engine.as_ref()) { + (Some((config, request_info)), Some(engine)) => Some(ForwardL7Reevaluation { + config, + engine, + request_info, + }), + _ => None, + }; + let pipeline = ForwardMiddlewarePipeline { + ctx: &l7_ctx, + scheme: &scheme, + runner: &middleware_runner, + generation_guard: &forward_generation_guard, + l7_reevaluation, + }; + forward_request_bytes = match pipeline.apply(request, client, chain).await? { crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header, crate::l7::middleware::MiddlewareApplyResult::Denied(reason) => { emit_activity_simple(activity_tx, true, "middleware"); @@ -4782,7 +4808,7 @@ network_policies: } #[test] - fn forward_l7_hard_deny_reason_includes_jsonrpc_errors() { + fn l7_hard_deny_reason_includes_jsonrpc_errors() { let request_info = crate::l7::L7RequestInfo { action: "POST".to_string(), target: "/rpc".to_string(), @@ -4797,8 +4823,11 @@ network_policies: }), }; - let reason = forward_l7_hard_deny_reason(crate::l7::L7Protocol::JsonRpc, &request_info) - .expect("JSON-RPC parse error"); + let reason = crate::l7::relay::l7_request_hard_deny_reason( + crate::l7::L7Protocol::JsonRpc, + &request_info, + ) + .expect("JSON-RPC parse error"); assert_eq!( reason, @@ -4807,7 +4836,7 @@ network_policies: } #[test] - fn forward_l7_hard_deny_reason_includes_jsonrpc_response_frames() { + fn l7_hard_deny_reason_includes_jsonrpc_response_frames() { let request_info = crate::l7::L7RequestInfo { action: "POST".to_string(), target: "/rpc".to_string(), @@ -4822,16 +4851,125 @@ network_policies: }), }; - let reason = forward_l7_hard_deny_reason(crate::l7::L7Protocol::JsonRpc, &request_info) - .expect("JSON-RPC response hard deny"); + let reason = crate::l7::relay::l7_request_hard_deny_reason( + crate::l7::L7Protocol::JsonRpc, + &request_info, + ) + .expect("JSON-RPC response hard deny"); assert_eq!(reason, crate::l7::relay::JSONRPC_RESPONSE_FRAME_DENY_REASON); assert!( - forward_l7_hard_deny_reason(crate::l7::L7Protocol::Mcp, &request_info).is_none(), + crate::l7::relay::l7_request_hard_deny_reason( + crate::l7::L7Protocol::Mcp, + &request_info, + ) + .is_none(), "MCP response frames are evaluated by policy instead of hard-denied" ); } + #[tokio::test] + async fn forward_middleware_pipeline_denies_policy_invalid_transformation() { + const TEST_POLICY: &str = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: api.example.test + port: 80 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: sk-ABCDEFGHIJKLMNOP + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = crate::opa::NetworkInput { + host: "api.example.test".into(), + port: 80, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint, generation) = engine + .query_endpoint_config_with_generation(&input) + .expect("endpoint config"); + let config = crate::l7::parse_l7_config(&endpoint.expect("JSON-RPC endpoint")) + .expect("parse JSON-RPC config"); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let body = br#"{"jsonrpc":"2.0","id":1,"method":"sk-ABCDEFGHIJKLMNOP"}"#; + let request_info = crate::l7::L7RequestInfo { + action: "POST".into(), + target: "/rpc".into(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::parse_jsonrpc_body_with_options( + body, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(&config), + )), + }; + let raw = format!( + "POST /rpc HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + std::str::from_utf8(body).unwrap() + ) + .into_bytes(); + let request = + crate::l7::rest::request_from_buffered_http("POST", "/rpc", "/rpc", raw).unwrap(); + let ctx = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 80, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let runner = openshell_supervisor_middleware::ChainRunner::default(); + let pipeline = ForwardMiddlewarePipeline { + ctx: &ctx, + scheme: "http", + runner: &runner, + generation_guard: tunnel_engine.generation_guard(), + l7_reevaluation: Some(ForwardL7Reevaluation { + config: &config, + engine: &tunnel_engine, + request_info: &request_info, + }), + }; + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "redactor".into(), + implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let (_app, mut client) = tokio::io::duplex(8192); + + let outcome = pipeline + .apply(request, &mut client, chain) + .await + .expect("forward middleware pipeline"); + + match outcome { + crate::l7::middleware::MiddlewareApplyResult::Denied(reason) => assert!( + reason.contains("middleware transformation denied by policy"), + "{reason}" + ), + crate::l7::middleware::MiddlewareApplyResult::Allowed(_) => { + panic!("policy-invalid transformed request must be denied") + } + } + } + #[test] fn forward_l7_allowed_activity_is_deferred_until_after_ssrf() { let (tx, mut rx) = mpsc::channel(4); diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 3a4ad3c806..1678042f88 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -24,6 +24,8 @@ For each inspected HTTP request, the supervisor: Because each transformed body is re-checked before the next stage runs, a middleware hook always receives a request that satisfies the sandbox policy. A stage whose output the policy rejects stops the chain; under `enforcement: audit` the rejection is logged and the request proceeds. +If post-transformation policy evaluation itself fails, OpenShell denies the request and emits a high-severity detection finding. This failure is separate from middleware `on_error` because the middleware completed successfully; the sandbox policy could not validate its output. + Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Request headers are delivered in wire order and repeated header names are preserved as separate entries, so a service inspects every value the upstream will receive. ## Choose a Middleware Type From 81144bbe4f85673cb5719ff362c2a651d466de8f Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 10 Jul 2026 09:18:09 -0700 Subject: [PATCH 37/59] refactor(middleware): extract built-in implementations Signed-off-by: Piotr Mlocek --- AGENTS.md | 8 + CONTRIBUTING.md | 1 + Cargo.lock | 21 +- architecture/sandbox.md | 28 +- crates/openshell-core/src/lib.rs | 1 - crates/openshell-core/src/middleware.rs | 124 --------- crates/openshell-policy/Cargo.toml | 1 + crates/openshell-policy/src/lib.rs | 70 +++-- crates/openshell-policy/src/middleware.rs | 46 +-- crates/openshell-sandbox/Cargo.toml | 2 + crates/openshell-sandbox/src/lib.rs | 16 +- crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/lib.rs | 9 +- crates/openshell-server/src/middleware.rs | 33 +++ .../Cargo.toml | 27 ++ .../src/lib.rs | 178 ++++++++++++ .../src}/secrets.rs | 57 ++-- .../Cargo.toml | 4 +- .../src/builtins/mod.rs | 20 -- .../src/lib.rs | 262 +++++++++++++----- .../src/service.rs | 57 ---- .../openshell-supervisor-network/Cargo.toml | 1 + .../src/l7/relay.rs | 26 +- .../src/l7/rest.rs | 25 +- .../openshell-supervisor-network/src/opa.rs | 95 ++++++- .../openshell-supervisor-network/src/proxy.rs | 9 +- 26 files changed, 725 insertions(+), 397 deletions(-) delete mode 100644 crates/openshell-core/src/middleware.rs create mode 100644 crates/openshell-supervisor-middleware-builtins/Cargo.toml create mode 100644 crates/openshell-supervisor-middleware-builtins/src/lib.rs rename crates/{openshell-supervisor-middleware/src/builtins => openshell-supervisor-middleware-builtins/src}/secrets.rs (72%) delete mode 100644 crates/openshell-supervisor-middleware/src/builtins/mod.rs delete mode 100644 crates/openshell-supervisor-middleware/src/service.rs diff --git a/AGENTS.md b/AGENTS.md index 53b4e80496..f8c60eeb4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,15 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring | | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods | | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers | +| `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers | | `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | +| `crates/openshell-prover/` | Policy prover | Policy verification and proof generation | +| `crates/openshell-server-macros/` | Server macros | Compile-time helpers for gateway RPC authorization | +| `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution | +| `crates/openshell-supervisor-middleware-builtins/` | Built-in middleware | First-party in-process middleware implementations | +| `crates/openshell-supervisor-network/` | Network supervisor | Proxying, L7 enforcement, policy evaluation, and inference routing | +| `crates/openshell-supervisor-process/` | Process supervisor | Process lifecycle, namespace, and bypass monitoring | +| `crates/openshell-vfio/` | VFIO support | PCI and GPU passthrough preparation and lifecycle | | `python/openshell/` | Python SDK | Python bindings and CLI packaging | | `proto/` | Protobuf definitions | gRPC service contracts | | `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b78f36522f..8ecf52ff81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,6 +75,7 @@ Skills live in `.agents/skills/`. Your agent's harness can discover and load the | Contributing | `create-github-pr` | Create pull requests with proper conventions | | Reviewing | `review-github-pr` | Summarize PR diffs and key design decisions | | Reviewing | `review-security-issue` | Assess security issues for severity and remediation | +| Reviewing | `fix-security-issue` | Implement an approved security remediation plan | | Reviewing | `watch-github-actions` | Monitor CI pipeline status and logs | | Reviewing | `test-release-canary` | Dispatch and iterate on the Release Canary workflow that smoke-tests published artifacts | | Triage | `triage-issue` | Assess, classify, and route community-filed issues | diff --git a/Cargo.lock b/Cargo.lock index c0882410bb..dce80f0dd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3772,6 +3772,7 @@ version = "0.0.0" dependencies = [ "miette", "openshell-core", + "prost-types", "serde", "serde_json", "serde_yml", @@ -3832,8 +3833,10 @@ dependencies = [ "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", + "openshell-supervisor-middleware-builtins", "openshell-supervisor-network", "openshell-supervisor-process", + "prost-types", "rustls", "serde_json", "temp-env", @@ -3887,6 +3890,7 @@ dependencies = [ "openshell-router", "openshell-server-macros", "openshell-supervisor-middleware", + "openshell-supervisor-middleware-builtins", "petname", "pin-project-lite", "prost", @@ -3935,13 +3939,27 @@ version = "0.0.0" dependencies = [ "miette", "openshell-core", + "openshell-supervisor-middleware-builtins", "prost-types", - "regex", "tokio", "tokio-stream", "tonic", ] +[[package]] +name = "openshell-supervisor-middleware-builtins" +version = "0.0.0" +dependencies = [ + "miette", + "openshell-core", + "prost-types", + "regex", + "serde", + "serde_json", + "tokio", + "tonic", +] + [[package]] name = "openshell-supervisor-network" version = "0.0.0" @@ -3965,6 +3983,7 @@ dependencies = [ "openshell-policy", "openshell-router", "openshell-supervisor-middleware", + "openshell-supervisor-middleware-builtins", "prost-types", "rcgen", "regorus", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 1226f9a541..3a8ff20b08 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -73,17 +73,23 @@ values with stable name tie-breaking, and the gRPC contract represents operation and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external -service capabilities and policy-owned config before delivery. Supervisors keep -the last-known-good service registry when a live config reload fails. Built-in -middleware identifiers live in `openshell-core::middleware`; reusable compiled -DNS host patterns and selectors live in the neutral -`openshell-core::host_pattern` module. Structural policy validation lives in -`openshell-policy`; the supervisor's local-file path projects its JSON into the -same typed validator instead of maintaining an OPA-specific copy. Rego exposes -the middleware list as policy data, but Rust performs selector validation, -overlap detection, matching, and chain ordering. The policy and runtime also -share the core JSON/protobuf adapter for middleware configuration, keeping -serialization consistent across that boundary. +service capabilities and implementation-owned config before delivery. +Supervisors keep the last-known-good service registry when a live config reload +fails. The generic registry and chain runner live in +`openshell-supervisor-middleware`; +first-party implementations and their config schemas live in +`openshell-supervisor-middleware-builtins`. The gateway and supervisor inject +those services explicitly and discover their binding IDs through the same +`Describe` contract used by external services. Reusable compiled DNS host +patterns and selectors remain in `openshell-core::host_pattern`. + +`openshell-policy` validates policy-owned middleware structure without knowing +which implementations are installed. The active middleware registry validates +implementation-owned config before gateway admission. The supervisor's +local-file path supplies its built-in catalog to the same JSON projection so it +retains early config validation. Rego exposes the middleware list as policy +data, but Rust performs selector validation, overlap detection, matching, chain +ordering, implementation discovery, and config validation. The Rust HTTP pipeline makes transformed-body handling explicit for every middleware invocation: body-independent protocols select a no-recheck mode, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 4595f8965a..a2be6daa7d 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -24,7 +24,6 @@ pub mod host_pattern; pub mod image; pub mod inference; pub mod metadata; -pub mod middleware; pub mod net; pub mod paths; pub mod policy; diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs deleted file mode 100644 index 07a3b08657..0000000000 --- a/crates/openshell-core/src/middleware.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared supervisor middleware identifiers and policy validation contracts. - -use miette::{Result, miette}; -use serde::Deserialize; - -/// Binding identifier for the built-in secret redaction middleware. -pub const BUILTIN_SECRETS: &str = "openshell/secrets"; - -/// Policy-owned configuration for the built-in `openshell/secrets` middleware. -/// -/// Unknown fields and wrong-typed values are rejected so a config typo fails -/// policy validation instead of silently running with defaults. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct SecretsConfig { - /// Redaction mode. Omitting the field selects [`SecretsMode::Redact`]. - pub secrets: SecretsMode, -} - -/// Supported `openshell/secrets` modes. Phase 1 supports only `redact`. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum SecretsMode { - #[default] - Redact, -} - -impl SecretsConfig { - /// Parse and validate a policy-owned protobuf config. - pub fn from_struct(config: &prost_types::Struct) -> Result { - serde_json::from_value(crate::proto_struct::struct_to_json_value(config)).map_err(|error| { - miette!( - "invalid {BUILTIN_SECRETS} config: {error}; phase 1 supports only secrets: redact" - ) - }) - } -} - -/// Validate policy-owned configuration for a built-in middleware. -pub fn validate_builtin_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { - match implementation { - BUILTIN_SECRETS => SecretsConfig::from_struct(config).map(|_| ()), - other => Err(miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn string_config(key: &str, value: &str) -> prost_types::Struct { - prost_types::Struct { - fields: std::iter::once(( - key.to_string(), - prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue(value.into())), - }, - )) - .collect(), - } - } - - #[test] - fn secrets_config_defaults_to_redact() { - let config = SecretsConfig::from_struct(&prost_types::Struct::default()).unwrap(); - assert_eq!(config.secrets, SecretsMode::Redact); - } - - #[test] - fn secrets_config_accepts_explicit_redact() { - let config = SecretsConfig::from_struct(&string_config("secrets", "redact")).unwrap(); - assert_eq!(config.secrets, SecretsMode::Redact); - } - - #[test] - fn secrets_config_rejects_unsupported_mode() { - let error = validate_builtin_config(BUILTIN_SECRETS, &string_config("secrets", "allow")) - .unwrap_err(); - assert!(error.to_string().contains("supports only secrets: redact")); - } - - #[test] - fn secrets_config_rejects_unknown_field() { - let error = validate_builtin_config(BUILTIN_SECRETS, &string_config("secret", "redact")) - .unwrap_err(); - assert!(error.to_string().contains("unknown field")); - } - - #[test] - fn secrets_config_rejects_non_string_mode() { - let config = prost_types::Struct { - fields: std::iter::once(( - "secrets".to_string(), - prost_types::Value { - kind: Some(prost_types::value::Kind::NumberValue(42.0)), - }, - )) - .collect(), - }; - - let error = validate_builtin_config(BUILTIN_SECRETS, &config).unwrap_err(); - assert!( - error - .to_string() - .contains("invalid openshell/secrets config") - ); - } - - #[test] - fn rejects_unknown_builtin() { - let error = validate_builtin_config("openshell/unknown", &prost_types::Struct::default()) - .unwrap_err(); - assert!( - error - .to_string() - .contains("implementation 'openshell/unknown' is not available") - ); - } -} diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 16719de13d..b69da8d2b5 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -16,6 +16,7 @@ serde = { workspace = true } serde_json = { workspace = true } serde_yml = { workspace = true } miette = { workspace = true } +prost-types = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 2cc4d98a0a..29c7c7a0aa 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -35,6 +35,7 @@ pub use merge::{ }; pub use middleware::middleware_host_matches; pub use middleware::validate_json as validate_network_middleware_json; +pub use middleware::validate_json_with_config as validate_network_middleware_json_with_config; // --------------------------------------------------------------------------- // YAML serde types (canonical — used for both parsing and serialization) @@ -1133,8 +1134,6 @@ pub enum PolicyViolation { }, /// `credential_signing` and `request_body_credential_rewrite` are both set. CredentialSigningWithBodyRewrite { policy_name: String, host: String }, - /// A built-in middleware configuration is invalid. - InvalidBuiltinMiddlewareConfig { name: String, reason: String }, /// A middleware configuration is structurally invalid. InvalidMiddlewareConfig { name: String, reason: String }, /// Middleware configuration names must be unique. @@ -1209,8 +1208,7 @@ impl fmt::Display for PolicyViolation { and request_body_credential_rewrite set; these options are mutually exclusive" ) } - Self::InvalidBuiltinMiddlewareConfig { name, reason } - | Self::InvalidMiddlewareConfig { name, reason } => { + Self::InvalidMiddlewareConfig { name, reason } => { write!(f, "middleware config '{name}' is invalid: {reason}") } Self::DuplicateMiddlewareConfigName { name } => { @@ -1798,6 +1796,45 @@ network_policies: } } + #[test] + fn structural_validation_defers_implementation_owned_config() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("future", "openshell/future"); + middleware.config = Some( + openshell_core::proto_struct::json_object_to_struct( + std::iter::once(("implementation_field".into(), serde_json::json!(42))).collect(), + ) + .unwrap(), + ); + policy.network_middlewares.push(middleware); + + validate_sandbox_policy(&policy) + .expect("generic policy validation must not select installed implementations"); + } + + #[test] + fn json_validation_delegates_implementation_owned_config() { + let data = serde_json::json!({ + "network_middlewares": [{ + "name": "future", + "middleware": "openshell/future", + "config": {"implementation_field": 42}, + "endpoints": {"include": ["api.example.com"]} + }] + }); + + let violations = + validate_network_middleware_json_with_config(&data, |implementation, _config| { + Err(format!("{implementation} is not installed")) + }) + .expect("parse middleware policy"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::InvalidMiddlewareConfig { name, reason } + if name == "future" && reason.contains("not installed") + ))); + } + #[test] fn validate_rejects_root_run_as_user() { let mut policy = restrictive_default_policy(); @@ -1826,27 +1863,6 @@ network_policies: assert_eq!(violations.len(), 2); } - #[test] - fn validate_rejects_invalid_builtin_middleware_config() { - let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redact-secrets", "openshell/secrets"); - middleware.config = Some( - openshell_core::proto_struct::json_object_to_struct( - std::iter::once(("secrets".into(), serde_json::Value::String("allow".into()))) - .collect(), - ) - .unwrap(), - ); - policy.network_middlewares.push(middleware); - - let violations = validate_sandbox_policy(&policy).expect_err("invalid config"); - assert!(violations.iter().any(|violation| matches!( - violation, - PolicyViolation::InvalidBuiltinMiddlewareConfig { name, .. } - if name == "redact-secrets" - ))); - } - #[test] fn validate_rejects_invalid_middleware_control_fields() { let cases = [ @@ -1858,10 +1874,6 @@ network_policies: middleware_config("redactor", ""), "implementation must not be empty", ), - ( - middleware_config("redactor", "openshell/unknown"), - "unsupported built-in", - ), ( { let mut middleware = middleware_config("redactor", "openshell/secrets"); diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 52b70596dc..72a27e0603 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -127,6 +127,18 @@ pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec Result, String> { + validate_json_with_config(data, |_implementation, _config| Ok(())) +} + +/// Validate middleware policy structure and delegate implementation-owned +/// configuration to the supplied registry or catalog. +pub fn validate_json_with_config( + data: &serde_json::Value, + validate_config: F, +) -> Result, String> +where + F: Fn(&str, &prost_types::Struct) -> Result<(), String>, +{ let definition: MiddlewareValidationPolicyDef = serde_json::from_value(data.clone()) .map_err(|error| format!("failed to parse network middleware policy: {error}"))?; let network_middlewares = into_proto(definition.network_middlewares) @@ -151,11 +163,22 @@ pub fn validate_json(data: &serde_json::Value) -> Result, S (key, rule) }) .collect(); - Ok(validate(&SandboxPolicy { + let policy = SandboxPolicy { network_middlewares, network_policies, ..Default::default() - })) + }; + let mut violations = validate(&policy); + for middleware in &policy.network_middlewares { + let config = middleware.config.clone().unwrap_or_default(); + if let Err(reason) = validate_config(&middleware.middleware, &config) { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason, + }); + } + } + Ok(violations) } pub fn validate(policy: &SandboxPolicy) -> Vec { @@ -179,13 +202,6 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { name: middleware.name.clone(), reason: "implementation must not be empty".to_string(), }); - } else if middleware.middleware.starts_with("openshell/") - && middleware.middleware != openshell_core::middleware::BUILTIN_SECRETS - { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason: format!("unsupported built-in '{}'", middleware.middleware), - }); } if !matches!( @@ -227,18 +243,6 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { None }; - if middleware.middleware == openshell_core::middleware::BUILTIN_SECRETS { - let config = middleware.config.clone().unwrap_or_default(); - if let Err(error) = - openshell_core::middleware::validate_builtin_config(&middleware.middleware, &config) - { - violations.push(PolicyViolation::InvalidBuiltinMiddlewareConfig { - name: middleware.name.clone(), - reason: error.to_string(), - }); - } - } - for (key, rule) in &policy.network_policies { let policy_name = if rule.name.is_empty() { key diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index d3c3e7108a..30e99c70cd 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -20,6 +20,7 @@ openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } openshell-supervisor-process = { path = "../openshell-supervisor-process" } # Async runtime @@ -27,6 +28,7 @@ tokio = { workspace = true } # gRPC (tonic::Status downcast in error mapping) tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +prost-types = { workspace = true } # CLI clap = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 4a3d46a18b..b6fbf4f87e 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1388,10 +1388,22 @@ async fn load_policy( "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" )) .build()); - let engine = OpaEngine::from_files( + let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + let engine = OpaEngine::from_files_with_middleware_config( std::path::Path::new(policy_file), std::path::Path::new(data_file), + Some(&validate_middleware_config), )?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; let config = engine.query_sandbox_config()?; let mut policy = SandboxPolicy { version: 1, @@ -1539,6 +1551,7 @@ async fn load_policy( let middleware_services = snapshot.supervisor_middleware_services.clone(); let middleware_registry_status = match grpc_retry("Middleware connect", || { openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), middleware_services.clone(), ) }) @@ -2009,6 +2022,7 @@ async fn reconcile_middleware_registry( } match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), desired_services.to_vec(), ) .await diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index fafc72ba76..6192b4224a 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -27,6 +27,7 @@ openshell-providers = { path = "../openshell-providers" } openshell-router = { path = "../openshell-router" } openshell-server-macros = { path = "../openshell-server-macros" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } # Kubernetes client (used by the `generate-certs` subcommand) kube = { workspace = true } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 86afe4ef42..81b0fb666b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -241,9 +241,12 @@ pub(crate) async fn run_server( }) .unwrap_or_default(); let middleware_registry = Arc::new( - MiddlewareRegistry::connect_services(middleware_registrations) - .await - .map_err(|error| Error::config(format!("middleware registration failed: {error}")))?, + MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + middleware_registrations, + ) + .await + .map_err(|error| Error::config(format!("middleware registration failed: {error}")))?, ); let store = Arc::new(Store::connect(database_url).await?); diff --git a/crates/openshell-server/src/middleware.rs b/crates/openshell-server/src/middleware.rs index 4c94f021a3..603a086131 100644 --- a/crates/openshell-server/src/middleware.rs +++ b/crates/openshell-server/src/middleware.rs @@ -40,4 +40,37 @@ mod tests { assert_eq!(error.code(), tonic::Code::InvalidArgument); assert!(error.message().contains("not registered")); } + + #[tokio::test] + async fn invalid_builtin_config_is_rejected_by_implementation() { + let registry = MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await + .expect("built-in registry"); + let policy = SandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "redactor".into(), + middleware: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + config: Some(prost_types::Struct { + fields: std::iter::once(( + "secrets".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("allow".into())), + }, + )) + .collect(), + }), + ..Default::default() + }], + ..Default::default() + }; + + let error = validate_policy(®istry, &policy) + .await + .expect_err("invalid built-in config must fail admission"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + assert!(error.message().contains("supports only secrets: redact")); + } } diff --git a/crates/openshell-supervisor-middleware-builtins/Cargo.toml b/crates/openshell-supervisor-middleware-builtins/Cargo.toml new file mode 100644 index 0000000000..f892c718fd --- /dev/null +++ b/crates/openshell-supervisor-middleware-builtins/Cargo.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor-middleware-builtins" +description = "First-party in-process supervisor middleware implementations for OpenShell" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } + +miette = { workspace = true } +prost-types = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tonic = { workspace = true, features = ["server"] } + +[dev-dependencies] +tokio = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-supervisor-middleware-builtins/src/lib.rs b/crates/openshell-supervisor-middleware-builtins/src/lib.rs new file mode 100644 index 0000000000..ca040c3783 --- /dev/null +++ b/crates/openshell-supervisor-middleware-builtins/src/lib.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! First-party in-process supervisor middleware implementations. + +mod secrets; + +use std::sync::Arc; + +use miette::{Result, miette}; +use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; +use openshell_core::proto::{ + HttpRequestEvaluation, HttpRequestResult, MiddlewareManifest, ValidateConfigRequest, + ValidateConfigResponse, +}; +use tonic::{Request, Response, Status}; + +pub use secrets::{BINDING_ID as BUILTIN_SECRETS, SecretsConfig, SecretsMode}; + +/// Return the first-party services that the gateway and supervisor install. +pub fn services() -> Vec> { + vec![Arc::new(BuiltinMiddlewareService)] +} + +/// Validate configuration for a first-party binding. +pub fn validate_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { + match implementation { + BUILTIN_SECRETS => secrets::validate_config(config), + other => Err(miette!( + "middleware implementation '{other}' is not a registered OpenShell built-in" + )), + } +} + +fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { + match evaluation.binding_id.as_str() { + BUILTIN_SECRETS => secrets::evaluate_http_request(evaluation), + other => Err(miette!( + "middleware implementation '{other}' is not a registered OpenShell built-in" + )), + } +} + +/// Aggregate service exposing all first-party bindings through the standard +/// supervisor middleware contract. +#[derive(Debug, Default)] +pub struct BuiltinMiddlewareService; + +#[tonic::async_trait] +impl SupervisorMiddleware for BuiltinMiddlewareService { + async fn describe( + &self, + _request: Request<()>, + ) -> Result, Status> { + Ok(Response::new(MiddlewareManifest { + name: "openshell/builtins".into(), + service_version: env!("CARGO_PKG_VERSION").into(), + bindings: vec![secrets::describe()], + })) + } + + async fn validate_config( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let config = request.config.unwrap_or_default(); + Ok(Response::new( + match validate_config(&request.binding_id, &config) { + Ok(()) => ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + Err(error) => ValidateConfigResponse { + valid: false, + reason: error.to_string(), + }, + }, + )) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> Result, Status> { + evaluate_http_request(&request.into_inner()) + .map(Response::new) + .map_err(|error| Status::invalid_argument(error.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{ + Decision, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, + }; + + fn string_config(key: &str, value: &str) -> prost_types::Struct { + prost_types::Struct { + fields: std::iter::once(( + key.to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue(value.into())), + }, + )) + .collect(), + } + } + + #[tokio::test] + async fn service_describes_secrets_binding() { + let manifest = BuiltinMiddlewareService + .describe(Request::new(())) + .await + .expect("describe") + .into_inner(); + assert_eq!(manifest.bindings.len(), 1); + assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); + assert_eq!( + manifest.bindings[0].operation, + SupervisorMiddlewareOperation::HttpRequest as i32 + ); + assert_eq!( + manifest.bindings[0].phase, + SupervisorMiddlewarePhase::PreCredentials as i32 + ); + assert_eq!(manifest.bindings[0].max_body_bytes, 256 * 1024); + } + + #[test] + fn secrets_config_defaults_to_redact() { + let config = SecretsConfig::from_struct(&prost_types::Struct::default()).unwrap(); + assert_eq!(config.secrets, SecretsMode::Redact); + } + + #[test] + fn secrets_config_accepts_explicit_redact() { + let config = SecretsConfig::from_struct(&string_config("secrets", "redact")).unwrap(); + assert_eq!(config.secrets, SecretsMode::Redact); + } + + #[test] + fn secrets_config_rejects_unsupported_or_malformed_values() { + for config in [ + string_config("secrets", "allow"), + string_config("secret", "redact"), + prost_types::Struct { + fields: std::iter::once(( + "secrets".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::NumberValue(42.0)), + }, + )) + .collect(), + }, + ] { + assert!(validate_config(BUILTIN_SECRETS, &config).is_err()); + } + } + + #[test] + fn secrets_redaction_evaluates_through_binding() { + let result = evaluate_http_request(&HttpRequestEvaluation { + binding_id: BUILTIN_SECRETS.into(), + body: br#"{"password":"top-secret","token":"sk-ABCDEFGHIJKLMNOP"}"#.to_vec(), + config: Some(prost_types::Struct::default()), + ..Default::default() + }) + .expect("evaluate secrets binding"); + + assert_eq!(result.decision, Decision::Allow as i32); + assert!(result.has_body); + let body = String::from_utf8(result.body).unwrap(); + assert!(!body.contains("top-secret")); + assert!(!body.contains("sk-ABCDEFGHIJKLMNOP")); + } +} diff --git a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs similarity index 72% rename from crates/openshell-supervisor-middleware/src/builtins/secrets.rs rename to crates/openshell-supervisor-middleware-builtins/src/secrets.rs index e51b6e7af1..3deaf84992 100644 --- a/crates/openshell-supervisor-middleware/src/builtins/secrets.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs @@ -10,10 +10,37 @@ use openshell_core::proto::{ SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, }; use regex::Regex; +use serde::Deserialize; -pub const BINDING_ID: &str = openshell_core::middleware::BUILTIN_SECRETS; +pub const BINDING_ID: &str = "openshell/secrets"; const MAX_BODY_BYTES: u64 = 256 * 1024; +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SecretsConfig { + /// Redaction mode. Omitting the field selects [`SecretsMode::Redact`]. + pub secrets: SecretsMode, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SecretsMode { + #[default] + Redact, +} + +impl SecretsConfig { + pub fn from_struct(config: &prost_types::Struct) -> Result { + serde_json::from_value(openshell_core::proto_struct::struct_to_json_value(config)).map_err( + |error| { + miette!( + "invalid {BINDING_ID} config: {error}; phase 1 supports only secrets: redact" + ) + }, + ) + } +} + pub fn describe() -> MiddlewareBinding { MiddlewareBinding { id: BINDING_ID.into(), @@ -23,9 +50,6 @@ pub fn describe() -> MiddlewareBinding { } } -/// A named secret-detection pattern. The `kind` is an audit-safe label that -/// flows into findings so operators can see *what* matched without seeing the -/// raw value. struct SecretPattern { kind: &'static str, regex: Regex, @@ -40,8 +64,6 @@ impl SecretPattern { } } -/// Compiled once: recompiling per request would put regex construction on the -/// egress hot path. static SECRET_PATTERNS: LazyLock<[SecretPattern; 2]> = LazyLock::new(|| { [ SecretPattern::new( @@ -53,7 +75,7 @@ static SECRET_PATTERNS: LazyLock<[SecretPattern; 2]> = LazyLock::new(|| { }); pub fn validate_config(config: &prost_types::Struct) -> Result<()> { - openshell_core::middleware::validate_builtin_config(BINDING_ID, config) + SecretsConfig::from_struct(config).map(|_| ()) } pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { @@ -74,17 +96,16 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result Result (String, Vec<(&'static str, u32)>) { let mut output = input.to_string(); let mut matches = Vec::new(); diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index f36fc854b6..2c095c64a6 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openshell-supervisor-middleware" -description = "In-process supervisor middleware contract and built-ins for OpenShell" +description = "Supervisor middleware registry and chain execution for OpenShell" version.workspace = true edition.workspace = true license.workspace = true @@ -15,11 +15,11 @@ openshell-core = { path = "../openshell-core", default-features = false } miette = { workspace = true } prost-types = { workspace = true } -regex = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["channel", "server", "tls-native-roots"] } [dev-dependencies] +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } tokio-stream = { workspace = true, features = ["net"] } [lints] diff --git a/crates/openshell-supervisor-middleware/src/builtins/mod.rs b/crates/openshell-supervisor-middleware/src/builtins/mod.rs deleted file mode 100644 index 176f4786cf..0000000000 --- a/crates/openshell-supervisor-middleware/src/builtins/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub mod secrets; - -use miette::{Result, miette}; -use openshell_core::proto::{HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding}; - -pub fn describe() -> Vec { - vec![secrets::describe()] -} - -pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { - match evaluation.binding_id.as_str() { - secrets::BINDING_ID => secrets::evaluate_http_request(evaluation), - other => Err(miette!( - "middleware implementation '{other}' is not available in phase 1" - )), - } -} diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 3444816732..2a95d698d8 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -3,16 +3,12 @@ //! Supervisor middleware registration and chain execution. -mod builtins; mod remote; -mod service; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; use miette::{Result, miette}; -pub use openshell_core::middleware::{BUILTIN_SECRETS, validate_builtin_config}; -pub use service::InProcessMiddlewareService; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ @@ -223,23 +219,17 @@ struct MiddlewareServiceState { operator_max_body_bytes: Option, } -static IN_PROCESS_SERVICE: LazyLock> = LazyLock::new(|| { - Arc::new(MiddlewareServiceState { - service: Arc::new(InProcessMiddlewareService), - manifest: OnceCell::new(), - operator_max_body_bytes: None, - }) -}); - /// Validated middleware services available to a gateway or one supervisor. /// -/// The registry always contains the in-process built-ins. Operator-registered -/// services are connected and described before construction succeeds, so callers never +/// In-process services are supplied by the composition root; the generic +/// registry does not select concrete built-ins. All in-process and remote +/// services are described before construction succeeds, so callers never /// observe a partially registered service set. #[derive(Clone)] pub struct MiddlewareRegistry { services: Arc>>, registered_services: Arc>, + binding_ids: Arc>, } impl std::fmt::Debug for MiddlewareRegistry { @@ -248,6 +238,7 @@ impl std::fmt::Debug for MiddlewareRegistry { .debug_struct("MiddlewareRegistry") .field("service_count", &self.services.len()) .field("registered_service_count", &self.registered_services.len()) + .field("binding_count", &self.binding_ids.len()) .finish() } } @@ -261,8 +252,9 @@ struct RegisteredMiddlewareService { impl Default for MiddlewareRegistry { fn default() -> Self { Self { - services: Arc::new(vec![Arc::clone(&IN_PROCESS_SERVICE)]), + services: Arc::new(Vec::new()), registered_services: Arc::new(Vec::new()), + binding_ids: Arc::new(HashSet::new()), } } } @@ -290,31 +282,25 @@ fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<( Ok(()) } -fn validate_external_manifest( - registration: &SupervisorMiddlewareService, +fn validate_manifest_bindings( + source: &str, manifest: &MiddlewareManifest, - operator_max_body_bytes: usize, + operator_max_body_bytes: Option, + allow_reserved_bindings: bool, known_binding_ids: &mut HashSet, ) -> Result> { if manifest.bindings.is_empty() { - return Err(miette!( - "middleware registration '{}' describes no bindings", - registration.name - )); + return Err(miette!("{source} describes no bindings")); } let mut described_ids = Vec::with_capacity(manifest.bindings.len()); for binding in &manifest.bindings { if binding.id.trim().is_empty() { - return Err(miette!( - "middleware registration '{}' describes an empty binding id", - registration.name - )); + return Err(miette!("{source} describes an empty binding id")); } - if binding.id.starts_with("openshell/") { + if !allow_reserved_bindings && binding.id.starts_with("openshell/") { return Err(miette!( - "external middleware registration '{}' cannot claim reserved binding '{}'", - registration.name, + "{source} cannot claim reserved binding '{}'", binding.id )); } @@ -338,10 +324,10 @@ fn validate_external_manifest( binding.id )); } - if operator_max_body_bytes > advertised { + if operator_max_body_bytes.is_some_and(|limit| limit > advertised) { return Err(miette!( - "middleware registration '{}' max_body_bytes ({operator_max_body_bytes}) exceeds binding '{}' capability ({advertised})", - registration.name, + "{source} max_body_bytes ({}) exceeds binding '{}' capability ({advertised})", + operator_max_body_bytes.expect("operator limit checked above"), binding.id )); } @@ -356,13 +342,60 @@ fn validate_external_manifest( Ok(described_ids) } +fn validate_external_manifest( + registration: &SupervisorMiddlewareService, + manifest: &MiddlewareManifest, + operator_max_body_bytes: usize, + known_binding_ids: &mut HashSet, +) -> Result> { + validate_manifest_bindings( + &format!("external middleware registration '{}'", registration.name), + manifest, + Some(operator_max_body_bytes), + false, + known_binding_ids, + ) +} + impl MiddlewareRegistry { - /// Connect and validate every operator-provided service registration. - pub async fn connect_services(registrations: Vec) -> Result { - let mut services = vec![Arc::clone(&IN_PROCESS_SERVICE)]; + /// Describe in-process services, then connect and validate every + /// operator-provided service registration. + pub async fn connect_services( + in_process_services: Vec>, + registrations: Vec, + ) -> Result { + let mut services = Vec::with_capacity(in_process_services.len() + registrations.len()); let mut registered_services = Vec::with_capacity(registrations.len()); let mut registration_names = HashSet::new(); - let mut binding_ids = HashSet::from([BUILTIN_SECRETS.to_string()]); + let mut binding_ids = HashSet::new(); + + for service in in_process_services { + let manifest = service + .describe(Request::new(())) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "in-process middleware Describe failed: {}", + safe_reason(&error.to_string()) + ) + })?; + let source = if manifest.name.trim().is_empty() { + "in-process middleware service".to_string() + } else { + format!("in-process middleware service '{}'", manifest.name) + }; + validate_manifest_bindings(&source, &manifest, None, true, &mut binding_ids)?; + let manifest_cell = OnceCell::new(); + manifest_cell + .set(manifest) + .map_err(|_| miette!("middleware manifest cache initialized twice"))?; + services.push(Arc::new(MiddlewareServiceState { + service, + manifest: manifest_cell, + operator_max_body_bytes: None, + })); + } for registration in registrations { validate_registration(®istration)?; @@ -422,6 +455,7 @@ impl MiddlewareRegistry { Ok(Self { services: Arc::new(services), registered_services: Arc::new(registered_services), + binding_ids: Arc::new(binding_ids), }) } @@ -450,14 +484,7 @@ impl MiddlewareRegistry { /// registry without making a network call. pub fn ensure_policy_bindings_registered(&self, policy: &SandboxPolicy) -> Result<()> { for config in &policy.network_middlewares { - let registered = config.middleware == BUILTIN_SECRETS - || self.registered_services.iter().any(|service| { - service - .binding_ids - .iter() - .any(|binding| binding == &config.middleware) - }); - if !registered { + if !self.binding_ids.contains(&config.middleware) { return Err(miette!( "middleware binding '{}' used by config '{}' is not registered", config.middleware, @@ -510,6 +537,7 @@ impl ChainRunner { operator_max_body_bytes: None, })]), registered_services: Arc::new(Vec::new()), + binding_ids: Arc::new(HashSet::new()), }), } } @@ -997,8 +1025,18 @@ mod tests { use openshell_core::proto::middleware::v1::supervisor_middleware_server::{ SupervisorMiddleware, SupervisorMiddlewareServer, }; + use openshell_supervisor_middleware_builtins::{BUILTIN_SECRETS, services}; use tokio_stream::wrappers::TcpListenerStream; + fn builtin_runner() -> ChainRunner { + ChainRunner::new( + services() + .into_iter() + .next() + .expect("built-in middleware service"), + ) + } + fn entry(name: &str, on_error: OnError) -> ChainEntry { ChainEntry { name: name.into(), @@ -1034,7 +1072,7 @@ mod tests { #[tokio::test] async fn phase_one_evaluation_omits_originating_process() { - let entries = ChainRunner::default() + let entries = builtin_runner() .describe_chain(&[entry("redact", OnError::FailClosed)]) .await .expect("describe chain"); @@ -1058,7 +1096,7 @@ mod tests { #[tokio::test] async fn redacts_common_secret_patterns() { - let outcome = ChainRunner::default() + let outcome = builtin_runner() .evaluate( &[entry("redact", OnError::FailClosed)], input(r#"{"api_key":"sk-1234567890abcdef"}"#), @@ -1079,7 +1117,7 @@ mod tests { entry("first", OnError::FailClosed), entry("second", OnError::FailClosed), ]; - let outcome = ChainRunner::default() + let outcome = builtin_runner() .evaluate(&entries, input(r#"password="top-secret""#)) .await .expect("evaluate"); @@ -1100,7 +1138,7 @@ mod tests { let mut alpha = entry("alpha", OnError::FailClosed); alpha.order = 10; - let described = ChainRunner::default() + let described = builtin_runner() .describe_chain(&[later, beta, alpha]) .await .expect("describe ordered chain"); @@ -1120,7 +1158,7 @@ mod tests { config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; - let outcome = ChainRunner::default() + let outcome = builtin_runner() .evaluate(&[unavailable], input("hello")) .await .expect("evaluate"); @@ -1137,7 +1175,7 @@ mod tests { config: prost_types::Struct::default(), on_error: OnError::FailClosed, }; - let outcome = ChainRunner::default() + let outcome = builtin_runner() .evaluate(&[unavailable], input("hello")) .await .expect("evaluate"); @@ -1146,22 +1184,54 @@ mod tests { } #[tokio::test] - async fn in_process_service_describes_builtin_binding() { - let manifest = InProcessMiddlewareService - .describe(Request::new(())) + async fn injected_service_bindings_drive_registration_checks() { + let registry = MiddlewareRegistry::connect_services(services(), Vec::new()) .await - .expect("describe") - .into_inner(); - assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); - assert_eq!( - manifest.bindings[0].operation, - SupervisorMiddlewareOperation::HttpRequest as i32 - ); - assert_eq!( - manifest.bindings[0].phase, - SupervisorMiddlewarePhase::PreCredentials as i32 + .expect("connect built-in service"); + let policy = SandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "redactor".into(), + middleware: BUILTIN_SECRETS.into(), + ..Default::default() + }], + ..Default::default() + }; + registry + .ensure_policy_bindings_registered(&policy) + .expect("described binding is registered"); + + let unknown = SandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "unknown".into(), + middleware: "openshell/unknown".into(), + ..Default::default() + }], + ..Default::default() + }; + assert!( + registry + .ensure_policy_bindings_registered(&unknown) + .is_err() ); - assert_eq!(manifest.bindings[0].max_body_bytes, 256 * 1024); + } + + #[tokio::test] + async fn injected_services_cannot_duplicate_binding_ids() { + let first: Arc = Arc::new(ScriptedService { + binding_id: "openshell/test".into(), + max_body_bytes: 1024, + result: allow_result(), + }); + let second: Arc = Arc::new(ScriptedService { + binding_id: "openshell/test".into(), + max_body_bytes: 1024, + result: allow_result(), + }); + + let error = MiddlewareRegistry::connect_services(vec![first, second], Vec::new()) + .await + .expect_err("duplicate injected binding must fail registry construction"); + assert!(error.to_string().contains("more than one service")); } #[test] @@ -1606,13 +1676,35 @@ mod tests { service: Arc, registration: SupervisorMiddlewareService, ) -> MiddlewareRegistry { + let builtin_service = services() + .into_iter() + .next() + .expect("built-in middleware service"); + let builtin_manifest = builtin_service + .describe(Request::new(())) + .await + .expect("describe built-in service") + .into_inner(); + let mut known = HashSet::new(); + validate_manifest_bindings( + "test built-in service", + &builtin_manifest, + None, + true, + &mut known, + ) + .expect("valid built-in manifest"); + let builtin_manifest_cell = OnceCell::new(); + builtin_manifest_cell + .set(builtin_manifest) + .expect("built-in manifest cache"); + let manifest = service .describe(Request::new(())) .await .expect("describe test service") .into_inner(); let operator_max_body_bytes = usize::try_from(registration.max_body_bytes).unwrap(); - let mut known = HashSet::from([BUILTIN_SECRETS.to_string()]); let binding_ids = validate_external_manifest( ®istration, &manifest, @@ -1624,7 +1716,11 @@ mod tests { manifest_cell.set(manifest).expect("manifest cache"); MiddlewareRegistry { services: Arc::new(vec![ - Arc::clone(&IN_PROCESS_SERVICE), + Arc::new(MiddlewareServiceState { + service: builtin_service, + manifest: builtin_manifest_cell, + operator_max_body_bytes: None, + }), Arc::new(MiddlewareServiceState { service, manifest: manifest_cell, @@ -1635,6 +1731,7 @@ mod tests { registration, binding_ids, }]), + binding_ids: Arc::new(known), } } @@ -1647,7 +1744,7 @@ mod tests { config: prost_types::Struct::default(), on_error: OnError::FailOpen, }; - let described = ChainRunner::default() + let described = builtin_runner() .describe_chain(&[entry("redact", OnError::FailClosed), unresolved]) .await .expect("describe chain"); @@ -1827,16 +1924,29 @@ mod tests { max_body_bytes: 4096, }], }; - let error = validate_external_manifest( - ®istration, - &manifest, - 4097, - &mut HashSet::from([BUILTIN_SECRETS.to_string()]), - ) - .expect_err("operator limit must fit capability"); + let error = validate_external_manifest(®istration, &manifest, 4097, &mut HashSet::new()) + .expect_err("operator limit must fit capability"); assert!(error.to_string().contains("exceeds")); } + #[test] + fn external_manifest_cannot_claim_reserved_binding() { + let registration = external_registration(4096); + let manifest = MiddlewareManifest { + name: "example/service".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "openshell/secrets".into(), + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, + max_body_bytes: 4096, + }], + }; + let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) + .expect_err("external service cannot claim reserved namespace"); + assert!(error.to_string().contains("reserved binding")); + } + #[test] fn external_registration_accepts_http_and_https_grpc_endpoints() { for grpc_endpoint in [ @@ -1877,7 +1987,7 @@ mod tests { let mut registration = external_registration(1024); registration.grpc_endpoint = format!("http://{address}"); - let registry = MiddlewareRegistry::connect_services(vec![registration.clone()]) + let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration.clone()]) .await .expect("connect external middleware"); let policy = SandboxPolicy { diff --git a/crates/openshell-supervisor-middleware/src/service.rs b/crates/openshell-supervisor-middleware/src/service.rs deleted file mode 100644 index 1cb713f389..0000000000 --- a/crates/openshell-supervisor-middleware/src/service.rs +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; -use openshell_core::proto::{ - HttpRequestEvaluation, HttpRequestResult, MiddlewareManifest, ValidateConfigRequest, - ValidateConfigResponse, -}; -use tonic::{Request, Response, Status}; - -use crate::{builtins, safe_reason, validate_builtin_config}; - -#[derive(Debug, Default)] -pub struct InProcessMiddlewareService; - -#[tonic::async_trait] -impl SupervisorMiddleware for InProcessMiddlewareService { - async fn describe( - &self, - _request: Request<()>, - ) -> Result, Status> { - Ok(Response::new(MiddlewareManifest { - name: "openshell/in-process".into(), - service_version: env!("CARGO_PKG_VERSION").into(), - bindings: builtins::describe(), - })) - } - - async fn validate_config( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - let config = request.config.unwrap_or_default(); - let validation = validate_builtin_config(&request.binding_id, &config); - Ok(Response::new(match validation { - Ok(()) => ValidateConfigResponse { - valid: true, - reason: String::new(), - }, - Err(err) => ValidateConfigResponse { - valid: false, - reason: safe_reason(&err.to_string()), - }, - })) - } - - async fn evaluate_http_request( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - let result = builtins::evaluate_http_request(&request) - .map_err(|err| Status::invalid_argument(safe_reason(&err.to_string())))?; - Ok(Response::new(result)) - } -} diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 46430af58a..1449276f4f 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -51,6 +51,7 @@ webpki-roots = { workspace = true } [dev-dependencies] openshell-core = { path = "../openshell-core", features = ["test-helpers"] } +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } tempfile = "3" tonic = { workspace = true } temp-env = "0.3" diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 45f65280a0..96b1265de3 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2203,6 +2203,15 @@ mod tests { const TEST_POLICY: &str = include_str!("../../data/sandbox-policy.rego"); + fn install_builtin_middleware(engine: &OpaEngine) { + engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( + openshell_supervisor_middleware_builtins::services() + .into_iter() + .next() + .expect("built-in middleware service"), + )); + } + fn rest_token_grant_relay_context( resolver_response: std::result::Result<&str, &str>, ) -> ( @@ -2309,6 +2318,7 @@ network_policies: "# ); let engine = OpaEngine::from_strings(TEST_POLICY, &data).unwrap(); + install_builtin_middleware(&engine); let input = NetworkInput { host: "api.example.test".into(), port: 8080, @@ -3146,7 +3156,7 @@ network_policies: let resolved = ChainEntry { name: "redact".into(), - implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -3161,10 +3171,15 @@ network_policies: // A single unresolved (0-limit) entry must not drag the chain limit to // zero: the buffer limit reflects only the resolved built-in. - let mixed = ChainRunner::default() - .describe_chain(&[resolved, unresolved.clone()]) - .await - .expect("describe mixed chain"); + let mixed = ChainRunner::new( + openshell_supervisor_middleware_builtins::services() + .into_iter() + .next() + .expect("built-in middleware service"), + ) + .describe_chain(&[resolved, unresolved.clone()]) + .await + .expect("describe mixed chain"); assert_eq!(middleware_chain_body_limit(&mixed), Some(256 * 1024)); // When nothing resolves, there is no body limit and the caller skips @@ -4057,6 +4072,7 @@ network_policies: - { path: /usr/bin/curl } "#; let engine = Arc::new(OpaEngine::from_strings(TEST_POLICY, data).unwrap()); + install_builtin_middleware(engine.as_ref()); let generation_guard = engine .generation_guard(engine.current_generation()) .unwrap(); diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 72b0e96f90..0340d69500 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -29,16 +29,21 @@ const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; const MAX_SIGV4_BODY_BYTES: usize = 10 * 1024 * 1024; #[cfg(test)] async fn max_middleware_body_bytes() -> usize { - let chain = openshell_supervisor_middleware::ChainRunner::default() - .describe_chain(&[openshell_supervisor_middleware::ChainEntry { - name: "test".into(), - implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]) - .await - .expect("describe built-in middleware"); + let chain = openshell_supervisor_middleware::ChainRunner::new( + openshell_supervisor_middleware_builtins::services() + .into_iter() + .next() + .expect("built-in middleware service"), + ) + .describe_chain(&[openshell_supervisor_middleware::ChainEntry { + name: "test".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]) + .await + .expect("describe built-in middleware"); chain[0].max_body_bytes() } const RELAY_BUF_SIZE: usize = 8192; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index a6a73482fd..cc5c6d0298 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -26,6 +26,11 @@ use std::sync::{ /// passthroughs. They reference `data.sandbox.*` for policy data. const BAKED_POLICY_RULES: &str = include_str!("../data/sandbox-policy.rego"); +/// Implementation-owned middleware config validation supplied by the active +/// in-process catalog for local policy files. +pub type MiddlewareConfigValidator = + dyn Fn(&str, &prost_types::Struct) -> Result<(), String> + Send + Sync; + /// Result of evaluating a network access request against OPA policy. pub struct PolicyDecision { pub allowed: bool, @@ -156,6 +161,16 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_files(policy_path: &Path, data_path: &Path) -> Result { + Self::from_files_with_middleware_config(policy_path, data_path, None) + } + + /// Load local policy files and validate implementation-owned middleware + /// config through the catalog installed by the supervisor. + pub fn from_files_with_middleware_config( + policy_path: &Path, + data_path: &Path, + validate_middleware_config: Option<&MiddlewareConfigValidator>, + ) -> Result { let yaml_str = std::fs::read_to_string(data_path).map_err(|e| { miette::miette!("failed to read YAML data from {}: {e}", data_path.display()) })?; @@ -163,7 +178,7 @@ impl OpaEngine { engine .add_policy_from_file(policy_path) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(&yaml_str)?; + let data_json = preprocess_yaml_data(&yaml_str, validate_middleware_config)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -178,11 +193,19 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_strings(policy: &str, data_yaml: &str) -> Result { + Self::from_strings_with_middleware_config(policy, data_yaml, None) + } + + pub fn from_strings_with_middleware_config( + policy: &str, + data_yaml: &str, + validate_middleware_config: Option<&MiddlewareConfigValidator>, + ) -> Result { let mut engine = regorus::Engine::new(); engine .add_policy("policy.rego".into(), policy.into()) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(data_yaml)?; + let data_json = preprocess_yaml_data(data_yaml, validate_middleware_config)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -856,7 +879,10 @@ fn parse_process_policy(val: ®orus::Value) -> ProcessPolicy { } /// Preprocess YAML policy data: parse, normalize, validate, expand access presets, return JSON. -fn preprocess_yaml_data(yaml_str: &str) -> Result { +fn preprocess_yaml_data( + yaml_str: &str, + validate_middleware_config: Option<&MiddlewareConfigValidator>, +) -> Result { let mut data: serde_json::Value = serde_yml::from_str(yaml_str) .map_err(|e| miette::miette!("failed to parse YAML data: {e}"))?; @@ -871,7 +897,13 @@ fn preprocess_yaml_data(yaml_str: &str) -> Result { } // Validate BEFORE expanding presets (catches user errors like rules+access) - let middleware_errors = openshell_policy::validate_network_middleware_json(&data) + let middleware_errors = validate_middleware_config + .map_or_else( + || openshell_policy::validate_network_middleware_json(&data), + |validate| { + openshell_policy::validate_network_middleware_json_with_config(&data, validate) + }, + ) .map_err(|error| miette::miette!(error))?; if !middleware_errors.is_empty() { return Err(miette::miette!( @@ -6807,17 +6839,6 @@ network_middlewares: "#, "duplicate middleware config 'redactor'", ), - ( - "reserved builtin", - r#" -network_middlewares: - - name: sigv4 - middleware: openshell/sigv4 - endpoints: - include: ["api.example.com"] -"#, - "unsupported built-in", - ), ( "missing selector", r#" @@ -6890,6 +6911,50 @@ network_policies: } } + #[test] + fn middleware_catalog_validation_rejects_unknown_or_invalid_builtins() { + let validate = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + for (name, data, expected) in [ + ( + "unknown built-in", + r#" +network_middlewares: + - name: unknown + middleware: openshell/unknown + endpoints: + include: ["api.example.com"] +"#, + "not a registered OpenShell built-in", + ), + ( + "invalid secrets config", + r#" +network_middlewares: + - name: redactor + middleware: openshell/secrets + config: + secrets: allow + endpoints: + include: ["api.example.com"] +"#, + "supports only secrets: redact", + ), + ] { + let error = + OpaEngine::from_strings_with_middleware_config(TEST_POLICY, data, Some(&validate)) + .err() + .unwrap_or_else(|| panic!("{name}: expected catalog validation failure")) + .to_string(); + assert!( + error.contains(expected), + "{name}: expected {expected:?} in {error:?}" + ); + } + } + #[test] fn from_proto_revalidates_middleware_policy() { let mut policy = openshell_policy::restrictive_default_policy(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 0b9dbd9fd5..9dddbe1fd8 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4933,7 +4933,12 @@ network_policies: dynamic_credentials: None, token_grant_resolver: None, }; - let runner = openshell_supervisor_middleware::ChainRunner::default(); + let runner = openshell_supervisor_middleware::ChainRunner::new( + openshell_supervisor_middleware_builtins::services() + .into_iter() + .next() + .expect("built-in middleware service"), + ); let pipeline = ForwardMiddlewarePipeline { ctx: &ctx, scheme: "http", @@ -4947,7 +4952,7 @@ network_policies: }; let chain = vec![openshell_supervisor_middleware::ChainEntry { name: "redactor".into(), - implementation: openshell_supervisor_middleware::BUILTIN_SECRETS.into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, From 33501b3246beeb14b4e6facdcbe3ebb35353123b Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 10 Jul 2026 12:41:48 -0700 Subject: [PATCH 38/59] fix(middleware): commit policy and registry as one runtime generation The policy poll loop reconciled the middleware registry before the new OPA policy was committed, so concurrent requests could evaluate the old policy against the new registry. A failed policy reload also advanced the applied hash, freezing the mismatch and letting an unresolved fail-open guard be skipped indefinitely. Prepare the policy engine and middleware registry off to the side and swap both under the engine and runner locks with a single generation increment. A failure keeps the last-known-good pair and leaves the applied hash and service set unchanged, so the snapshot is retried on every poll without waiting for a config revision change. A policy-only change swaps the engine alone and reuses the connected registry: middleware configs are validated at gateway admission and manifests are already cached, so middleware reachability must not block an unrelated policy update. The registry is rebuilt, reconnecting every delivered service, only when the delivered service set changes or the installed registry is degraded. Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 15 +- crates/openshell-sandbox/src/lib.rs | 400 ++++++++++++++---- .../openshell-supervisor-network/src/opa.rs | 232 +++++++++- 3 files changed, 537 insertions(+), 110 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 3a8ff20b08..59c6b41ab2 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,8 +74,15 @@ and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. -Supervisors keep the last-known-good service registry when a live config reload -fails. The generic registry and chain runner live in +For a gateway policy snapshot, the supervisor prepares replacements off to the +side and installs them as one runtime generation. A policy-only change swaps +the policy engine alone and reuses the connected registry, so middleware +reachability cannot block it. The registry is rebuilt, reconnecting every +delivered service, only when the delivered service set changes or the installed +registry is degraded; that rebuild commits together with the policy engine. +A failure preserves the complete last-known-good runtime and leaves its applied +hash and registrations unchanged so the snapshot is retried. The generic +registry and chain runner live in `openshell-supervisor-middleware`; first-party implementations and their config schemas live in `openshell-supervisor-middleware-builtins`. The gateway and supervisor inject @@ -239,8 +246,8 @@ engine with a gateway policy revision. ## Failure Behavior - If gateway config polling fails, the sandbox keeps its last-known-good policy. -- If a live policy update is invalid, the supervisor rejects it and keeps the - current policy. +- If a live policy or middleware-registry update is invalid, the supervisor + rejects the combined update and keeps the current runtime pair. - If an operator-run middleware call fails, the selected config's `on_error` behavior decides whether to deny the request or continue without that stage. - Existing raw byte streams are connection scoped. Dynamic policy changes apply diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b6fbf4f87e..982de8b528 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1720,6 +1720,39 @@ enum MiddlewareRegistryStatus { NeedsReconciliation, } +/// True when the installed middleware registry no longer matches the desired +/// service set and must be rebuilt (reconnecting every delivered service). +/// +/// A policy-only change never requires a rebuild: middleware configs were +/// validated at gateway admission and the installed registry's manifests +/// already cover the unchanged service set, so requiring the services to be +/// reachable would only let a middleware outage block the policy update. +fn middleware_registry_needs_rebuild( + registry_status: MiddlewareRegistryStatus, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> bool { + registry_status == MiddlewareRegistryStatus::NeedsReconciliation + || current_services != desired_services +} + +fn gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy: bool, + current_policy_hash: &str, + desired_policy_hash: &str, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + registry_status: MiddlewareRegistryStatus, +) -> bool { + reloads_gateway_policy + && (current_policy_hash != desired_policy_hash + || middleware_registry_needs_rebuild( + registry_status, + current_services, + desired_services, + )) +} + /// Identity returned with the exact policy snapshot used to construct OPA. #[derive(Clone, Debug, PartialEq, Eq)] struct LoadedPolicyRevision { @@ -2009,6 +2042,16 @@ struct PolicyPollLoopContext { middleware_registry_status: MiddlewareRegistryStatus, } +async fn connect_middleware_registry( + services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> Result { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await +} + async fn reconcile_middleware_registry( opa_engine: &OpaEngine, desired_services: &[openshell_core::proto::SupervisorMiddlewareService], @@ -2021,12 +2064,9 @@ async fn reconcile_middleware_registry( return; } - match openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - desired_services.to_vec(), - ) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + match connect_middleware_registry(desired_services) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) { Ok(()) => { current_services.clear(); @@ -2090,6 +2130,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { String, openshell_core::proto::EffectiveSetting, > = std::collections::HashMap::new(); + let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); + let mut last_failed_runtime_revision: Option<(u64, String)> = None; // A first poll that does not match the policy already loaded into OPA must // pass through the normal reconciliation path immediately. It must never @@ -2149,41 +2191,59 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } }; - // Reconcile independently of the config revision so a recovered - // operator service is picked up without waiting for a policy change. - // Failure preserves the last-known-good registry and does not block - // the remaining config updates below. - reconcile_middleware_registry( - &ctx.opa_engine, + let config_changed = result.config_revision != current_config_revision; + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; + let policy_changed = result.policy_hash != current_policy_hash; + let middleware_registry_changed = middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, &result.supervisor_middleware_services, - &mut current_middleware_services, - &mut middleware_registry_status, - ) - .await; + ); + let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - if result.config_revision == current_config_revision && !provider_env_changed { - continue; + // A local policy override is not coupled to the gateway policy + // snapshot, so its service registry can still be reconciled alone. + // Gateway policy snapshots, however, must install policy and registry + // as one generation below. + if !reloads_gateway_policy { + reconcile_middleware_registry( + &ctx.opa_engine, + &result.supervisor_middleware_services, + &mut current_middleware_services, + &mut middleware_registry_status, + ) + .await; } - let policy_changed = result.policy_hash != current_policy_hash; + if !config_changed && !provider_env_changed && !policy_runtime_changed { + continue; + } - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); + if config_changed || provider_env_changed { + // Log which settings changed. + log_setting_changes(¤t_settings, &result.settings); - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "detected") + .unmapped("old_config_revision", serde_json::json!(current_config_revision)) + .unmapped("new_config_revision", serde_json::json!(result.config_revision)) + .unmapped("policy_changed", serde_json::json!(policy_changed)) + .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) + .message(format!( + "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", + result.config_revision + )) + .build()); + } if provider_env_changed { match openshell_core::grpc_client::fetch_provider_environment( @@ -2225,79 +2285,124 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } } - // Only reload OPA when the policy payload actually changed. - if policy_changed && ctx.loaded_policy_origin.allows_gateway_policy_reload() { - let Some(policy) = result.policy.as_ref() else { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "skipped") - .message("Settings poll: policy hash changed but no policy payload present; skipping reload") - .build()); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash; - current_settings = result.settings; - continue; + if policy_runtime_changed { + let pid = ctx.entrypoint_pid.load(Ordering::Acquire); + let runtime_result = match result.policy.as_ref() { + Some(policy) if middleware_registry_changed => { + match connect_middleware_registry(&result.supervisor_middleware_services).await + { + Ok(registry) => ctx + .opa_engine + .reload_policy_and_middleware_from_proto_with_pid( + policy, pid, registry, + ), + Err(error) => Err(error), + } + } + // Policy-only change: the installed registry already matches + // the delivered service set, so swap the engine alone. This + // must not require middleware reachability. + Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), + None => Err(miette::miette!( + "runtime reload requires a policy payload but none was returned" + )), }; - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - match ctx.opa_engine.reload_from_proto_with_pid(policy, pid) { + match runtime_result { Ok(()) => { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; + let policy = result + .policy + .as_ref() + .expect("successful runtime reload requires a policy payload"); + if policy_changed { + if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { + policy_local_ctx.set_current_policy(policy.clone()).await; + } + if result.global_policy_version > 0 { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .message(format!( + "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", + result.policy_hash, + result.global_policy_version + )) + .build()); + } else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + } + if result.version > 0 && result.policy_source == PolicySource::Sandbox { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + } } - if result.global_policy_version > 0 { + + if middleware_registry_changed { ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(result.supervisor_middleware_services.len()) + ) .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version + "Supervisor policy runtime reloaded atomically [service_count:{}]", + result.supervisor_middleware_services.len() )) .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); } + + current_policy_hash.clone_from(&result.policy_hash); + current_middleware_services.clone_from(&result.supervisor_middleware_services); + middleware_registry_status = MiddlewareRegistryStatus::Synchronized; + last_failed_runtime_revision = None; } Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(e.to_string())) - .message(format!( - "Policy reload failed, keeping last-known-good policy [version:{} error:{e}]", - result.version - )) - .build()); - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), - ); + let failed_revision = (result.config_revision, result.policy_hash.clone()); + if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(e.to_string())) + .message(format!( + "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", + result.version + )) + .build()); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, e.to_string()), + ); + } } + last_failed_runtime_revision = Some(failed_revision); + // Nothing was installed, so the registry status still + // describes the live registry. The retry is driven by the + // persisting hash/service-set mismatch (or an existing + // NeedsReconciliation), not by degrading the status here. } } } @@ -2340,7 +2445,9 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { } current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash; + if !reloads_gateway_policy { + current_policy_hash = result.policy_hash; + } current_settings = result.settings; } } @@ -2604,6 +2711,111 @@ filesystem_policy: } } + #[test] + fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { + let services = Vec::new(); + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + } + + #[test] + fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &no_services, + &no_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &no_services, + &desired_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + false, + "local-policy", + "hash-v2", + &no_services, + &desired_services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + } + + #[test] + fn policy_only_change_does_not_rebuild_middleware_registry() { + let services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + // The runtime must reconcile, but the registry (and therefore + // middleware reachability) is not part of that reconciliation. + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &services, + &services, + )); + } + + #[test] + fn registry_rebuild_requires_service_set_change_or_degraded_registry() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &no_services, + &desired_services, + )); + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::NeedsReconciliation, + &desired_services, + &desired_services, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &desired_services, + &desired_services, + )); + } + #[test] fn initial_ack_candidate_matches_sandbox_revision() { let canonical = settings_poll_result( diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index cc5c6d0298..1a2f4ff339 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -439,6 +439,41 @@ impl OpaEngine { Ok(()) } + /// Reload the policy and middleware registry as one runtime generation. + /// + /// Both replacements are prepared before the live locks are acquired. The + /// engine and runner are then swapped while holding both locks, followed by + /// a single generation increment. A preparation or lock failure leaves the + /// live pair and generation untouched. + pub fn reload_policy_and_middleware_from_proto_with_pid( + &self, + proto: &ProtoSandboxPolicy, + entrypoint_pid: u32, + registry: MiddlewareRegistry, + ) -> Result<()> { + let new = Self::from_proto_with_pid(proto, entrypoint_pid)?; + let new_engine = new + .engine + .into_inner() + .map_err(|_| miette::miette!("lock poisoned on new engine"))?; + let new_runner = ChainRunner::from_registry(registry); + + // Match clone_engine_for_tunnel's lock order (engine, then runner) so + // readers can observe only the old pair or the new pair. + let mut engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let mut runner = self + .middleware_runner + .write() + .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; + *engine = new_engine; + *runner = new_runner; + self.generation.fetch_add(1, Ordering::AcqRel); + Ok(()) + } + /// Current policy generation. Successful reloads increment this value. pub fn current_generation(&self) -> u64 { self.generation.load(Ordering::Acquire) @@ -1583,7 +1618,7 @@ mod tests { use openshell_core::proto::{ FilesystemPolicy as ProtoFs, L7Allow, L7QueryMatcher, L7Rule, NetworkBinary, - NetworkEndpoint, NetworkPolicyRule, ProcessPolicy as ProtoProc, + NetworkEndpoint, NetworkMiddlewareConfig, NetworkPolicyRule, ProcessPolicy as ProtoProc, SandboxPolicy as ProtoSandboxPolicy, }; @@ -6302,6 +6337,181 @@ network_policies: ); } + #[tokio::test] + async fn policy_and_middleware_reload_commit_as_one_generation() { + let proto = test_proto(); + let engine = OpaEngine::from_proto(&proto).expect("initial load should succeed"); + let mut new_proto = proto; + new_proto.network_policies.insert( + "python_api".to_string(), + NetworkPolicyRule { + name: "python_api".to_string(), + endpoints: vec![NetworkEndpoint { + host: "pypi.org".to_string(), + port: 443, + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/python3".to_string(), + ..Default::default() + }], + }, + ); + let registry = MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await + .expect("built-in registry"); + + engine + .reload_policy_and_middleware_from_proto_with_pid(&new_proto, 0, registry) + .expect("combined reload"); + + assert_eq!(engine.current_generation(), 1); + let python_input = NetworkInput { + host: "pypi.org".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + assert!(engine.evaluate_network(&python_input).unwrap().allowed); + + let entry = ChainEntry { + name: "secrets".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }; + let described = engine + .middleware_runner() + .expect("middleware runner") + .describe_chain(&[entry]) + .await + .expect("describe chain"); + assert!(described[0].is_resolved()); + } + + #[tokio::test] + async fn policy_only_reload_keeps_connected_middleware_registry() { + let proto = test_proto(); + let engine = OpaEngine::from_proto(&proto).expect("initial load should succeed"); + let registry = MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await + .expect("built-in registry"); + engine + .replace_middleware_registry(registry) + .expect("install registry"); + let generation_with_registry = engine.current_generation(); + + let mut new_proto = proto; + new_proto.network_policies.insert( + "python_api".to_string(), + NetworkPolicyRule { + name: "python_api".to_string(), + endpoints: vec![NetworkEndpoint { + host: "pypi.org".to_string(), + port: 443, + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/python3".to_string(), + ..Default::default() + }], + }, + ); + engine + .reload_from_proto_with_pid(&new_proto, 0) + .expect("policy-only reload"); + + assert_eq!(engine.current_generation(), generation_with_registry + 1); + let python_input = NetworkInput { + host: "pypi.org".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + assert!(engine.evaluate_network(&python_input).unwrap().allowed); + + let entry = ChainEntry { + name: "secrets".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }; + let described = engine + .middleware_runner() + .expect("middleware runner") + .describe_chain(&[entry]) + .await + .expect("describe chain"); + assert!(described[0].is_resolved()); + } + + #[tokio::test] + async fn failed_combined_reload_preserves_policy_registry_and_generation() { + let proto = test_proto(); + let engine = OpaEngine::from_proto(&proto).expect("initial load should succeed"); + let builtins = MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await + .expect("built-in registry"); + engine + .reload_policy_and_middleware_from_proto_with_pid(&proto, 0, builtins) + .expect("install last-known-good runtime"); + + let mut invalid = proto; + invalid.network_middlewares.push(NetworkMiddlewareConfig { + name: String::new(), + middleware: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + ..Default::default() + }); + let empty_registry = MiddlewareRegistry::connect_services(Vec::new(), Vec::new()) + .await + .expect("empty registry"); + + engine + .reload_policy_and_middleware_from_proto_with_pid(&invalid, 0, empty_registry) + .expect_err("invalid policy must reject the combined reload"); + + assert_eq!(engine.current_generation(), 1); + let claude_input = NetworkInput { + host: "api.anthropic.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/local/bin/claude"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + assert!(engine.evaluate_network(&claude_input).unwrap().allowed); + + let entry = ChainEntry { + name: "secrets".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }; + let described = engine + .middleware_runner() + .expect("middleware runner") + .describe_chain(&[entry]) + .await + .expect("describe chain"); + assert!(described[0].is_resolved()); + } + #[test] fn deny_reason_includes_symlink_hint() { // Verify the deny reason includes an actionable symlink hint @@ -6958,17 +7168,15 @@ network_middlewares: #[test] fn from_proto_revalidates_middleware_policy() { let mut policy = openshell_policy::restrictive_default_policy(); - policy - .network_middlewares - .push(openshell_core::proto::NetworkMiddlewareConfig { - name: "redactor".into(), - middleware: "openshell/secrets".into(), - endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { - include: vec!["api[.example.com".into()], - exclude: Vec::new(), - }), - ..Default::default() - }); + policy.network_middlewares.push(NetworkMiddlewareConfig { + name: "redactor".into(), + middleware: "openshell/secrets".into(), + endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { + include: vec!["api[.example.com".into()], + exclude: Vec::new(), + }), + ..Default::default() + }); let error = OpaEngine::from_proto(&policy) .err() From c66f9ea4ad5b7d29e1f3958377909678533f3bdf Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 10 Jul 2026 16:17:19 -0700 Subject: [PATCH 39/59] feat(middleware): support ordered header mutations Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 5 + .../src/secrets.rs | 2 +- .../src/headers.rs | 312 +++++++++++++++ .../src/lib.rs | 357 ++++++++++-------- .../src/l7/middleware.rs | 2 +- .../src/l7/relay.rs | 4 +- .../src/l7/rest.rs | 130 ++++++- docs/extensibility/supervisor-middleware.mdx | 14 +- proto/supervisor_middleware.proto | 47 ++- 9 files changed, 691 insertions(+), 182 deletions(-) create mode 100644 crates/openshell-supervisor-middleware/src/headers.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 59c6b41ab2..f5c760aba7 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -105,6 +105,11 @@ policy generation. Each replacement is reclassified and evaluated before the next stage runs. Hard-deny classification is shared by CONNECT relays, route-selected relays, and forward proxying; evaluator failures become structured fail-closed outcomes instead of escaping the middleware pipeline. +Each middleware stage can also return ordered header write and remove +operations. Rust validates and applies a stage atomically to the logical header +state before the next stage runs, then replays the validated operations against +the raw request before credential injection. Credential, routing, framing, and +hop-by-hop headers remain supervisor-owned and cannot be mutated. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs index 3deaf84992..b9c41f02f8 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs @@ -92,7 +92,7 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result Result> { + if mutations.len() > MAX_HEADER_MUTATIONS { + return Err(miette!( + "middleware returned too many header mutations: {} exceeds {MAX_HEADER_MUTATIONS}", + mutations.len() + )); + } + + let mut headers = existing_headers.to_vec(); + let mut mutation_bytes = 0usize; + for mutation in mutations { + match mutation.operation.as_ref() { + Some(header_mutation::Operation::Write(write)) => { + let name = validate_name(&write.name)?; + if is_connection_nominated(&headers, &name) { + return Err(miette!( + "middleware cannot mutate hop-by-hop header '{}'", + write.name + )); + } + if !name.starts_with("x-openshell-middleware-") { + return Err(miette!( + "middleware can only write request headers prefixed with \ + x-openshell-middleware- and cannot write '{}'", + write.name + )); + } + if !is_safe_value(&write.value) { + return Err(miette!( + "middleware cannot write header '{}' with an unsafe value", + write.name + )); + } + mutation_bytes = mutation_bytes + .saturating_add(name.len()) + .saturating_add(write.value.len()); + enforce_size_limit(mutation_bytes)?; + + let action = ExistingHeaderAction::try_from(write.on_existing) + .map_err(|_| miette!("middleware returned invalid on_existing action"))?; + if action == ExistingHeaderAction::Unspecified { + return Err(miette!( + "middleware must specify on_existing for header '{}'", + write.name + )); + } + let exists = headers.iter().any(|(existing, _)| *existing == name); + if !exists || action == ExistingHeaderAction::Append { + headers.push((name, write.value.clone())); + } else if action == ExistingHeaderAction::Overwrite { + headers.retain(|(existing, _)| *existing != name); + headers.push((name, write.value.clone())); + } else if action != ExistingHeaderAction::Skip { + return Err(miette!( + "middleware returned unsupported on_existing action" + )); + } + } + Some(header_mutation::Operation::Remove(remove)) => { + let name = validate_name(&remove.name)?; + if is_connection_nominated(&headers, &name) { + return Err(miette!( + "middleware cannot mutate hop-by-hop header '{}'", + remove.name + )); + } + mutation_bytes = mutation_bytes.saturating_add(name.len()); + enforce_size_limit(mutation_bytes)?; + headers.retain(|(existing, _)| *existing != name); + } + None => return Err(miette!("middleware returned an empty header mutation")), + } + } + Ok(headers) +} + +fn enforce_size_limit(mutation_bytes: usize) -> Result<()> { + if mutation_bytes > MAX_HEADER_MUTATION_BYTES { + return Err(miette!( + "middleware header mutations exceed {MAX_HEADER_MUTATION_BYTES} bytes" + )); + } + Ok(()) +} + +fn validate_name(name: &str) -> Result { + let lower = name.to_ascii_lowercase(); + if lower.is_empty() || !lower.bytes().all(is_name_token_byte) { + return Err(miette!("middleware returned invalid header name '{name}'")); + } + if is_protected(&lower) { + return Err(miette!( + "middleware cannot mutate protected header '{name}'" + )); + } + Ok(lower) +} + +fn is_name_token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) +} + +/// A header value is safe to write only if it contains no control characters. +/// Horizontal tab, printable ASCII, and obs-text (>= 0x80) are permitted; CR, LF, +/// NUL, and other control bytes are rejected. +fn is_safe_value(value: &str) -> bool { + value + .bytes() + .all(|byte| byte == b'\t' || (0x20..=0x7e).contains(&byte) || byte >= 0x80) +} + +fn is_protected(name: &str) -> bool { + matches!( + name, + "authorization" + | "proxy-authorization" + | "proxy-authenticate" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-connection" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) || name.starts_with("x-amz-") + || name.starts_with("x-openshell-credential") +} + +fn is_connection_nominated(headers: &[(String, String)], name: &str) -> bool { + headers + .iter() + .filter(|(header, _)| header == "connection") + .flat_map(|(_, value)| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case(name)) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{RemoveHeader, WriteHeader}; + + fn write(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { + HeaderMutation { + operation: Some(header_mutation::Operation::Write(WriteHeader { + name: name.into(), + value: value.into(), + on_existing: on_existing as i32, + })), + } + } + + fn remove(name: &str) -> HeaderMutation { + HeaderMutation { + operation: Some(header_mutation::Operation::Remove(RemoveHeader { + name: name.into(), + })), + } + } + + #[test] + fn protected_header_write_is_rejected() { + let error = apply( + &[], + &[write( + "Authorization", + "Bearer nope", + ExistingHeaderAction::Overwrite, + )], + ) + .expect_err("protected header"); + assert!( + error + .to_string() + .contains("protected header 'Authorization'") + ); + } + + #[test] + fn unsafe_header_value_is_rejected() { + let error = apply( + &[], + &[write( + "x-openshell-middleware-inject", + "ok\r\nAuthorization: Bearer evil", + ExistingHeaderAction::Append, + )], + ) + .expect_err("CRLF value"); + assert!(error.to_string().contains("unsafe value")); + } + + #[test] + fn existing_header_write_obeys_collision_action() { + let existing = [ + ("x-openshell-middleware-tag".to_string(), "one".to_string()), + ("accept".to_string(), "application/json".to_string()), + ]; + let appended = apply( + &existing, + &[write( + "X-OpenShell-Middleware-Tag", + "two", + ExistingHeaderAction::Append, + )], + ) + .expect("append existing header"); + assert_eq!( + appended, + vec![ + ("x-openshell-middleware-tag".into(), "one".into()), + ("accept".into(), "application/json".into()), + ("x-openshell-middleware-tag".into(), "two".into()), + ] + ); + + let overwritten = apply( + &existing, + &[write( + "X-OpenShell-Middleware-Tag", + "two", + ExistingHeaderAction::Overwrite, + )], + ) + .expect("overwrite existing header"); + assert_eq!( + overwritten, + vec![ + ("accept".into(), "application/json".into()), + ("x-openshell-middleware-tag".into(), "two".into()), + ] + ); + + let skipped = apply( + &existing, + &[write( + "X-OpenShell-Middleware-Tag", + "two", + ExistingHeaderAction::Skip, + )], + ) + .expect("skip existing header"); + assert_eq!(skipped, existing); + } + + #[test] + fn remove_drops_every_case_insensitive_value() { + let existing = [ + ("x-trace".to_string(), "one".to_string()), + ("accept".to_string(), "application/json".to_string()), + ("x-trace".to_string(), "two".to_string()), + ]; + let updated = apply(&existing, &[remove("X-Trace")]).expect("remove visible header"); + assert_eq!(updated, vec![("accept".into(), "application/json".into())]); + } + + #[test] + fn protected_header_remove_is_rejected_even_when_not_visible() { + let error = apply(&[], &[remove("Authorization")]).expect_err("protected removal"); + assert!( + error + .to_string() + .contains("protected header 'Authorization'") + ); + } + + #[test] + fn connection_nominated_header_is_protected() { + let existing = [ + ("connection".to_string(), "keep-alive, x-hop".to_string()), + ("x-hop".to_string(), "value".to_string()), + ]; + let error = apply(&existing, &[remove("X-Hop")]).expect_err("hop-by-hop removal"); + assert!(error.to_string().contains("hop-by-hop header 'X-Hop'")); + } +} diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 2a95d698d8..e6fd8ae7a5 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -3,17 +3,20 @@ //! Supervisor middleware registration and chain execution. +mod headers; mod remote; -use std::collections::{BTreeMap, HashMap, HashSet}; +#[cfg(test)] +use std::collections::HashMap; +use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; use miette::{Result, miette}; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ - Decision, Finding, HttpHeader, HttpRequestEvaluation, HttpRequestTarget, MiddlewareBinding, - MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, + Decision, Finding, HeaderMutation, HttpHeader, HttpRequestEvaluation, HttpRequestTarget, + MiddlewareBinding, MiddlewareManifest, NetworkMiddlewareConfig, RequestContext, SandboxPolicy, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, SupervisorMiddlewareService, ValidateConfigRequest, }; @@ -146,7 +149,8 @@ pub struct ChainOutcome { pub allowed: bool, pub reason: String, pub body: Vec, - pub added_headers: BTreeMap, + /// Ordered, validated mutations to replay against the original raw request. + pub header_mutations: Vec, pub findings: Vec, pub metadata: BTreeMap>, pub applied: Vec, @@ -690,7 +694,7 @@ impl ChainRunner { ) -> Result { let mut headers = input.headers.clone(); let mut body = input.body.clone(); - let mut added_headers = BTreeMap::new(); + let mut header_mutations = Vec::new(); let mut findings = Vec::new(); let mut metadata = BTreeMap::new(); let mut applied = Vec::new(); @@ -704,7 +708,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -720,7 +724,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -746,7 +750,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -766,7 +770,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -800,7 +804,7 @@ impl ChainRunner { allowed: false, reason: safe_reason(&result.reason), body, - added_headers, + header_mutations, findings, metadata, applied, @@ -815,7 +819,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -824,32 +828,34 @@ impl ChainRunner { } } - // A result proposing unsafe header mutations is a malformed response: - // route it through `on_error` instead of applying any of it. The - // validation error names the offending header and the required - // x-openshell-middleware- prefix so operators can fix the service. - if let Err(error) = validate_header_mutations(&headers, &result.add_headers) { - match apply_on_error(entry, &safe_reason(&error.to_string()), &mut applied) { - OnErrorAction::FailOpen => continue, - OnErrorAction::FailClosed(reason) => { - return Ok(ChainOutcome { - allowed: false, - reason, - body, - added_headers, - findings, - metadata, - applied, - }); + // Validate and apply the entire stage atomically. Under fail-open, + // one malformed mutation must not leave earlier mutations from the + // same response visible to later middleware. + let updated_headers = match headers::apply(&headers, &result.header_mutations) { + Ok(updated) => updated, + Err(error) => { + match apply_on_error(entry, &safe_reason(&error.to_string()), &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + header_mutations, + findings, + metadata, + applied, + }); + } } } - } - for (name, value) in &result.add_headers { - headers.push((name.to_ascii_lowercase(), value.clone())); - added_headers.insert(name.to_ascii_lowercase(), value.clone()); - } - let transformed = result.has_body; - if result.has_body { + }; + let headers_transformed = updated_headers != headers; + headers = updated_headers; + header_mutations.extend(result.header_mutations.iter().cloned()); + + let body_transformed = result.has_body; + if body_transformed { result.body.clone_into(&mut body); } for finding in result.findings { @@ -868,7 +874,7 @@ impl ChainRunner { name: entry.entry.name.clone(), implementation: entry.entry.implementation.clone(), decision, - transformed, + transformed: body_transformed || headers_transformed, failed: false, }); @@ -876,7 +882,7 @@ impl ChainRunner { // sandbox policy the original body was admitted under. Re-check now, // before the next stage or the upstream sees the replaced body. A // policy deny here is a hard deny, independent of `on_error`. - if transformed + if body_transformed && let TransformedBodyPolicy::Reevaluate(validate) = transformed_body_policy { let denied = match validate(&body) { @@ -891,7 +897,7 @@ impl ChainRunner { allowed: false, reason, body, - added_headers, + header_mutations, findings, metadata, applied, @@ -904,7 +910,7 @@ impl ChainRunner { allowed: true, reason: String::new(), body, - added_headers, + header_mutations, findings, metadata, applied, @@ -956,61 +962,6 @@ fn build_evaluation( } } -fn validate_header_mutations( - existing_headers: &[(String, String)], - mutations: &HashMap, -) -> Result<()> { - let mut seen = HashSet::new(); - for (name, value) in mutations { - let lower = name.to_ascii_lowercase(); - if !seen.insert(lower.clone()) || existing_headers.iter().any(|(name, _)| *name == lower) { - return Err(miette!( - "middleware cannot rewrite existing header '{name}'" - )); - } - if !is_safe_append_header(&lower) { - return Err(miette!( - "middleware can only append new request headers prefixed with \ - x-openshell-middleware- and cannot append '{name}'" - )); - } - // Reject CR/LF and other control characters in the value: writing them - // verbatim into the upstream header block would enable header injection - // and request smuggling past the credential boundary. - if !is_safe_header_value(value) { - return Err(miette!( - "middleware cannot append header '{name}' with an unsafe value" - )); - } - } - Ok(()) -} - -/// A header value is safe to append only if it contains no control characters. -/// Horizontal tab, printable ASCII, and obs-text (>= 0x80) are permitted; CR, LF, -/// NUL, and other control bytes are rejected. -fn is_safe_header_value(value: &str) -> bool { - value - .bytes() - .all(|b| b == b'\t' || (0x20..=0x7e).contains(&b) || b >= 0x80) -} - -fn is_safe_append_header(name: &str) -> bool { - if name.is_empty() - || name.contains(':') - || name.bytes().any(|b| b <= 0x20 || b >= 0x7f) - || matches!( - name, - "authorization" | "cookie" | "host" | "content-length" | "transfer-encoding" - ) - || name.starts_with("x-amz-") - || name.starts_with("x-openshell-credential") - { - return false; - } - name.starts_with("x-openshell-middleware-") -} - pub(crate) fn safe_reason(reason: &str) -> String { reason .chars() @@ -1025,6 +976,7 @@ mod tests { use openshell_core::proto::middleware::v1::supervisor_middleware_server::{ SupervisorMiddleware, SupervisorMiddlewareServer, }; + use openshell_core::proto::{ExistingHeaderAction, header_mutation}; use openshell_supervisor_middleware_builtins::{BUILTIN_SECRETS, services}; use tokio_stream::wrappers::TcpListenerStream; @@ -1070,6 +1022,18 @@ mod tests { } } + fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { + HeaderMutation { + operation: Some(header_mutation::Operation::Write( + openshell_core::proto::WriteHeader { + name: name.into(), + value: value.into(), + on_existing: on_existing as i32, + }, + )), + } + } + #[tokio::test] async fn phase_one_evaluation_omits_originating_process() { let entries = builtin_runner() @@ -1234,33 +1198,6 @@ mod tests { assert!(error.to_string().contains("more than one service")); } - #[test] - fn unsafe_header_mutation_is_rejected() { - let err = validate_header_mutations( - &[], - &std::iter::once(("Authorization".into(), "Bearer nope".into())).collect(), - ) - .expect_err("unsafe header"); - assert!(err.to_string().contains("x-openshell-middleware-")); - assert!(err.to_string().contains("'Authorization'")); - } - - #[test] - fn header_value_with_crlf_is_rejected() { - // A safe header *name* with a CRLF-bearing value must still be rejected, - // otherwise it would inject extra headers into the upstream request. - let err = validate_header_mutations( - &[], - &std::iter::once(( - "x-openshell-middleware-inject".into(), - "ok\r\nAuthorization: Bearer evil".into(), - )) - .collect(), - ) - .expect_err("crlf value"); - assert!(err.to_string().contains("unsafe value")); - } - /// A mock middleware that returns a fixed, caller-supplied result for every /// evaluation. Used to exercise chain behavior the built-in cannot produce /// (explicit deny, metadata, findings, unsafe header mutations). @@ -1540,7 +1477,7 @@ mod tests { reason: String::new(), body: Vec::new(), has_body: false, - add_headers: HashMap::new(), + header_mutations: Vec::new(), findings: Vec::new(), metadata: HashMap::new(), } @@ -1600,6 +1537,130 @@ mod tests { } } + /// Three-stage service used to verify that each stage observes the header + /// state produced by all preceding stages. + struct HeaderChainService { + second_action: ExistingHeaderAction, + received: std::sync::Mutex>, + } + + #[tonic::async_trait] + impl SupervisorMiddleware for HeaderChainService { + async fn describe( + &self, + _request: Request<()>, + ) -> std::result::Result, tonic::Status> { + Ok(tonic::Response::new(MiddlewareManifest { + name: "test/header-chain".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/header-chain".into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: 4096, + }], + })) + } + + async fn validate_config( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + let evaluation = request.into_inner(); + let invocation = { + let mut received = self.received.lock().expect("header chain lock"); + let invocation = received.len(); + received.push(evaluation); + invocation + }; + let mut result = allow_result(); + if invocation == 0 { + result.header_mutations.push(write_header( + "x-openshell-middleware-chain", + "first", + ExistingHeaderAction::Overwrite, + )); + } else if invocation == 1 { + result.header_mutations.push(write_header( + "x-openshell-middleware-chain", + "second", + self.second_action, + )); + } + Ok(tonic::Response::new(result)) + } + } + + #[tokio::test] + async fn later_middleware_observes_prior_header_mutations() { + for (action, expected) in [ + (ExistingHeaderAction::Append, vec!["first", "second"]), + (ExistingHeaderAction::Overwrite, vec!["second"]), + (ExistingHeaderAction::Skip, vec!["first"]), + ] { + let service = Arc::new(HeaderChainService { + second_action: action, + received: std::sync::Mutex::new(Vec::new()), + }); + let runner = ChainRunner::new(service.clone()); + let entries = [ + ChainEntry { + name: "first".into(), + implementation: "example/header-chain".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ChainEntry { + name: "second".into(), + implementation: "example/header-chain".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ChainEntry { + name: "observer".into(), + implementation: "example/header-chain".into(), + order: 20, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ]; + + let outcome = runner + .evaluate(&entries, input("payload")) + .await + .expect("evaluate header chain"); + assert!(outcome.allowed); + let received = service.received.lock().expect("recorded header chain"); + let observed: Vec<&str> = received[2] + .headers + .iter() + .filter(|header| header.name == "x-openshell-middleware-chain") + .map(|header| header.value.as_str()) + .collect(); + assert_eq!(observed, expected, "action {action:?}"); + } + } + #[tokio::test] async fn repeated_request_headers_reach_middleware_in_wire_order() { // A map contract would collapse repeated header names to one value @@ -1648,22 +1709,6 @@ mod tests { ); } - #[test] - fn existing_header_rewrite_is_rejected() { - // Any occurrence of an existing name blocks the mutation, including - // repeated names where only one entry matches. - let existing = [ - ("x-openshell-middleware-tag".to_string(), "one".to_string()), - ("accept".to_string(), "application/json".to_string()), - ]; - let err = validate_header_mutations( - &existing, - &std::iter::once(("X-OpenShell-Middleware-Tag".into(), "two".into())).collect(), - ) - .expect_err("rewrite of existing header"); - assert!(err.to_string().contains("cannot rewrite existing header")); - } - fn external_registration(max_body_bytes: u64) -> SupervisorMiddlewareService { SupervisorMiddlewareService { name: "local-guard-service".into(), @@ -2066,11 +2111,11 @@ mod tests { openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, reason: "blocked_by_policy".into(), - add_headers: std::iter::once(( - "x-openshell-middleware-inject".to_string(), - "ok\r\nHost: evil".to_string(), - )) - .collect(), + header_mutations: vec![write_header( + "x-openshell-middleware-inject", + "ok\r\nHost: evil", + ExistingHeaderAction::Append, + )], ..allow_result() }, ))); @@ -2082,7 +2127,7 @@ mod tests { assert!(!outcome.allowed); assert_eq!(outcome.reason, "blocked_by_policy"); - assert!(outcome.added_headers.is_empty()); + assert!(outcome.header_mutations.is_empty()); assert_eq!(outcome.applied.len(), 1); assert_eq!(outcome.applied[0].decision, Decision::Deny); assert!(!outcome.applied[0].failed); @@ -2157,11 +2202,18 @@ mod tests { fn unsafe_header_service() -> ScriptedService { scripted_service(openshell_core::proto::HttpRequestResult { - add_headers: std::iter::once(( - "x-openshell-middleware-inject".to_string(), - "ok\r\nHost: evil".to_string(), - )) - .collect(), + header_mutations: vec![ + write_header( + "x-openshell-middleware-safe", + "safe", + ExistingHeaderAction::Append, + ), + write_header( + "x-openshell-middleware-inject", + "ok\r\nHost: evil", + ExistingHeaderAction::Append, + ), + ], ..allow_result() }) } @@ -2183,8 +2235,9 @@ mod tests { outcome.reason ); assert!(outcome.applied.iter().any(|inv| inv.failed)); - // The unsafe header is never forwarded. - assert!(outcome.added_headers.is_empty()); + // The stage is atomic: neither the unsafe mutation nor the safe + // mutation preceding it is forwarded. + assert!(outcome.header_mutations.is_empty()); } #[tokio::test] @@ -2196,7 +2249,7 @@ mod tests { .expect("evaluate"); assert!(outcome.allowed); assert_eq!(outcome.body, b"hello"); - assert!(outcome.added_headers.is_empty()); + assert!(outcome.header_mutations.is_empty()); assert_eq!(outcome.applied.len(), 1); assert!(outcome.applied[0].failed); } diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index b0e236fd1e..611fa81cc8 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -182,7 +182,7 @@ pub async fn apply_middleware_chain_for_scheme, + header_mutations: &[HeaderMutation], ) -> Result { let mut header_bytes = set_content_length(headers, body.len())?; header_bytes = strip_header(&header_bytes, "transfer-encoding")?; - header_bytes = append_headers(&header_bytes, add_headers); + header_bytes = apply_header_mutations(&header_bytes, header_mutations)?; header_bytes.extend_from_slice(body); Ok(L7Request { action: req.action.clone(), @@ -1403,29 +1404,61 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { Ok(out.into_bytes()) } -fn append_headers( - headers: &[u8], - add_headers: &std::collections::BTreeMap, -) -> Vec { - if add_headers.is_empty() { - return headers.to_vec(); - } +fn append_header(headers: &[u8], name: &str, value: &str) -> Vec { let split = headers .windows(4) .position(|w| w == b"\r\n\r\n") .map_or(headers.len(), |pos| pos); - let mut out = Vec::with_capacity(headers.len() + add_headers.len() * 32); + let mut out = Vec::with_capacity(headers.len() + name.len() + value.len() + 4); out.extend_from_slice(&headers[..split]); - for (name, value) in add_headers { - out.extend_from_slice(b"\r\n"); - out.extend_from_slice(name.as_bytes()); - out.extend_from_slice(b": "); - out.extend_from_slice(value.as_bytes()); - } + out.extend_from_slice(b"\r\n"); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(b": "); + out.extend_from_slice(value.as_bytes()); out.extend_from_slice(b"\r\n\r\n"); out } +fn apply_header_mutations(headers: &[u8], mutations: &[HeaderMutation]) -> Result> { + let mut out = headers.to_vec(); + for mutation in mutations { + match mutation.operation.as_ref() { + Some(header_mutation::Operation::Write(write)) => { + let action = ExistingHeaderAction::try_from(write.on_existing) + .map_err(|_| miette!("invalid middleware header on_existing action"))?; + if action == ExistingHeaderAction::Unspecified { + return Err(miette!( + "middleware header mutation has unspecified on_existing action" + )); + } + let exists = has_header(&out, &write.name)?; + if !exists || action == ExistingHeaderAction::Append { + out = append_header(&out, &write.name, &write.value); + } else if action == ExistingHeaderAction::Overwrite { + out = strip_header(&out, &write.name)?; + out = append_header(&out, &write.name, &write.value); + } else if action != ExistingHeaderAction::Skip { + return Err(miette!("unsupported middleware header on_existing action")); + } + } + Some(header_mutation::Operation::Remove(remove)) => { + out = strip_header(&out, &remove.name)?; + } + None => return Err(miette!("empty middleware header mutation")), + } + } + Ok(out) +} + +fn has_header(headers: &[u8], name: &str) -> Result { + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + Ok(header_str.lines().skip(1).any(|line| { + line.split_once(':') + .is_some_and(|(candidate, _)| candidate.trim().eq_ignore_ascii_case(name)) + })) +} + pub(crate) fn request_is_websocket_upgrade(raw_header: &[u8]) -> bool { let header_end = raw_header .windows(4) @@ -2687,6 +2720,71 @@ mod tests { const VALID_WS_ACCEPT: &str = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="; const TEXT_OPCODE: u8 = 0x1; + fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { + HeaderMutation { + operation: Some(header_mutation::Operation::Write( + openshell_core::proto::WriteHeader { + name: name.into(), + value: value.into(), + on_existing: on_existing as i32, + }, + )), + } + } + + fn remove_header(name: &str) -> HeaderMutation { + HeaderMutation { + operation: Some(header_mutation::Operation::Remove( + openshell_core::proto::RemoveHeader { name: name.into() }, + )), + } + } + + #[test] + fn ordered_header_mutations_replay_against_raw_request() { + let raw = b"GET / HTTP/1.1\r\nHost: example.test\r\nX-OpenShell-Middleware-Chain: first\r\nX-Drop: one\r\nX-Drop: two\r\n\r\n"; + let mutations = [ + write_header( + "x-openshell-middleware-chain", + "second", + ExistingHeaderAction::Append, + ), + write_header( + "x-openshell-middleware-chain", + "ignored", + ExistingHeaderAction::Skip, + ), + write_header( + "x-openshell-middleware-chain", + "replacement", + ExistingHeaderAction::Overwrite, + ), + write_header( + "x-openshell-middleware-chain", + "tail", + ExistingHeaderAction::Append, + ), + remove_header("x-drop"), + ]; + + let updated = String::from_utf8( + apply_header_mutations(raw, &mutations).expect("apply ordered header mutations"), + ) + .expect("UTF-8 request"); + let values: Vec<&str> = updated + .lines() + .filter_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("x-openshell-middleware-chain") + .then_some(value.trim()) + }) + }) + .collect(); + assert_eq!(values, vec!["replacement", "tail"]); + assert!(!updated.to_ascii_lowercase().contains("x-drop:")); + assert!(updated.contains("Host: example.test")); + } + #[derive(Debug)] struct CapturedFrame { fin_opcode: u8, diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 1678042f88..473984c99e 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -111,11 +111,19 @@ Every middleware binding declares the largest request or replacement body it sup The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. A body larger than every stage limit cannot be inspected at all and is denied unless every selected stage is `fail_open`. -## Add Request Headers +## Mutate Request Headers -A middleware result can add new request headers before OpenShell injects credentials. Header names are case-insensitive and must start with `x-openshell-middleware-`. Values must not contain control characters. +A middleware result can return ordered header mutations before OpenShell injects credentials. A `write` mutation adds a value when the case-insensitive header name is absent and selects one behavior when it is already present: -A result that rewrites an existing request header, uses a name outside the required prefix, or carries an unsafe value is treated as a middleware failure and follows that config's `on_error` behavior. The failure reason names the offending header. Protected headers, including `authorization`, `cookie`, `host`, `content-length`, `transfer-encoding`, and OpenShell credential and signature headers, are always rejected. +- `append` adds another field value. +- `overwrite` removes every existing value before adding the new value. +- `skip` leaves existing values unchanged. + +A `remove` mutation removes every value for a case-insensitive header name. OpenShell applies each successful stage's mutations before invoking the next middleware, so later stages observe the accumulated header state. + +Header writes must use the `x-openshell-middleware-` prefix. Removes may target other middleware-visible request headers. Protected credential, routing, framing, and hop-by-hop headers are always rejected. Header values must not contain control characters. + +OpenShell validates and applies each stage's mutations atomically. An invalid operation discards every mutation from that stage and follows its `on_error` behavior. The failure reason names the offending header. ## Operate Middleware Services diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index fdc7f03c4b..2614bf8a3b 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -164,6 +164,39 @@ message Finding { string severity = 5; } +// ExistingHeaderAction controls how a header write behaves when the +// case-insensitive header name is already present. Every action writes the +// value when the header is absent. +enum ExistingHeaderAction { + EXISTING_HEADER_ACTION_UNSPECIFIED = 0; + // Add another field value without changing existing values. + EXISTING_HEADER_ACTION_APPEND = 1; + // Remove every existing value, then add the new value. + EXISTING_HEADER_ACTION_OVERWRITE = 2; + // Leave the existing values unchanged. + EXISTING_HEADER_ACTION_SKIP = 3; +} + +// WriteHeader proposes one header value and defines collision behavior. +message WriteHeader { + string name = 1; + string value = 2; + ExistingHeaderAction on_existing = 3; +} + +// RemoveHeader removes every value for a case-insensitive header name. +message RemoveHeader { + string name = 1; +} + +// HeaderMutation is one ordered request-header operation. +message HeaderMutation { + oneof operation { + WriteHeader write = 1; + RemoveHeader remove = 2; + } +} + // HttpRequestResult contains the decision and optional request mutations. message HttpRequestResult { // Allow or deny decision for this request. @@ -176,13 +209,13 @@ message HttpRequestResult { bytes body = 3; // True when body should replace the request body, including with an empty body. bool has_body = 4; - // Request headers to add before forwarding. Only new headers whose - // case-insensitive names start with "x-openshell-middleware-" are accepted. - // Existing request headers cannot be rewritten, protected headers such as - // authorization or transfer-encoding are always rejected, and values must - // not contain control characters. A violating result is a middleware - // failure handled according to the policy failure mode. - map add_headers = 5; + // Ordered request-header mutations applied before the next middleware and + // before forwarding. Header writes are restricted to the + // "x-openshell-middleware-" namespace. Removes may target other visible + // request headers, but credential, routing, framing, and hop-by-hop headers + // are always protected. A violating result is a middleware failure handled + // according to the policy failure mode. + repeated HeaderMutation header_mutations = 5; // Audit-safe findings produced during evaluation. repeated Finding findings = 6; // Non-secret service-defined metadata included in diagnostics. From 2757672afd6321002eedba8ed1fe3c10853b7be7 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Mon, 13 Jul 2026 16:44:30 -0700 Subject: [PATCH 40/59] fix(middleware): distinguish body buffering failures Signed-off-by: Piotr Mlocek --- .../src/l7/rest.rs | 286 +++++++++++++++--- 1 file changed, 247 insertions(+), 39 deletions(-) diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 51fd7192d1..4f24ccbaa5 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -818,12 +818,14 @@ struct PreparedRequestBody { body: Vec, } +#[derive(Debug)] pub(crate) struct BufferedRequestBody { pub(crate) headers: Vec, pub(crate) body: Vec, } /// Result of attempting to buffer a request body for middleware inspection. +#[derive(Debug)] pub(crate) enum BufferResult { /// The full body was buffered within the size cap. Buffered(BufferedRequestBody), @@ -849,10 +851,23 @@ pub(crate) async fn buffer_request_body_for_middleware Ok(BufferResult::Buffered(BufferedRequestBody { - headers, - body: already_read.to_vec(), - })), + BodyLength::None => { + // Leftover bytes after `\r\n\r\n` on a no-body request are + // read-ahead or pipelined data, not a framed body. Promoting them + // into `body` would later stamp `Content-Length` and forward them + // upstream as if they belonged to this request. + if !already_read.is_empty() { + return Err(miette!( + "HTTP request with no body framing has {} unread byte(s) after headers; \ + refusing to treat read-ahead bytes as a request body", + already_read.len() + )); + } + Ok(BufferResult::Buffered(BufferedRequestBody { + headers, + body: Vec::new(), + })) + } BodyLength::ContentLength(len) => { // The declared length is known before any further reads, so an // over-cap body here has not consumed the stream and can be passed @@ -900,13 +915,18 @@ pub(crate) async fn buffer_request_body_for_middleware Ok(BufferResult::Buffered(BufferedRequestBody { + headers, + body, + })), + Err(CollectChunkedError::OverCapacity) => { + Ok(BufferResult::OverCapacity { recoverable: false }) + } + Err(CollectChunkedError::Failed(error)) => Err(error), + } } } } @@ -936,6 +956,23 @@ pub(crate) fn rebuild_request_with_buffered_body( body: &[u8], header_mutations: &[HeaderMutation], ) -> Result { + // Preserve no-body framing when middleware did not introduce a body. + // Stamping `Content-Length: 0` onto GET/HEAD-style requests is unnecessary + // and would mask the BodyLength::None overshoot bug if leftovers were ever + // accepted as an empty-looking rewrite. + if matches!(req.body_length, BodyLength::None) && body.is_empty() { + let mut header_bytes = strip_header(headers, "content-length")?; + header_bytes = strip_header(&header_bytes, "transfer-encoding")?; + header_bytes = apply_header_mutations(&header_bytes, header_mutations)?; + return Ok(L7Request { + action: req.action.clone(), + target: req.target.clone(), + query_params: req.query_params.clone(), + raw_header: header_bytes, + body_length: BodyLength::None, + }); + } + let mut header_bytes = set_content_length(headers, body.len())?; header_bytes = strip_header(&header_bytes, "transfer-encoding")?; header_bytes = apply_header_mutations(&header_bytes, header_mutations)?; @@ -1002,7 +1039,9 @@ async fn collect_and_rewrite_request_body( Ok(PreparedRequestBody { headers, body }) } BodyLength::Chunked => { - let body = collect_chunked_body(client, already_read, generation_guard, None).await?; + let body = collect_chunked_body(client, already_read, generation_guard, None) + .await + .map_err(CollectChunkedError::into_report)?; let (mut headers, body) = rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; headers = set_content_length(&headers, body.len())?; @@ -1170,12 +1209,41 @@ fn hex_value(byte: u8) -> Option { } } +/// Outcome of collecting a chunked request body for buffering. +#[derive(Debug)] +enum CollectChunkedError { + /// The caller-supplied wire/decoded cap was exceeded. Bytes may already + /// have been consumed from the client stream, so fail-open streaming is + /// unsafe. + OverCapacity, + /// Protocol, I/O, or policy-generation failure. Not an over-capacity event. + Failed(miette::Error), +} + +impl CollectChunkedError { + fn context(self, f: impl FnOnce(miette::Error) -> miette::Error) -> Self { + match self { + Self::OverCapacity => Self::OverCapacity, + Self::Failed(error) => Self::Failed(f(error)), + } + } + + fn into_report(self) -> miette::Error { + match self { + Self::OverCapacity => { + miette!("chunked body wire representation exceeds middleware buffer limit") + } + Self::Failed(error) => error, + } + } +} + async fn collect_chunked_body( client: &mut C, already_read: &[u8], generation_guard: Option<&PolicyGenerationGuard>, max_wire_bytes: Option, -) -> Result> { +) -> std::result::Result, CollectChunkedError> { let max_decoded_bytes = max_wire_bytes.unwrap_or(MAX_REWRITE_BODY_BYTES); let mut read_state = ChunkedReadState { buffered_pos: 0, @@ -1187,26 +1255,30 @@ async fn collect_chunked_body( loop { let size_line = read_chunked_line(client, already_read, &mut read_state, generation_guard) .await - .map_err(|e| miette!("Chunked body ended before chunk-size line: {e}"))?; - let size_line = std::str::from_utf8(&size_line) - .into_diagnostic() - .map_err(|_| miette!("Invalid UTF-8 in chunk-size line"))?; + .map_err(|error| { + error.context(|e| miette!("Chunked body ended before chunk-size line: {e}")) + })?; + let size_line = std::str::from_utf8(&size_line).map_err(|_| { + CollectChunkedError::Failed(miette!("Invalid UTF-8 in chunk-size line")) + })?; let size_token = size_line .split(';') .next() .map(str::trim) .unwrap_or_default(); - let chunk_size = usize::from_str_radix(size_token, 16) - .into_diagnostic() - .map_err(|_| miette!("Invalid chunk size token: {size_token:?}"))?; + let chunk_size = usize::from_str_radix(size_token, 16).map_err(|_| { + CollectChunkedError::Failed(miette!("Invalid chunk size token: {size_token:?}")) + })?; if chunk_size == 0 { loop { let trailer_line = read_chunked_line(client, already_read, &mut read_state, generation_guard) .await - .map_err(|e| { - miette!("Chunked body ended before trailer terminator: {e}") + .map_err(|error| { + error.context(|e| { + miette!("Chunked body ended before trailer terminator: {e}") + }) })?; if trailer_line.is_empty() { return Ok(body); @@ -1215,9 +1287,13 @@ async fn collect_chunked_body( } if body.len().saturating_add(chunk_size) > max_decoded_bytes { - return Err(miette!( - "decoded chunked body exceeds {max_decoded_bytes} byte buffer limit" - )); + return Err(if max_wire_bytes.is_some() { + CollectChunkedError::OverCapacity + } else { + CollectChunkedError::Failed(miette!( + "decoded chunked body exceeds {max_decoded_bytes} byte buffer limit" + )) + }); } read_buffered_exact( client, @@ -1228,7 +1304,7 @@ async fn collect_chunked_body( generation_guard, ) .await - .map_err(|e| miette!("Chunked body ended mid-chunk: {e}"))?; + .map_err(|error| error.context(|e| miette!("Chunked body ended mid-chunk: {e}")))?; let mut chunk_crlf = Vec::with_capacity(2); read_buffered_exact( @@ -1240,9 +1316,13 @@ async fn collect_chunked_body( generation_guard, ) .await - .map_err(|e| miette!("Chunked body ended before chunk terminator: {e}"))?; + .map_err(|error| { + error.context(|e| miette!("Chunked body ended before chunk terminator: {e}")) + })?; if chunk_crlf.as_slice() != b"\r\n" { - return Err(miette!("Chunk missing terminating CRLF")); + return Err(CollectChunkedError::Failed(miette!( + "Chunk missing terminating CRLF" + ))); } } } @@ -1258,15 +1338,15 @@ async fn read_chunked_line( already_read: &[u8], state: &mut ChunkedReadState, generation_guard: Option<&PolicyGenerationGuard>, -) -> Result> { +) -> std::result::Result, CollectChunkedError> { let mut line = Vec::new(); loop { let byte = read_buffered_byte(client, already_read, state, generation_guard).await?; line.push(byte); if line.len() > MAX_REWRITE_BODY_BYTES { - return Err(miette!( + return Err(CollectChunkedError::Failed(miette!( "request body credential rewrite buffers at most {MAX_REWRITE_BODY_BYTES} bytes" - )); + ))); } if line.ends_with(b"\r\n") { line.truncate(line.len() - 2); @@ -1282,7 +1362,7 @@ async fn read_buffered_exact( len: usize, out: &mut Vec, generation_guard: Option<&PolicyGenerationGuard>, -) -> Result<()> { +) -> std::result::Result<(), CollectChunkedError> { for _ in 0..len { let byte = read_buffered_byte(client, already_read, state, generation_guard).await?; out.push(byte); @@ -1295,14 +1375,12 @@ async fn read_buffered_byte( already_read: &[u8], state: &mut ChunkedReadState, generation_guard: Option<&PolicyGenerationGuard>, -) -> Result { +) -> std::result::Result { if state .max_wire_bytes .is_some_and(|max| state.wire_bytes >= max) { - return Err(miette!( - "chunked body wire representation exceeds middleware buffer limit" - )); + return Err(CollectChunkedError::OverCapacity); } let byte = if state.buffered_pos < already_read.len() { @@ -1310,9 +1388,15 @@ async fn read_buffered_byte( state.buffered_pos += 1; byte } else { - let byte = client.read_u8().await.into_diagnostic()?; + let byte = client + .read_u8() + .await + .into_diagnostic() + .map_err(CollectChunkedError::Failed)?; if let Some(guard) = generation_guard { - guard.ensure_current()?; + guard + .ensure_current() + .map_err(CollectChunkedError::Failed)?; } byte }; @@ -3472,7 +3556,10 @@ mod tests { .await .expect_err("wire framing over the cap must be rejected"); - assert!(error.to_string().contains("wire representation")); + assert!( + matches!(error, CollectChunkedError::OverCapacity), + "over-cap wire body must be OverCapacity, got {error:?}" + ); } #[tokio::test] @@ -3491,6 +3578,127 @@ mod tests { assert_eq!(body.len(), payload_len); } + #[tokio::test] + async fn middleware_chunked_invalid_size_is_not_over_capacity() { + let mut raw = + b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n" + .to_vec(); + raw.extend_from_slice(b"xyz\r\n"); + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw, + body_length: BodyLength::Chunked, + }; + let err = + buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, 64 * 1024) + .await + .expect_err("invalid chunk framing must surface as an error"); + + assert!( + err.to_string().contains("Invalid chunk size token"), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("over_capacity") + && !err.to_string().contains("exceeds middleware buffer limit"), + "protocol errors must not be reported as over-capacity: {err}" + ); + } + + #[tokio::test] + async fn middleware_chunked_over_capacity_still_maps_to_buffer_over_capacity() { + let max_body_bytes = 32; + let payload = "hello world that is definitely over the tiny cap"; + let wire = format!("{:x}\r\n{payload}\r\n0\r\n\r\n", payload.len()); + let mut raw = + b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n" + .to_vec(); + raw.extend_from_slice(wire.as_bytes()); + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw, + body_length: BodyLength::Chunked, + }; + + let result = + buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, max_body_bytes) + .await + .expect("over-capacity is a BufferResult, not an Err"); + + assert!( + matches!(result, BufferResult::OverCapacity { recoverable: false }), + "expected OverCapacity, got {result:?}" + ); + } + + #[tokio::test] + async fn middleware_none_body_with_header_overshoot_is_rejected() { + // Mimic the forward-proxy multi-byte read: headers plus pipelined bytes + // after `\r\n\r\n` on a request with no body framing. + let raw = b"GET /api HTTP/1.1\r\nHost: example.com\r\n\r\nGET /other HTTP/1.1\r\n"; + let req = L7Request { + action: "GET".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw.to_vec(), + body_length: BodyLength::None, + }; + + let err = + buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, 64 * 1024) + .await + .expect_err("read-ahead leftovers must not become a request body"); + + assert!( + err.to_string().contains("no body framing"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn middleware_none_body_without_overshoot_buffers_empty() { + let raw = b"GET /api HTTP/1.1\r\nHost: example.com\r\n\r\n"; + let req = L7Request { + action: "GET".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw.to_vec(), + body_length: BodyLength::None, + }; + + let result = + buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, 64 * 1024) + .await + .expect("empty no-body request should buffer"); + + match result { + BufferResult::Buffered(buffered) => { + assert!(buffered.body.is_empty()); + let rebuilt = rebuild_request_with_buffered_body( + &req, + &buffered.headers, + &buffered.body, + &[], + ) + .expect("rebuild no-body request"); + assert!(matches!(rebuilt.body_length, BodyLength::None)); + let text = String::from_utf8(rebuilt.raw_header).unwrap(); + assert!( + !text.to_ascii_lowercase().contains("content-length"), + "rebuild must preserve no-body framing: {text}" + ); + assert!(!text.contains("GET /other")); + } + other @ BufferResult::OverCapacity { .. } => { + panic!("expected Buffered, got {other:?}") + } + } + } + /// SEC-009: Bare LF in headers enables header injection. #[tokio::test] async fn reject_bare_lf_in_headers() { From 2f84161defe97e31308faeb002ec1cf7f3b871fd Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 12:47:17 -0700 Subject: [PATCH 41/59] fix(middleware): address review findings Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 7 + crates/openshell-ocsf/src/format/shorthand.rs | 83 +++++- crates/openshell-sandbox/src/lib.rs | 74 +++-- .../src/lib.rs | 276 ++++++++++++++++-- .../src/remote.rs | 6 +- .../src/l7/rest.rs | 136 ++++++++- docs/extensibility/supervisor-middleware.mdx | 14 +- docs/reference/gateway-config.mdx | 2 +- proto/supervisor_middleware.proto | 14 +- 9 files changed, 542 insertions(+), 70 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index f5c760aba7..4f51bc83ac 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,6 +74,13 @@ and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. +The platform caps middleware bodies at 4 MiB and configures the gRPC client +with 64 KiB of protobuf envelope headroom. Registration and manifest limits +above that boundary fail before request-body allocation. External per-request +reason, finding text, and diagnostic metadata are untrusted: the supervisor +maps logged findings to the validated binding ID and platform-owned text. +At startup, the supervisor installs the in-process registry before connecting +to external services, so an external outage cannot remove built-in bindings. For a gateway policy snapshot, the supervisor prepares replacements off to the side and installs them as one runtime generation. A policy-only change swaps the policy engine alone and reuses the connected registry, so middleware diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 4dbc4eb321..60df77639f 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -82,6 +82,42 @@ fn truncate_with_ellipsis(text: &str, max: usize) -> String { format!("{}...", &text[..end]) } +fn escape_quoted_field(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '\\' => escaped.push_str("\\\\"), + '"' => escaped.push_str("\\\""), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + character if character.is_control() => { + use std::fmt::Write as _; + let _ = write!(escaped, "\\u{{{:x}}}", u32::from(character)); + } + character => escaped.push(character), + } + } + escaped +} + +fn escape_context_field(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '\\' => escaped.push_str("\\\\"), + '[' => escaped.push_str("\\["), + ']' => escaped.push_str("\\]"), + character if character.is_whitespace() || character.is_control() => { + use std::fmt::Write as _; + let _ = write!(escaped, "\\u{{{:x}}}", u32::from(character)); + } + character => escaped.push(character), + } + } + escaped +} + fn reason_text(text: Option<&str>) -> Option { let text = text?; if text.is_empty() { @@ -109,12 +145,12 @@ fn unmapped_fields(base: &BaseEventData) -> Vec { serde_json::Value::Bool(value) => value.to_string(), serde_json::Value::Number(value) => value.to_string(), serde_json::Value::String(value) => { - let value = value.replace(['\n', '\r'], " "); - truncate_with_ellipsis(&value, MAX_REASON_LEN) + let value = truncate_with_ellipsis(value, MAX_REASON_LEN); + escape_context_field(&value) } _ => return None, }; - Some(format!("{key}:{value}")) + Some(format!("{}:{value}", escape_context_field(key))) }) .collect() } @@ -326,8 +362,14 @@ impl OcsfEvent { || e.base.activity_name.to_uppercase(), |d| d.label().to_uppercase(), ); - let title = &e.finding_info.title; - let mut context = vec![format!("type:{}", e.finding_info.uid)]; + let title = escape_quoted_field(&truncate_with_ellipsis( + &e.finding_info.title, + MAX_MESSAGE_LEN, + )); + let mut context = vec![format!( + "type:{}", + escape_context_field(&e.finding_info.uid) + )]; context.extend(unmapped_fields(&e.base)); if let Some(confidence) = e.confidence { context.push(format!("confidence:{}", confidence.label().to_lowercase())); @@ -1027,6 +1069,37 @@ mod tests { assert!(!shorthand.contains("must-not-appear")); } + #[test] + fn detection_finding_shorthand_escapes_control_characters_and_delimiters() { + let mut base = base(2004, "Detection Finding", 2, "Findings", 1, "Create"); + base.add_unmapped( + "source\nforged", + serde_json::json!("guard]\nFINDING:FORGED"), + ); + let event = OcsfEvent::DetectionFinding(DetectionFindingEvent { + base, + finding_info: FindingInfo::new( + "content_guard\nforged", + "matched \"value\"\nFINDING:FORGED", + ), + evidences: None, + attacks: None, + remediation: None, + is_alert: None, + confidence: None, + risk_level: None, + action: None, + disposition: None, + }); + + let shorthand = event.format_shorthand(); + + assert_eq!(shorthand.lines().count(), 1); + assert!(shorthand.contains("matched \\\"value\\\"\\nFINDING:FORGED")); + assert!(shorthand.contains("type:content_guard\\u{a}forged")); + assert!(shorthand.contains("source\\u{a}forged:guard\\]\\u{a}FINDING:FORGED")); + } + #[test] fn test_lifecycle_shorthand() { let mut b = base( diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index a37c5d17f6..e9d9f0fe57 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1967,12 +1967,19 @@ async fn load_policy( } }; + // Install the in-process catalog before any external connection can + // fail. A newly started sandbox must always be able to resolve built-in + // bindings, even while operator-run services are unavailable. + install_builtin_middleware_registry(&engine).await?; + // Connect operator-registered middleware services. A connect/describe // failure keeps the built-in registry active so each request's // `on_error` policy governs matched traffic. The policy poll loop // retries the install without waiting for a config change. let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = match grpc_retry("Middleware connect", || { + let middleware_registry_status = if middleware_services.is_empty() { + MiddlewareRegistryStatus::Synchronized + } else if let Err(error) = grpc_retry("Middleware connect", || { openshell_supervisor_middleware::MiddlewareRegistry::connect_services( openshell_supervisor_middleware_builtins::services(), middleware_services.clone(), @@ -1981,24 +1988,23 @@ async fn load_policy( .await .and_then(|registry| engine.replace_middleware_registry(registry)) { - Ok(()) => MiddlewareRegistryStatus::Synchronized, - Err(error) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + MiddlewareRegistryStatus::NeedsReconciliation + } else { + MiddlewareRegistryStatus::Synchronized }; let opa_engine = Some(Arc::new(engine)); @@ -2476,6 +2482,15 @@ async fn connect_middleware_registry( .await } +async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + opa_engine.replace_middleware_registry(registry) +} + async fn reconcile_middleware_registry( opa_engine: &OpaEngine, desired_services: &[openshell_core::proto::SupervisorMiddlewareService], @@ -3261,6 +3276,27 @@ filesystem_policy: } } + #[tokio::test] + async fn failed_external_startup_registry_build_preserves_installed_builtins() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let builtins_generation = engine.current_generation(); + assert_eq!(builtins_generation, 1); + + let invalid_external = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_body_bytes: 1024, + }; + connect_middleware_registry(&[invalid_external]) + .await + .expect_err("unavailable external service must not replace built-ins"); + + assert_eq!(engine.current_generation(), builtins_generation); + } + #[test] fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { let services = Vec::new(); diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index e6fd8ae7a5..a885228e44 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -23,9 +23,16 @@ use openshell_core::proto::{ use tokio::sync::OnceCell; use tonic::Request; +/// Largest request or replacement body accepted by the middleware platform. +pub const MAX_MIDDLEWARE_BODY_BYTES: usize = 4 * 1024 * 1024; +/// gRPC message limit for middleware calls, including protobuf envelope fields. +pub const MIDDLEWARE_GRPC_MESSAGE_BYTES: usize = MAX_MIDDLEWARE_BODY_BYTES + 64 * 1024; + const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation = SupervisorMiddlewareOperation::HttpRequest; const PRE_CREDENTIALS_PHASE: SupervisorMiddlewarePhase = SupervisorMiddlewarePhase::PreCredentials; +const MAX_STABLE_IDENTIFIER_BYTES: usize = 128; +const EXTERNAL_FINDING_LABEL: &str = "External middleware finding"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OnError { FailClosed, @@ -223,6 +230,12 @@ struct MiddlewareServiceState { operator_max_body_bytes: Option, } +impl MiddlewareServiceState { + fn is_external(&self) -> bool { + self.operator_max_body_bytes.is_some() + } +} + /// Validated middleware services available to a gateway or one supervisor. /// /// In-process services are supplied by the composition root; the generic @@ -283,9 +296,44 @@ fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<( registration.name )); } + if registration.max_body_bytes > MAX_MIDDLEWARE_BODY_BYTES as u64 { + return Err(miette!( + "middleware registration '{}' max_body_bytes exceeds the platform maximum of {MAX_MIDDLEWARE_BODY_BYTES}", + registration.name + )); + } Ok(()) } +fn is_stable_identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_STABLE_IDENTIFIER_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'/')) +} + +fn validate_body_limit(source: &str, binding: &MiddlewareBinding) -> Result { + if binding.max_body_bytes == 0 { + return Err(miette!( + "middleware binding '{}' must advertise a non-zero body limit", + binding.id + )); + } + if binding.max_body_bytes > MAX_MIDDLEWARE_BODY_BYTES as u64 { + return Err(miette!( + "{source} binding '{}' body limit exceeds the platform maximum of {MAX_MIDDLEWARE_BODY_BYTES}", + binding.id + )); + } + usize::try_from(binding.max_body_bytes).map_err(|_| { + miette!( + "middleware binding '{}' reports a body limit too large for this platform", + binding.id + ) + }) +} + fn validate_manifest_bindings( source: &str, manifest: &MiddlewareManifest, @@ -299,8 +347,10 @@ fn validate_manifest_bindings( let mut described_ids = Vec::with_capacity(manifest.bindings.len()); for binding in &manifest.bindings { - if binding.id.trim().is_empty() { - return Err(miette!("{source} describes an empty binding id")); + if !is_stable_identifier(&binding.id) { + return Err(miette!( + "{source} binding ids must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'" + )); } if !allow_reserved_bindings && binding.id.starts_with("openshell/") { return Err(miette!( @@ -316,18 +366,7 @@ fn validate_manifest_bindings( binding.id, )); } - let advertised = usize::try_from(binding.max_body_bytes).map_err(|_| { - miette!( - "middleware binding '{}' reports a body limit too large for this platform", - binding.id - ) - })?; - if advertised == 0 { - return Err(miette!( - "middleware binding '{}' must advertise a non-zero body limit", - binding.id - )); - } + let advertised = validate_body_limit(source, binding)?; if operator_max_body_bytes.is_some_and(|limit| limit > advertised) { return Err(miette!( "{source} max_body_bytes ({}) exceeds binding '{}' capability ({advertised})", @@ -361,6 +400,39 @@ fn validate_external_manifest( ) } +/// External diagnostic text is untrusted and may contain request data. Keep +/// only values derived from the validated, startup-time binding identifier and +/// numeric finding counts; do not carry per-request free-form text into logs. +fn normalize_external_result( + binding: &MiddlewareBinding, + result: &mut openshell_core::proto::HttpRequestResult, +) { + let reason_id: String = binding + .id + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character + } else { + '_' + } + }) + .collect(); + result.reason = format!("middleware_denied:{reason_id}"); + result.metadata.clear(); + for finding in &mut result.findings { + finding.r#type = format!("{}.finding", binding.id); + finding.label = EXTERNAL_FINDING_LABEL.to_string(); + finding.confidence.clear(); + finding.severity = match finding.severity.as_str() { + "low" => "low", + "high" => "high", + _ => "medium", + } + .to_string(); + } +} + impl MiddlewareRegistry { /// Describe in-process services, then connect and validate every /// operator-provided service registration. @@ -597,16 +669,13 @@ impl ChainRunner { let max_body_bytes = binding .as_ref() .map(|binding| { - let advertised = usize::try_from(binding.max_body_bytes).map_err(|_| { - miette!( - "middleware binding '{}' reports a body limit too large for this platform", - binding.id - ) - })?; - Ok::<_, miette::Report>(service - .as_ref() - .and_then(|state| state.operator_max_body_bytes) - .unwrap_or(advertised)) + let advertised = validate_body_limit("middleware manifest", binding)?; + Ok::<_, miette::Report>( + service + .as_ref() + .and_then(|state| state.operator_max_body_bytes) + .unwrap_or(advertised), + ) }) .transpose()? .unwrap_or(0); @@ -736,14 +805,19 @@ impl ChainRunner { let Some(service) = entry.service.as_ref() else { unreachable!("described binding always has a service") }; - let result = match service + let mut result = match service .service .evaluate_http_request(Request::new(evaluation)) .await { Ok(result) => result.into_inner(), Err(err) => { - match apply_on_error(entry, &safe_reason(&err.to_string()), &mut applied) { + let reason = if service.is_external() { + "external_service_error".to_string() + } else { + safe_reason(&err.to_string()) + }; + match apply_on_error(entry, &reason, &mut applied) { OnErrorAction::FailOpen => continue, OnErrorAction::FailClosed(reason) => { return Ok(ChainOutcome { @@ -760,6 +834,10 @@ impl ChainRunner { } }; + if service.is_external() { + normalize_external_result(binding, &mut result); + } + let decision = match Decision::try_from(result.decision) { Ok(decision @ (Decision::Allow | Decision::Deny)) => decision, Ok(Decision::Unspecified) | Err(_) => { @@ -1974,6 +2052,50 @@ mod tests { assert!(error.to_string().contains("exceeds")); } + #[test] + fn external_registration_rejects_body_limit_above_platform_maximum() { + let registration = external_registration(u64::MAX); + let error = validate_registration(®istration) + .expect_err("extreme body limit must be rejected before allocation"); + assert!(error.to_string().contains("platform maximum")); + } + + #[test] + fn manifest_rejects_body_limit_above_platform_maximum() { + let registration = external_registration(4096); + let manifest = MiddlewareManifest { + name: "example/service".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/content-guard".into(), + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, + max_body_bytes: u64::MAX, + }], + }; + let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) + .expect_err("extreme advertised body limit must be rejected"); + assert!(error.to_string().contains("platform maximum")); + } + + #[test] + fn manifest_rejects_unstable_binding_identifier() { + let registration = external_registration(4096); + let manifest = MiddlewareManifest { + name: "example/service".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/content-guard\nforged".into(), + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, + max_body_bytes: 4096, + }], + }; + let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) + .expect_err("control characters must be rejected in binding ids"); + assert!(error.to_string().contains("binding ids must be")); + } + #[test] fn external_manifest_cannot_claim_reserved_binding() { let registration = external_registration(4096); @@ -2078,6 +2200,108 @@ mod tests { .expect("serve"); } + #[tokio::test] + async fn remote_transport_accepts_platform_maximum_response_body() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test middleware"); + let address = listener.local_addr().expect("test middleware address"); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tonic::transport::Server::builder() + .add_service( + SupervisorMiddlewareServer::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: MAX_MIDDLEWARE_BODY_BYTES as u64, + result: openshell_core::proto::HttpRequestResult { + body: vec![b'x'; MAX_MIDDLEWARE_BODY_BYTES], + has_body: true, + ..allow_result() + }, + }) + .max_decoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES) + .max_encoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES), + ) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = shutdown_rx.await; + }); + let server_task = tokio::spawn(server); + + let mut registration = external_registration(MAX_MIDDLEWARE_BODY_BYTES as u64); + registration.grpc_endpoint = format!("http://{address}"); + let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration]) + .await + .expect("connect external middleware"); + let outcome = ChainRunner::from_registry(registry) + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }], + input("hello"), + ) + .await + .expect("maximum-size response should fit configured transport limit"); + + assert!(outcome.allowed); + assert_eq!(outcome.body.len(), MAX_MIDDLEWARE_BODY_BYTES); + let _ = shutdown_tx.send(()); + server_task + .await + .expect("join test middleware") + .expect("serve"); + } + + #[tokio::test] + async fn external_diagnostics_are_normalized_before_reaching_logs() { + let secret = "sk-secret-request-value"; + let registration = external_registration(4096); + let service = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: openshell_core::proto::HttpRequestResult { + decision: Decision::Deny as i32, + reason: format!("denied body={secret}\nFINDING:FORGED"), + findings: vec![Finding { + r#type: format!("secret.{secret}\nforged"), + label: format!("matched {secret}\nFINDING:FORGED"), + count: 1, + confidence: secret.into(), + severity: "high\nFINDING:FORGED".into(), + }], + metadata: std::iter::once(("request".into(), secret.into())).collect(), + ..allow_result() + }, + }); + let registry = registry_with_external(service, registration).await; + let outcome = ChainRunner::from_registry(registry) + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }], + input("hello"), + ) + .await + .expect("evaluate external middleware"); + + assert_eq!(outcome.reason, "middleware_denied:example_content-guard"); + assert_eq!( + outcome.findings[0].finding.r#type, + "example/content-guard.finding" + ); + assert_eq!(outcome.findings[0].finding.label, EXTERNAL_FINDING_LABEL); + assert_eq!(outcome.findings[0].finding.severity, "medium"); + assert!(outcome.metadata.is_empty()); + assert!(!format!("{outcome:?}").contains(secret)); + assert!(!format!("{outcome:?}").contains("FINDING:FORGED")); + } + #[tokio::test] async fn deny_decision_short_circuits_chain() { let runner = ChainRunner::new(Arc::new(scripted_service( diff --git a/crates/openshell-supervisor-middleware/src/remote.rs b/crates/openshell-supervisor-middleware/src/remote.rs index 7645ed811f..b95e8d9f0f 100644 --- a/crates/openshell-supervisor-middleware/src/remote.rs +++ b/crates/openshell-supervisor-middleware/src/remote.rs @@ -13,6 +13,8 @@ use openshell_core::proto::{ use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; use tonic::{Request, Response, Status}; +use crate::MIDDLEWARE_GRPC_MESSAGE_BYTES; + const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const RPC_TIMEOUT: Duration = Duration::from_secs(5); @@ -51,7 +53,9 @@ impl RemoteMiddlewareService { })?; Ok(Self { registration_name: registration_name.to_string(), - client: SupervisorMiddlewareClient::new(channel), + client: SupervisorMiddlewareClient::new(channel) + .max_decoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES) + .max_encoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES), }) } diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index c4f01861ca..b295129e72 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -879,7 +879,10 @@ pub(crate) async fn buffer_request_body_for_middleware 0 && already_read.is_empty() { @@ -1363,9 +1366,48 @@ async fn read_buffered_exact( out: &mut Vec, generation_guard: Option<&PolicyGenerationGuard>, ) -> std::result::Result<(), CollectChunkedError> { - for _ in 0..len { - let byte = read_buffered_byte(client, already_read, state, generation_guard).await?; - out.push(byte); + if state + .max_wire_bytes + .is_some_and(|max| len > max.saturating_sub(state.wire_bytes)) + { + return Err(CollectChunkedError::OverCapacity); + } + out.try_reserve_exact(len).map_err(|_| { + CollectChunkedError::Failed(miette!( + "unable to allocate {len} bytes for chunked request body" + )) + })?; + + let buffered = len.min(already_read.len().saturating_sub(state.buffered_pos)); + if buffered > 0 { + let end = state.buffered_pos + buffered; + out.extend_from_slice(&already_read[state.buffered_pos..end]); + state.buffered_pos = end; + state.wire_bytes += buffered; + } + + let mut remaining = len - buffered; + let mut buf = [0u8; RELAY_BUF_SIZE]; + while remaining > 0 { + let to_read = remaining.min(buf.len()); + let read = client + .read(&mut buf[..to_read]) + .await + .into_diagnostic() + .map_err(CollectChunkedError::Failed)?; + if read == 0 { + return Err(CollectChunkedError::Failed(miette!( + "connection closed with {remaining} bytes remaining" + ))); + } + if let Some(guard) = generation_guard { + guard + .ensure_current() + .map_err(CollectChunkedError::Failed)?; + } + out.extend_from_slice(&buf[..read]); + state.wire_bytes += read; + remaining -= read; } Ok(()) } @@ -2797,13 +2839,48 @@ mod tests { use crate::opa::OpaEngine; use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status}; use openshell_core::secrets::SecretResolver; + use std::pin::Pin; use std::sync::Arc; + use std::task::{Context, Poll}; + use tokio::io::ReadBuf; const TEST_POLICY: &str = include_str!("../../data/sandbox-policy.rego"); const VALID_WS_KEY: &str = "dGhlIHNhbXBsZSBub25jZQ=="; const VALID_WS_ACCEPT: &str = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="; const TEXT_OPCODE: u8 = 0x1; + struct CountingReader { + bytes: Vec, + position: usize, + reads: usize, + } + + impl CountingReader { + fn new(bytes: Vec) -> Self { + Self { + bytes, + position: 0, + reads: 0, + } + } + } + + impl AsyncRead for CountingReader { + fn poll_read( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + self.reads += 1; + let available = self.bytes.len().saturating_sub(self.position); + let amount = available.min(buffer.remaining()); + let end = self.position + amount; + buffer.put_slice(&self.bytes[self.position..end]); + self.position = end; + Poll::Ready(Ok(())) + } + } + fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { HeaderMutation { operation: Some(header_mutation::Operation::Write( @@ -3525,6 +3602,57 @@ mod tests { assert_eq!(body, b"hello world"); } + #[tokio::test] + async fn collect_chunked_body_reads_payload_in_blocks() { + let payload_len = 64 * 1024; + let mut wire = format!("{payload_len:x}\r\n").into_bytes(); + wire.extend(std::iter::repeat_n(b'x', payload_len)); + wire.extend_from_slice(b"\r\n0\r\n\r\n"); + let mut client = CountingReader::new(wire); + + let body = collect_chunked_body( + &mut client, + &[], + None, + Some(openshell_supervisor_middleware::MAX_MIDDLEWARE_BODY_BYTES), + ) + .await + .expect("chunked body should decode"); + + assert_eq!(body.len(), payload_len); + assert!( + client.reads <= 32, + "payload should be read in blocks, observed {} reads", + client.reads + ); + } + + #[tokio::test] + async fn extreme_content_length_is_rejected_before_allocation() { + let req = L7Request { + action: "POST".into(), + target: "/upload".into(), + query_params: HashMap::new(), + raw_header: b"POST /upload HTTP/1.1\r\nHost: example.com\r\nContent-Length: 18446744073709551615\r\n\r\n".to_vec(), + body_length: BodyLength::ContentLength(u64::MAX), + }; + let (mut client, _peer) = tokio::io::duplex(1); + + let result = buffer_request_body_for_middleware( + &req, + &mut client, + None, + openshell_supervisor_middleware::MAX_MIDDLEWARE_BODY_BYTES, + ) + .await + .expect("oversized body should produce a capacity result"); + + assert!(matches!( + result, + BufferResult::OverCapacity { recoverable: true } + )); + } + #[tokio::test] async fn middleware_chunked_wire_body_at_cap_is_allowed() { let max_body_bytes = max_middleware_body_bytes().await; diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 473984c99e..32c017b340 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -26,7 +26,7 @@ Because each transformed body is re-checked before the next stage runs, a middle If post-transformation policy evaluation itself fails, OpenShell denies the request and emits a high-severity detection finding. This failure is separate from middleware `on_error` because the middleware completed successfully; the sandbox policy could not validate its output. -Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Request headers are delivered in wire order and repeated header names are preserved as separate entries, so a service inspects every value the upstream will receive. +Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Middleware-visible request headers are delivered in wire order and repeated header names are preserved as separate entries. OpenShell filters credential, routing, framing, and hop-by-hop headers before invoking middleware. ## Choose a Middleware Type @@ -54,7 +54,7 @@ max_body_bytes = 262144 | --- | --- | | `name` | Operator-facing registration name used in diagnostics. Policies do not reference this value. | | `grpc_endpoint` | Service address reachable from both the gateway and sandbox supervisors. Supports plaintext `http://` and TLS `https://` with platform trust roots. | -| `max_body_bytes` | Operator limit applied to every binding exposed by the service. | +| `max_body_bytes` | Operator limit applied to every binding exposed by the service, up to the 4 MiB platform maximum. | The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. @@ -109,7 +109,9 @@ Every middleware binding declares the largest request or replacement body it sup - A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. - The same per-stage limit applies to request bodies and replacement bodies. -The gateway rejects a registration whose operator limit exceeds the service capability instead of silently clamping it. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. A body larger than every stage limit cannot be inspected at all and is denied unless every selected stage is `fail_open`. +The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 64 KiB for protobuf envelope fields. + +At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. OpenShell can apply `fail_open` to an oversized `Content-Length` before consuming body bytes. A chunked body can cross the limit only after bytes have been consumed, so OpenShell denies that request because it cannot safely resume the original stream. ## Mutate Request Headers @@ -142,10 +144,10 @@ When the effective sandbox configuration changes, a running supervisor validates Middleware activity is emitted through OpenShell's OCSF logging: - Each invocation records its policy-local middleware name, binding, decision, transformation state, and failure state. -- A denied invocation records the service-provided audit-safe reason without request content, configured terms, credentials, or other secrets. +- A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the validated binding ID instead of service-provided per-request text. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. -- Findings include the service-provided type and label plus aggregate counts. Middleware services should keep those fields audit-safe and omit request content or matched values. +- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. - Registry reload success and failure are emitted as configuration state changes. See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. @@ -158,5 +160,3 @@ See [Logging](/observability/logging) for log access and [OCSF JSON Export](/obs - Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. - Operator-run services support plaintext `http://` and TLS `https://` endpoints. HTTPS certificates must chain to a CA in the platform trust store. - Custom trust roots, client authentication, health checks, and runtime registration are not available. - -For a runnable operator workflow, see the [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index b08c3b0d06..8b8ac82855 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -163,7 +163,7 @@ Each service implements the supervisor middleware gRPC contract and may expose m The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. -`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero and no larger than each binding's advertised limit. OpenShell rejects an oversized value instead of silently clamping it. +`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 64 KiB so a maximum-size body and its protobuf envelope fit on the transport. The service `grpc_endpoint` currently supports plaintext `http://` and TLS `https://` using the platform trust store. Custom trust roots, client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index 2614bf8a3b..3bb77d9e5f 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -76,9 +76,8 @@ message HttpRequestEvaluation { // Destination and HTTP request target. HttpRequestTarget target = 5; // HTTP request headers before OpenShell injects credentials, in wire - // order. Repeated header names are preserved as separate entries so a - // service inspects every value the upstream will receive. Protected - // headers such as authorization or cookie are omitted. + // order. Repeated header names are preserved as separate entries. Protected + // credential, routing, framing, and hop-by-hop headers are omitted. repeated HttpHeader headers = 6; // Buffered request body. Empty for a bodyless request. bytes body = 7; @@ -201,9 +200,8 @@ message HeaderMutation { message HttpRequestResult { // Allow or deny decision for this request. Decision decision = 1; - // Audit-safe human-readable reason used for diagnostics, denied responses, - // and security logs. Must not contain request content, configured terms, - // credentials, or other secrets. + // Audit-safe human-readable reason. OpenShell does not relay text from an + // operator-run service into denied responses or security logs. string reason = 2; // Replacement request body when has_body is true. bytes body = 3; @@ -216,7 +214,9 @@ message HttpRequestResult { // are always protected. A violating result is a middleware failure handled // according to the policy failure mode. repeated HeaderMutation header_mutations = 5; - // Audit-safe findings produced during evaluation. + // Audit-safe findings produced during evaluation. For operator-run services, + // OpenShell logs platform-owned fields derived from the validated binding id + // rather than service-provided type, label, confidence, or metadata text. repeated Finding findings = 6; // Non-secret service-defined metadata included in diagnostics. map metadata = 7; From bd4b319950b3c78ff174cf253922dae780a4727f Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 13:37:18 -0700 Subject: [PATCH 42/59] refactor(middleware): make diagnostic policy explicit Signed-off-by: Piotr Mlocek --- .../src/lib.rs | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index a885228e44..d1c313d3a0 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -227,12 +227,32 @@ pub struct ChainRunner { struct MiddlewareServiceState { service: Arc, manifest: OnceCell, + diagnostic_policy: MiddlewareDiagnosticPolicy, operator_max_body_bytes: Option, } -impl MiddlewareServiceState { - fn is_external(&self) -> bool { - self.operator_max_body_bytes.is_some() +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MiddlewareDiagnosticPolicy { + Preserve, + Normalize, +} + +impl MiddlewareDiagnosticPolicy { + fn error_reason(self, error: &tonic::Status) -> String { + match self { + Self::Preserve => safe_reason(&error.to_string()), + Self::Normalize => "external_service_error".to_string(), + } + } + + fn process_result( + self, + binding: &MiddlewareBinding, + result: &mut openshell_core::proto::HttpRequestResult, + ) { + if self == Self::Normalize { + normalize_untrusted_diagnostics(binding, result); + } } } @@ -403,7 +423,7 @@ fn validate_external_manifest( /// External diagnostic text is untrusted and may contain request data. Keep /// only values derived from the validated, startup-time binding identifier and /// numeric finding counts; do not carry per-request free-form text into logs. -fn normalize_external_result( +fn normalize_untrusted_diagnostics( binding: &MiddlewareBinding, result: &mut openshell_core::proto::HttpRequestResult, ) { @@ -469,6 +489,7 @@ impl MiddlewareRegistry { services.push(Arc::new(MiddlewareServiceState { service, manifest: manifest_cell, + diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, })); } @@ -520,6 +541,7 @@ impl MiddlewareRegistry { services.push(Arc::new(MiddlewareServiceState { service, manifest: manifest_cell, + diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, operator_max_body_bytes: Some(operator_max_body_bytes), })); registered_services.push(RegisteredMiddlewareService { @@ -610,6 +632,7 @@ impl ChainRunner { services: Arc::new(vec![Arc::new(MiddlewareServiceState { service, manifest: OnceCell::new(), + diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, })]), registered_services: Arc::new(Vec::new()), @@ -812,11 +835,7 @@ impl ChainRunner { { Ok(result) => result.into_inner(), Err(err) => { - let reason = if service.is_external() { - "external_service_error".to_string() - } else { - safe_reason(&err.to_string()) - }; + let reason = service.diagnostic_policy.error_reason(&err); match apply_on_error(entry, &reason, &mut applied) { OnErrorAction::FailOpen => continue, OnErrorAction::FailClosed(reason) => { @@ -834,9 +853,9 @@ impl ChainRunner { } }; - if service.is_external() { - normalize_external_result(binding, &mut result); - } + service + .diagnostic_policy + .process_result(binding, &mut result); let decision = match Decision::try_from(result.decision) { Ok(decision @ (Decision::Allow | Decision::Deny)) => decision, @@ -1842,11 +1861,13 @@ mod tests { Arc::new(MiddlewareServiceState { service: builtin_service, manifest: builtin_manifest_cell, + diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, }), Arc::new(MiddlewareServiceState { service, manifest: manifest_cell, + diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, operator_max_body_bytes: Some(operator_max_body_bytes), }), ]), From be5e2fbb15596e3c0ad87850e0194ae21fa8cb2e Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 13:53:42 -0700 Subject: [PATCH 43/59] fix(middleware): harden external service boundaries Signed-off-by: Piotr Mlocek --- Cargo.lock | 1 + architecture/sandbox.md | 13 +- .../Cargo.toml | 1 + .../src/headers.rs | 169 +++++++--- .../src/lib.rs | 316 +++++++++++++++++- .../src/l7/middleware.rs | 95 ++++-- .../src/l7/relay.rs | 38 +++ .../openshell-supervisor-network/src/proxy.rs | 16 + docs/extensibility/supervisor-middleware.mdx | 6 +- docs/observability/logging.mdx | 2 +- proto/supervisor_middleware.proto | 18 +- 11 files changed, 586 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9105b9049..1c4bd4c33a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3943,6 +3943,7 @@ dependencies = [ "miette", "openshell-core", "openshell-supervisor-middleware-builtins", + "prost", "prost-types", "tokio", "tokio-stream", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 4f51bc83ac..b37b757901 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,11 +74,14 @@ and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. -The platform caps middleware bodies at 4 MiB and configures the gRPC client -with 64 KiB of protobuf envelope headroom. Registration and manifest limits -above that boundary fail before request-body allocation. External per-request -reason, finding text, and diagnostic metadata are untrusted: the supervisor -maps logged findings to the validated binding ID and platform-owned text. +The platform caps middleware bodies at 4 MiB and derives the gRPC transport +limit from bounded request and response components, reserving 420 KiB for the +largest valid protobuf envelope. Config, context, target, headers, mutations, +reasons, findings, and metadata each have explicit size or cardinality limits. +Registration and manifest body limits above the platform boundary fail before +request-body allocation. External per-request reason, finding text, mutation +errors, and diagnostic metadata are untrusted: the supervisor maps logged +findings to the validated binding ID and uses platform-owned failure codes. At startup, the supervisor installs the in-process registry before connecting to external services, so an external outage cannot remove built-in bindings. For a gateway policy snapshot, the supervisor prepares replacements off to the diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index 2c095c64a6..9cdc53febb 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -14,6 +14,7 @@ rust-version.workspace = true openshell-core = { path = "../openshell-core", default-features = false } miette = { workspace = true } +prost = { workspace = true } prost-types = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["channel", "server", "tls-native-roots"] } diff --git a/crates/openshell-supervisor-middleware/src/headers.rs b/crates/openshell-supervisor-middleware/src/headers.rs index 8eb4c448e8..1716a90486 100644 --- a/crates/openshell-supervisor-middleware/src/headers.rs +++ b/crates/openshell-supervisor-middleware/src/headers.rs @@ -3,11 +3,103 @@ //! Validation and logical application of middleware request-header mutations. -use miette::{Result, miette}; use openshell_core::proto::{ExistingHeaderAction, HeaderMutation, header_mutation}; -const MAX_HEADER_MUTATIONS: usize = 64; -const MAX_HEADER_MUTATION_BYTES: usize = 32 * 1024; +pub const MAX_HEADER_MUTATIONS: usize = 64; +pub const MAX_HEADER_MUTATION_BYTES: usize = 32 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HeaderMutationError { + TooMany { count: usize }, + InvalidName { name: String }, + Protected { name: String }, + HopByHop { name: String }, + WriteNamespace { name: String }, + UnsafeValue { name: String }, + TooLarge, + InvalidExistingAction, + MissingExistingAction { name: String }, + UnsupportedExistingAction, + Empty, +} + +impl HeaderMutationError { + /// Stable platform-owned reason suitable for untrusted middleware failures. + pub(crate) fn code(&self) -> &'static str { + match self { + Self::TooMany { .. } => "header_mutation_count_over_capacity", + Self::InvalidName { .. } => "header_mutation_invalid_name", + Self::Protected { .. } => "header_mutation_protected_header", + Self::HopByHop { .. } => "header_mutation_hop_by_hop_header", + Self::WriteNamespace { .. } => "header_mutation_write_namespace", + Self::UnsafeValue { .. } => "header_mutation_unsafe_value", + Self::TooLarge => "header_mutation_bytes_over_capacity", + Self::InvalidExistingAction => "header_mutation_invalid_existing_action", + Self::MissingExistingAction { .. } => "header_mutation_missing_existing_action", + Self::UnsupportedExistingAction => "header_mutation_unsupported_existing_action", + Self::Empty => "header_mutation_empty", + } + } +} + +impl std::fmt::Display for HeaderMutationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooMany { count } => write!( + formatter, + "middleware returned too many header mutations: {count} exceeds {MAX_HEADER_MUTATIONS}" + ), + Self::InvalidName { name } => { + write!( + formatter, + "middleware returned invalid header name '{name}'" + ) + } + Self::Protected { name } => { + write!( + formatter, + "middleware cannot mutate protected header '{name}'" + ) + } + Self::HopByHop { name } => { + write!( + formatter, + "middleware cannot mutate hop-by-hop header '{name}'" + ) + } + Self::WriteNamespace { name } => write!( + formatter, + "middleware can only write request headers prefixed with x-openshell-middleware- and cannot write '{name}'" + ), + Self::UnsafeValue { name } => { + write!( + formatter, + "middleware cannot write header '{name}' with an unsafe value" + ) + } + Self::TooLarge => write!( + formatter, + "middleware header mutations exceed {MAX_HEADER_MUTATION_BYTES} bytes" + ), + Self::InvalidExistingAction => { + write!(formatter, "middleware returned invalid on_existing action") + } + Self::MissingExistingAction { name } => write!( + formatter, + "middleware must specify on_existing for header '{name}'" + ), + Self::UnsupportedExistingAction => { + write!( + formatter, + "middleware returned unsupported on_existing action" + ) + } + Self::Empty => write!(formatter, "middleware returned an empty header mutation"), + } + } +} + +impl std::error::Error for HeaderMutationError {} /// Validate and atomically apply one middleware response to the logical header /// state observed by the next middleware. Repeated values and wire order are @@ -15,12 +107,11 @@ const MAX_HEADER_MUTATION_BYTES: usize = 32 * 1024; pub fn apply( existing_headers: &[(String, String)], mutations: &[HeaderMutation], -) -> Result> { +) -> Result, HeaderMutationError> { if mutations.len() > MAX_HEADER_MUTATIONS { - return Err(miette!( - "middleware returned too many header mutations: {} exceeds {MAX_HEADER_MUTATIONS}", - mutations.len() - )); + return Err(HeaderMutationError::TooMany { + count: mutations.len(), + }); } let mut headers = existing_headers.to_vec(); @@ -30,23 +121,19 @@ pub fn apply( Some(header_mutation::Operation::Write(write)) => { let name = validate_name(&write.name)?; if is_connection_nominated(&headers, &name) { - return Err(miette!( - "middleware cannot mutate hop-by-hop header '{}'", - write.name - )); + return Err(HeaderMutationError::HopByHop { + name: write.name.clone(), + }); } if !name.starts_with("x-openshell-middleware-") { - return Err(miette!( - "middleware can only write request headers prefixed with \ - x-openshell-middleware- and cannot write '{}'", - write.name - )); + return Err(HeaderMutationError::WriteNamespace { + name: write.name.clone(), + }); } if !is_safe_value(&write.value) { - return Err(miette!( - "middleware cannot write header '{}' with an unsafe value", - write.name - )); + return Err(HeaderMutationError::UnsafeValue { + name: write.name.clone(), + }); } mutation_bytes = mutation_bytes .saturating_add(name.len()) @@ -54,12 +141,11 @@ pub fn apply( enforce_size_limit(mutation_bytes)?; let action = ExistingHeaderAction::try_from(write.on_existing) - .map_err(|_| miette!("middleware returned invalid on_existing action"))?; + .map_err(|_| HeaderMutationError::InvalidExistingAction)?; if action == ExistingHeaderAction::Unspecified { - return Err(miette!( - "middleware must specify on_existing for header '{}'", - write.name - )); + return Err(HeaderMutationError::MissingExistingAction { + name: write.name.clone(), + }); } let exists = headers.iter().any(|(existing, _)| *existing == name); if !exists || action == ExistingHeaderAction::Append { @@ -68,47 +154,44 @@ pub fn apply( headers.retain(|(existing, _)| *existing != name); headers.push((name, write.value.clone())); } else if action != ExistingHeaderAction::Skip { - return Err(miette!( - "middleware returned unsupported on_existing action" - )); + return Err(HeaderMutationError::UnsupportedExistingAction); } } Some(header_mutation::Operation::Remove(remove)) => { let name = validate_name(&remove.name)?; if is_connection_nominated(&headers, &name) { - return Err(miette!( - "middleware cannot mutate hop-by-hop header '{}'", - remove.name - )); + return Err(HeaderMutationError::HopByHop { + name: remove.name.clone(), + }); } mutation_bytes = mutation_bytes.saturating_add(name.len()); enforce_size_limit(mutation_bytes)?; headers.retain(|(existing, _)| *existing != name); } - None => return Err(miette!("middleware returned an empty header mutation")), + None => return Err(HeaderMutationError::Empty), } } Ok(headers) } -fn enforce_size_limit(mutation_bytes: usize) -> Result<()> { +fn enforce_size_limit(mutation_bytes: usize) -> Result<(), HeaderMutationError> { if mutation_bytes > MAX_HEADER_MUTATION_BYTES { - return Err(miette!( - "middleware header mutations exceed {MAX_HEADER_MUTATION_BYTES} bytes" - )); + return Err(HeaderMutationError::TooLarge); } Ok(()) } -fn validate_name(name: &str) -> Result { +fn validate_name(name: &str) -> Result { let lower = name.to_ascii_lowercase(); if lower.is_empty() || !lower.bytes().all(is_name_token_byte) { - return Err(miette!("middleware returned invalid header name '{name}'")); + return Err(HeaderMutationError::InvalidName { + name: name.to_string(), + }); } if is_protected(&lower) { - return Err(miette!( - "middleware cannot mutate protected header '{name}'" - )); + return Err(HeaderMutationError::Protected { + name: name.to_string(), + }); } Ok(lower) } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index d1c313d3a0..70e7e861f5 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -12,6 +12,7 @@ use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; use miette::{Result, miette}; +use prost::Message; use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware; use openshell_core::proto::{ @@ -25,8 +26,48 @@ use tonic::Request; /// Largest request or replacement body accepted by the middleware platform. pub const MAX_MIDDLEWARE_BODY_BYTES: usize = 4 * 1024 * 1024; -/// gRPC message limit for middleware calls, including protobuf envelope fields. -pub const MIDDLEWARE_GRPC_MESSAGE_BYTES: usize = MAX_MIDDLEWARE_BODY_BYTES + 64 * 1024; +/// Largest encoded service-specific configuration attached to one evaluation. +pub const MAX_MIDDLEWARE_CONFIG_BYTES: usize = 64 * 1024; +/// Largest encoded request identity context attached to one evaluation. +pub const MAX_MIDDLEWARE_CONTEXT_BYTES: usize = 4 * 1024; +/// Largest encoded destination and request target attached to one evaluation. +pub const MAX_MIDDLEWARE_TARGET_BYTES: usize = 32 * 1024; +/// Largest number of request header lines exposed to one middleware. +pub const MAX_MIDDLEWARE_HEADERS: usize = 128; +/// Largest encoded request header collection exposed to one middleware. +pub const MAX_MIDDLEWARE_HEADER_BYTES: usize = 64 * 1024; +/// Largest operator-provided reason accepted in one middleware result. +pub const MAX_MIDDLEWARE_REASON_BYTES: usize = 4 * 1024; +/// Largest number of findings accepted from one middleware stage. +pub const MAX_MIDDLEWARE_FINDINGS: usize = 64; +/// Largest encoded individual finding accepted from one middleware stage. +pub const MAX_MIDDLEWARE_FINDING_BYTES: usize = 4 * 1024; +/// Largest number of metadata entries accepted from one middleware stage. +pub const MAX_MIDDLEWARE_METADATA_ENTRIES: usize = 64; +/// Largest combined metadata key/value payload accepted from one middleware stage. +pub const MAX_MIDDLEWARE_METADATA_BYTES: usize = 32 * 1024; + +const MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES: usize = 64 * 1024; +const MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES: usize = 64 * 1024; +const MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_CONFIG_BYTES + + MAX_MIDDLEWARE_CONTEXT_BYTES + + MAX_MIDDLEWARE_TARGET_BYTES + + MAX_MIDDLEWARE_HEADER_BYTES + + MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES; +const MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_REASON_BYTES + + MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES + + MAX_MIDDLEWARE_FINDINGS * MAX_MIDDLEWARE_FINDING_BYTES + + MAX_MIDDLEWARE_METADATA_BYTES + + MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES; +const MAX_MIDDLEWARE_ENVELOPE_BYTES: usize = + if MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES > MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES { + MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES + } else { + MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES + }; +/// gRPC message limit derived from the body and bounded protobuf components. +pub const MIDDLEWARE_GRPC_MESSAGE_BYTES: usize = + MAX_MIDDLEWARE_BODY_BYTES + MAX_MIDDLEWARE_ENVELOPE_BYTES; const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation = SupervisorMiddlewareOperation::HttpRequest; @@ -254,6 +295,13 @@ impl MiddlewareDiagnosticPolicy { normalize_untrusted_diagnostics(binding, result); } } + + fn header_mutation_error_reason(self, error: &headers::HeaderMutationError) -> String { + match self { + Self::Preserve => safe_reason(&error.to_string()), + Self::Normalize => error.code().to_string(), + } + } } /// Validated middleware services available to a gateway or one supervisor. @@ -453,6 +501,94 @@ fn normalize_untrusted_diagnostics( } } +fn validate_request_envelope( + evaluation: &HttpRequestEvaluation, +) -> std::result::Result<(), &'static str> { + if evaluation.body.len() > MAX_MIDDLEWARE_BODY_BYTES { + return Err("request_body_over_capacity"); + } + if evaluation + .config + .as_ref() + .is_some_and(|config| config.encoded_len() > MAX_MIDDLEWARE_CONFIG_BYTES) + { + return Err("request_config_over_capacity"); + } + if evaluation + .context + .as_ref() + .is_some_and(|context| context.encoded_len() > MAX_MIDDLEWARE_CONTEXT_BYTES) + { + return Err("request_context_over_capacity"); + } + if evaluation + .target + .as_ref() + .is_some_and(|target| target.encoded_len() > MAX_MIDDLEWARE_TARGET_BYTES) + { + return Err("request_target_over_capacity"); + } + if evaluation.headers.len() > MAX_MIDDLEWARE_HEADERS { + return Err("request_header_count_over_capacity"); + } + let header_bytes = evaluation.headers.iter().fold(0usize, |total, header| { + total.saturating_add(header.encoded_len()) + }); + if header_bytes > MAX_MIDDLEWARE_HEADER_BYTES { + return Err("request_header_bytes_over_capacity"); + } + if evaluation.encoded_len() > MIDDLEWARE_GRPC_MESSAGE_BYTES { + return Err("request_envelope_over_capacity"); + } + Ok(()) +} + +fn validate_response_envelope( + result: &openshell_core::proto::HttpRequestResult, +) -> std::result::Result<(), &'static str> { + if result.body.len() > MAX_MIDDLEWARE_BODY_BYTES { + return Err("response_body_over_capacity"); + } + if result.reason.len() > MAX_MIDDLEWARE_REASON_BYTES { + return Err("response_reason_over_capacity"); + } + if result.header_mutations.len() > headers::MAX_HEADER_MUTATIONS { + return Err("header_mutation_count_over_capacity"); + } + let mutation_bytes = result + .header_mutations + .iter() + .fold(0usize, |total, mutation| { + total.saturating_add(mutation.encoded_len()) + }); + if mutation_bytes > MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES { + return Err("header_mutation_bytes_over_capacity"); + } + if result.findings.len() > MAX_MIDDLEWARE_FINDINGS { + return Err("response_findings_over_capacity"); + } + if result + .findings + .iter() + .any(|finding| finding.encoded_len() > MAX_MIDDLEWARE_FINDING_BYTES) + { + return Err("response_finding_over_capacity"); + } + if result.metadata.len() > MAX_MIDDLEWARE_METADATA_ENTRIES { + return Err("response_metadata_count_over_capacity"); + } + let metadata_bytes = result.metadata.iter().fold(0usize, |total, (key, value)| { + total.saturating_add(key.len()).saturating_add(value.len()) + }); + if metadata_bytes > MAX_MIDDLEWARE_METADATA_BYTES { + return Err("response_metadata_bytes_over_capacity"); + } + if result.encoded_len() > MIDDLEWARE_GRPC_MESSAGE_BYTES { + return Err("response_envelope_over_capacity"); + } + Ok(()) +} + impl MiddlewareRegistry { /// Describe in-process services, then connect and validate every /// operator-provided service registration. @@ -717,6 +853,11 @@ impl ChainRunner { implementation: &str, config: prost_types::Struct, ) -> Result<()> { + if config.encoded_len() > MAX_MIDDLEWARE_CONFIG_BYTES { + return Err(miette!( + "middleware config exceeds the platform maximum of {MAX_MIDDLEWARE_CONFIG_BYTES} encoded bytes" + )); + } let manifests = self.manifests().await?; let Some((state, _)) = manifests.iter().find(|(_, manifest)| { manifest @@ -825,6 +966,22 @@ impl ChainRunner { } } let evaluation = build_evaluation(entry, binding, &input, &headers, &body); + if let Err(reason) = validate_request_envelope(&evaluation) { + match apply_on_error(entry, reason, &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + header_mutations, + findings, + metadata, + applied, + }); + } + } + } let Some(service) = entry.service.as_ref() else { unreachable!("described binding always has a service") }; @@ -853,6 +1010,23 @@ impl ChainRunner { } }; + if let Err(reason) = validate_response_envelope(&result) { + match apply_on_error(entry, reason, &mut applied) { + OnErrorAction::FailOpen => continue, + OnErrorAction::FailClosed(reason) => { + return Ok(ChainOutcome { + allowed: false, + reason, + body, + header_mutations, + findings, + metadata, + applied, + }); + } + } + } + service .diagnostic_policy .process_result(binding, &mut result); @@ -931,7 +1105,10 @@ impl ChainRunner { let updated_headers = match headers::apply(&headers, &result.header_mutations) { Ok(updated) => updated, Err(error) => { - match apply_on_error(entry, &safe_reason(&error.to_string()), &mut applied) { + let reason = service + .diagnostic_policy + .header_mutation_error_reason(&error); + match apply_on_error(entry, &reason, &mut applied) { OnErrorAction::FailOpen => continue, OnErrorAction::FailClosed(reason) => { return Ok(ChainOutcome { @@ -2222,20 +2399,41 @@ mod tests { } #[tokio::test] - async fn remote_transport_accepts_platform_maximum_response_body() { + async fn remote_transport_accepts_maximum_bounded_request_and_response_envelopes() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind test middleware"); let address = listener.local_addr().expect("test middleware address"); let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let response_findings = (0..MAX_MIDDLEWARE_FINDINGS) + .map(|_| Finding { + r#type: "f".repeat(1024), + label: "finding".into(), + count: 1, + confidence: "medium".into(), + severity: "medium".into(), + }) + .collect(); let server = tonic::transport::Server::builder() .add_service( SupervisorMiddlewareServer::new(ScriptedService { binding_id: "example/content-guard".into(), max_body_bytes: MAX_MIDDLEWARE_BODY_BYTES as u64, result: openshell_core::proto::HttpRequestResult { + reason: "r".repeat(MAX_MIDDLEWARE_REASON_BYTES - 128), body: vec![b'x'; MAX_MIDDLEWARE_BODY_BYTES], has_body: true, + header_mutations: vec![write_header( + "x-openshell-middleware-envelope", + &"h".repeat(headers::MAX_HEADER_MUTATION_BYTES - 128), + ExistingHeaderAction::Append, + )], + findings: response_findings, + metadata: std::iter::once(( + "diagnostic".into(), + "m".repeat(MAX_MIDDLEWARE_METADATA_BYTES - 128), + )) + .collect(), ..allow_result() }, }) @@ -2252,22 +2450,44 @@ mod tests { let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration]) .await .expect("connect external middleware"); + let config = prost_types::Struct { + fields: std::iter::once(( + "payload".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue( + "c".repeat(MAX_MIDDLEWARE_CONFIG_BYTES - 256), + )), + }, + )) + .collect(), + }; + assert!(config.encoded_len() <= MAX_MIDDLEWARE_CONFIG_BYTES); + let mut request = input(""); + request.request_id = "r".repeat(MAX_MIDDLEWARE_CONTEXT_BYTES - 256); + request.path = format!("/{}", "p".repeat(MAX_MIDDLEWARE_TARGET_BYTES - 512)); + request.headers = vec![( + "x-large-envelope".into(), + "v".repeat(MAX_MIDDLEWARE_HEADER_BYTES - 256), + )]; + request.body = vec![b'b'; MAX_MIDDLEWARE_BODY_BYTES]; let outcome = ChainRunner::from_registry(registry) .evaluate( &[ChainEntry { name: "guard".into(), implementation: "example/content-guard".into(), order: 0, - config: prost_types::Struct::default(), + config, on_error: OnError::FailClosed, }], - input("hello"), + request, ) .await - .expect("maximum-size response should fit configured transport limit"); + .expect("maximum bounded envelopes should fit configured transport limit"); assert!(outcome.allowed); assert_eq!(outcome.body.len(), MAX_MIDDLEWARE_BODY_BYTES); + assert_eq!(outcome.header_mutations.len(), 1); + assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_FINDINGS); let _ = shutdown_tx.send(()); server_task .await @@ -2323,6 +2543,88 @@ mod tests { assert!(!format!("{outcome:?}").contains("FINDING:FORGED")); } + #[tokio::test] + async fn external_header_mutation_failure_uses_platform_reason() { + let secret = "sk-secret-request-value"; + let registration = external_registration(4096); + let service = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: openshell_core::proto::HttpRequestResult { + header_mutations: vec![write_header( + &format!("x-openshell-middleware-invalid\n{secret}"), + "value", + ExistingHeaderAction::Append, + )], + ..allow_result() + }, + }); + let registry = registry_with_external(service, registration).await; + let outcome = ChainRunner::from_registry(registry) + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }], + input("hello"), + ) + .await + .expect("evaluate external middleware"); + + assert!(!outcome.allowed); + assert_eq!( + outcome.reason, + "middleware_failed: header_mutation_invalid_name" + ); + assert!(outcome.findings.is_empty()); + assert!(!format!("{outcome:?}").contains(secret)); + } + + #[tokio::test] + async fn finding_overflow_is_an_invalid_response_governed_by_on_error() { + let registration = external_registration(4096); + let service = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: openshell_core::proto::HttpRequestResult { + findings: vec![Finding::default(); MAX_MIDDLEWARE_FINDINGS + 1], + ..allow_result() + }, + }); + let registry = registry_with_external(service, registration).await; + let runner = ChainRunner::from_registry(registry); + + for (on_error, allowed) in [(OnError::FailClosed, false), (OnError::FailOpen, true)] { + let outcome = runner + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error, + }], + input("hello"), + ) + .await + .expect("evaluate finding overflow"); + + assert_eq!(outcome.allowed, allowed); + assert!(outcome.findings.is_empty()); + assert_eq!(outcome.applied.len(), 1); + assert!(outcome.applied[0].failed); + if !allowed { + assert_eq!( + outcome.reason, + "middleware_failed: response_findings_over_capacity" + ); + } + } + } + #[tokio::test] async fn deny_decision_short_circuits_chain() { let runner = ChainRunner::new(Arc::new(scripted_service( diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 611fa81cc8..a988ef89ee 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -10,6 +10,7 @@ use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; +use std::collections::HashSet; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncWrite}; @@ -275,30 +276,47 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { fn safe_middleware_headers(headers: &[u8]) -> Result> { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - let mut out = Vec::new(); - for line in header_str.lines().skip(1) { - let Some((name, value)) = line.split_once(':') else { - continue; - }; - let name = name.trim().to_ascii_lowercase(); - if name.is_empty() - || matches!( - name.as_str(), - "authorization" - | "proxy-authorization" - | "cookie" - | "host" - | "content-length" - | "transfer-encoding" - ) - || name.starts_with("x-amz-") - || name.starts_with("x-openshell-credential") - { - continue; - } - out.push((name, value.trim().to_string())); - } - Ok(out) + let parsed: Vec<(String, String)> = header_str + .lines() + .skip(1) + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + Some((name.trim().to_ascii_lowercase(), value.trim().to_string())) + }) + .collect(); + let connection_nominated: HashSet = parsed + .iter() + .filter(|(name, _)| name == "connection") + .flat_map(|(_, value)| value.split(',')) + .map(|token| token.trim().to_ascii_lowercase()) + .filter(|token| !token.is_empty()) + .collect(); + + Ok(parsed + .into_iter() + .filter(|(name, _)| { + !name.is_empty() + && !matches!( + name.as_str(), + "authorization" + | "proxy-authorization" + | "proxy-authenticate" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-connection" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) + && !name.starts_with("x-amz-") + && !name.starts_with("x-openshell-credential") + && !connection_nominated.contains(name) + }) + .collect()) } pub fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { @@ -416,7 +434,14 @@ pub(super) fn middleware_events( .build(); events.push(event); } - for finding in &outcome.findings { + // The runner rejects stages above this cap through `on_error`. Keep the + // emission-side bound as defense in depth for manually constructed or + // future outcome producers. + for finding in outcome + .findings + .iter() + .take(openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS) + { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(match finding.finding.severity.as_str() { "high" => SeverityId::High, @@ -498,4 +523,24 @@ mod tests { ] ); } + + #[test] + fn middleware_headers_omit_standard_and_connection_nominated_hop_by_hop_fields() { + let headers = safe_middleware_headers( + b"GET /v1 HTTP/1.1\r\n\ + X-Hop: secret-hop-value\r\n\ + Connection: keep-alive, x-hop\r\n\ + Keep-Alive: timeout=5\r\n\ + TE: trailers\r\n\ + Trailer: X-Checksum\r\n\ + Upgrade: websocket\r\n\ + X-Visible: visible-value\r\n\r\n", + ) + .expect("headers should parse"); + + assert_eq!( + headers, + vec![("x-visible".to_string(), "visible-value".to_string())] + ); + } } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 18cb1594b5..4f51077d23 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -4046,6 +4046,31 @@ network_policies: // Safe finding metadata is still present. assert!(serialized.contains("secret.common")); + let mut bounded_outcome = outcome; + bounded_outcome.findings = vec![ + NamespacedFinding { + middleware: "external-guard".into(), + finding: openshell_core::proto::Finding { + r#type: "example/content-guard.finding".into(), + label: "External middleware finding".into(), + count: 1, + confidence: String::new(), + severity: "medium".into(), + }, + }; + openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS + + 10 + ]; + let bounded_events = middleware_events(&ctx, &req, &bounded_outcome); + assert_eq!( + bounded_events + .iter() + .filter(|event| event.class_uid() == 2004) + .count(), + openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS, + "finding emission must remain bounded even if an invalid outcome bypasses the runner" + ); + let denied_outcome = ChainOutcome { allowed: false, reason: "request matched configured policy".into(), @@ -4079,6 +4104,19 @@ network_policies: [policy:rest_api engine:middleware] \ [failed:false transformed:false reason:request matched configured policy]" ); + + let external_failure_outcome = ChainOutcome { + reason: "middleware_failed: header_mutation_invalid_name".into(), + applied: vec![MiddlewareInvocation { + failed: true, + ..denied_outcome.applied[0].clone() + }], + ..denied_outcome + }; + let failure_events = middleware_events(&ctx, &req, &external_failure_outcome); + let serialized = serde_json::to_string(&failure_events).expect("serialize failure events"); + assert!(serialized.contains("header_mutation_invalid_name")); + assert!(!serialized.contains(RAW_SECRET)); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index abdff50da4..3e8dd32048 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4843,6 +4843,22 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn middleware_denial_response_contains_only_platform_mutation_reason() { + const RAW_SECRET: &str = "sk-secret-request-value"; + let reason = "middleware_failed: header_mutation_invalid_name"; + let response = build_json_error_response( + 403, + "Forbidden", + "middleware_denied", + &format!("POST api.example.test:443/v1 denied by middleware: {reason}"), + ); + let response = String::from_utf8(response).expect("UTF-8 error response"); + + assert!(response.contains(reason)); + assert!(!response.contains(RAW_SECRET)); + } + #[test] fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { let policy = include_str!("../data/sandbox-policy.rego"); diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 32c017b340..3cbf940072 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -109,7 +109,7 @@ Every middleware binding declares the largest request or replacement body it sup - A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. - The same per-stage limit applies to request bodies and replacement bodies. -The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 64 KiB for protobuf envelope fields. +The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-body protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB reason, 64 header mutations totaling at most 64 KiB encoded, 64 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 420 KiB so every platform-valid envelope fits. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. OpenShell can apply `fail_open` to an oversized `Content-Length` before consuming body bytes. A chunked body can cross the limit only after bytes have been consumed, so OpenShell denies that request because it cannot safely resume the original stream. @@ -125,7 +125,7 @@ A `remove` mutation removes every value for a case-insensitive header name. Open Header writes must use the `x-openshell-middleware-` prefix. Removes may target other middleware-visible request headers. Protected credential, routing, framing, and hop-by-hop headers are always rejected. Header values must not contain control characters. -OpenShell validates and applies each stage's mutations atomically. An invalid operation discards every mutation from that stage and follows its `on_error` behavior. The failure reason names the offending header. +OpenShell validates and applies each stage's mutations atomically. An invalid operation discards every mutation from that stage and follows its `on_error` behavior. Built-in failures can name the offending header. Operator-run failures use a platform-owned error code so request-derived header text cannot reach logs or denied responses. ## Operate Middleware Services @@ -147,7 +147,7 @@ Middleware activity is emitted through OpenShell's OCSF logging: - A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the validated binding ID instead of service-provided per-request text. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. -- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. +- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 64 findings. Exceeding the cap is an invalid response handled through `on_error`. - Registry reload success and failure are emitted as configuration state changes. See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 6cd24c70e2..5941d34bec 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -167,7 +167,7 @@ OCSF CONFIG:LOADED [INFO] Policy reloaded successfully [policy_hash:0cc0c2b52557 Denied `NET:` and `HTTP:` events carry a `[reason:...]` suffix that surfaces the decision detail from the event's `status_detail` field. The reason helps distinguish between policy misses, SSRF hardening, and L7 enforcement without inspecting the full OCSF JSONL record. -For supervisor middleware denials, `status_detail` contains the service-provided audit-safe reason from the gRPC response. +For supervisor middleware denials, `status_detail` contains an audit-safe reason. Built-in middleware can provide that reason directly. Operator-run denial reasons are derived from the validated binding ID, and operator-run failure details use platform-owned error codes; OpenShell does not copy per-request service text into logs. Common reason phrases emitted by the sandbox include: diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index 3bb77d9e5f..e7a260f770 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -70,16 +70,20 @@ message HttpRequestEvaluation { // Evaluation phase selected for this request. SupervisorMiddlewarePhase phase = 2; // Sandbox and request identity available to the supervisor. + // The encoded context is limited to 4 KiB. RequestContext context = 3; // Validated service-specific policy configuration. + // The encoded configuration is limited to 64 KiB. google.protobuf.Struct config = 4; // Destination and HTTP request target. + // The encoded target is limited to 32 KiB. HttpRequestTarget target = 5; // HTTP request headers before OpenShell injects credentials, in wire // order. Repeated header names are preserved as separate entries. Protected // credential, routing, framing, and hop-by-hop headers are omitted. + // At most 128 lines and 64 KiB of encoded headers are included. repeated HttpHeader headers = 6; - // Buffered request body. Empty for a bodyless request. + // Buffered request body, limited to 4 MiB. Empty for a bodyless request. bytes body = 7; } @@ -201,9 +205,10 @@ message HttpRequestResult { // Allow or deny decision for this request. Decision decision = 1; // Audit-safe human-readable reason. OpenShell does not relay text from an - // operator-run service into denied responses or security logs. + // operator-run service into denied responses or security logs. Limited to + // 4 KiB before normalization. string reason = 2; - // Replacement request body when has_body is true. + // Replacement request body when has_body is true. Limited to 4 MiB. bytes body = 3; // True when body should replace the request body, including with an empty body. bool has_body = 4; @@ -212,12 +217,15 @@ message HttpRequestResult { // "x-openshell-middleware-" namespace. Removes may target other visible // request headers, but credential, routing, framing, and hop-by-hop headers // are always protected. A violating result is a middleware failure handled - // according to the policy failure mode. + // according to the policy failure mode. At most 64 operations, 32 KiB of + // validated name/value data, and 64 KiB encoded are accepted. repeated HeaderMutation header_mutations = 5; // Audit-safe findings produced during evaluation. For operator-run services, // OpenShell logs platform-owned fields derived from the validated binding id // rather than service-provided type, label, confidence, or metadata text. + // At most 64 findings of at most 4 KiB encoded each are accepted. repeated Finding findings = 6; - // Non-secret service-defined metadata included in diagnostics. + // Non-secret service-defined metadata included in diagnostics. At most 64 + // entries and 32 KiB of combined key/value data are accepted. map metadata = 7; } From 65ef2cdc2a28f212e6781aedda28aaadf4a380bf Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 16:11:09 -0700 Subject: [PATCH 44/59] fix(middleware): bound policy and response resources Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 8 +- crates/openshell-core/src/lib.rs | 1 + crates/openshell-core/src/middleware.rs | 16 ++ crates/openshell-policy/src/lib.rs | 113 ++++++++++ crates/openshell-policy/src/middleware.rs | 34 ++- .../src/headers.rs | 52 +++-- .../src/lib.rs | 196 +++++++++++++++++- .../src/l7/middleware.rs | 59 ++++-- .../src/l7/relay.rs | 40 +++- .../src/l7/rest.rs | 93 ++++++++- .../openshell-supervisor-network/src/opa.rs | 47 +++++ docs/extensibility/supervisor-middleware.mdx | 10 +- docs/reference/gateway-config.mdx | 2 +- docs/reference/policy-schema.mdx | 6 +- docs/sandboxes/policies.mdx | 2 +- proto/sandbox.proto | 6 +- proto/supervisor_middleware.proto | 3 +- 17 files changed, 613 insertions(+), 75 deletions(-) create mode 100644 crates/openshell-core/src/middleware.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index b37b757901..e20d07b7f5 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -75,9 +75,12 @@ operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. The platform caps middleware bodies at 4 MiB and derives the gRPC transport -limit from bounded request and response components, reserving 420 KiB for the +limit from bounded request and response components, reserving 292 KiB for the largest valid protobuf envelope. Config, context, target, headers, mutations, reasons, findings, and metadata each have explicit size or cardinality limits. +Policies contain at most 10 middleware configs with 32 combined host selector +patterns each. Runtime selection defensively allows at most 10 stages, each of +which can return 32 findings, for a derived chain maximum of 320 findings. Registration and manifest body limits above the platform boundary fail before request-body allocation. External per-request reason, finding text, mutation errors, and diagnostic metadata are untrusted: the supervisor maps logged @@ -120,6 +123,9 @@ operations. Rust validates and applies a stage atomically to the logical header state before the next stage runs, then replays the validated operations against the raw request before credential injection. Credential, routing, framing, and hop-by-hop headers remain supervisor-owned and cannot be mutated. +Names dynamically nominated by the original request's `Connection` header are +carried separately after header filtering so write and remove mutations cannot +reintroduce them. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index a2be6daa7d..4595f8965a 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -24,6 +24,7 @@ pub mod host_pattern; pub mod image; pub mod inference; pub mod metadata; +pub mod middleware; pub mod net; pub mod paths; pub mod policy; diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs new file mode 100644 index 0000000000..cc201d63e3 --- /dev/null +++ b/crates/openshell-core/src/middleware.rs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Platform-wide supervisor middleware cardinality limits. + +/// Largest number of middleware configurations accepted in one sandbox policy. +pub const MAX_MIDDLEWARE_CONFIGS: usize = 10; +/// Largest number of middleware stages selected for one request. +pub const MAX_MIDDLEWARE_CHAIN_STAGES: usize = MAX_MIDDLEWARE_CONFIGS; +/// Largest combined number of include and exclude patterns in one selector. +pub const MAX_MIDDLEWARE_SELECTOR_PATTERNS: usize = 32; +/// Largest number of findings accepted from one middleware stage. +pub const MAX_MIDDLEWARE_FINDINGS_PER_STAGE: usize = 32; +/// Largest number of findings retained and emitted for one complete chain. +pub const MAX_MIDDLEWARE_CHAIN_FINDINGS: usize = + MAX_MIDDLEWARE_CHAIN_STAGES * MAX_MIDDLEWARE_FINDINGS_PER_STAGE; diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 29c7c7a0aa..75163364df 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1136,6 +1136,10 @@ pub enum PolicyViolation { CredentialSigningWithBodyRewrite { policy_name: String, host: String }, /// A middleware configuration is structurally invalid. InvalidMiddlewareConfig { name: String, reason: String }, + /// Too many middleware configurations are attached to one policy. + TooManyMiddlewareConfigs { count: usize }, + /// Too many include and exclude patterns are attached to one middleware. + TooManyMiddlewareSelectorPatterns { name: String, count: usize }, /// Middleware configuration names must be unique. DuplicateMiddlewareConfigName { name: String }, /// A middleware selector conflicts with an endpoint that skips TLS inspection. @@ -1211,6 +1215,20 @@ impl fmt::Display for PolicyViolation { Self::InvalidMiddlewareConfig { name, reason } => { write!(f, "middleware config '{name}' is invalid: {reason}") } + Self::TooManyMiddlewareConfigs { count } => { + write!( + f, + "too many middleware configs ({count} > {})", + openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS + ) + } + Self::TooManyMiddlewareSelectorPatterns { name, count } => { + write!( + f, + "middleware config '{name}' has too many selector patterns ({count} > {})", + openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS + ) + } Self::DuplicateMiddlewareConfigName { name } => { write!(f, "duplicate middleware config '{name}'") } @@ -1835,6 +1853,34 @@ network_policies: ))); } + #[test] + fn json_validation_skips_config_callbacks_when_middleware_count_is_invalid() { + let configs: Vec<_> = (0..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS) + .map(|index| { + serde_json::json!({ + "name": format!("middleware-{index}"), + "middleware": "openshell/secrets", + "endpoints": {"include": ["api.example.com"]} + }) + }) + .collect(); + let data = serde_json::json!({"network_middlewares": configs}); + let calls = std::cell::Cell::new(0usize); + + let violations = validate_network_middleware_json_with_config(&data, |_, _| { + calls.set(calls.get() + 1); + Ok(()) + }) + .expect("parse middleware policy"); + + assert_eq!(calls.get(), 0, "invalid policy must not invoke services"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::TooManyMiddlewareConfigs { count } + if *count == openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS + 1 + ))); + } + #[test] fn validate_rejects_root_run_as_user() { let mut policy = restrictive_default_policy(); @@ -1933,6 +1979,73 @@ network_policies: ))); } + #[test] + fn validate_accepts_maximum_middleware_configs() { + let mut policy = restrictive_default_policy(); + for index in 0..openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { + policy.network_middlewares.push(middleware_config( + &format!("middleware-{index}"), + "openshell/secrets", + )); + } + + validate_sandbox_policy(&policy).expect("maximum middleware config count"); + } + + #[test] + fn validate_rejects_middleware_config_over_capacity() { + let mut policy = restrictive_default_policy(); + for index in 0..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { + policy.network_middlewares.push(middleware_config( + &format!("middleware-{index}"), + "openshell/secrets", + )); + } + + let violations = validate_sandbox_policy(&policy).expect_err("config count over capacity"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::TooManyMiddlewareConfigs { count } + if *count == openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS + 1 + ))); + } + + #[test] + fn validate_accepts_maximum_middleware_selector_patterns() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/secrets"); + let selector = middleware.endpoints.as_mut().expect("selector"); + selector.exclude = vec![ + "excluded.example.com".into(); + openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS - 1 + ]; + policy.network_middlewares.push(middleware); + + validate_sandbox_policy(&policy).expect("maximum selector pattern count"); + } + + #[test] + fn validate_rejects_middleware_selector_patterns_over_capacity() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/secrets"); + let selector = middleware.endpoints.as_mut().expect("selector"); + selector.exclude = vec![ + "excluded.example.com".into(); + openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS + ]; + policy.network_middlewares.push(middleware); + + let violations = + validate_sandbox_policy(&policy).expect_err("selector patterns over capacity"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::TooManyMiddlewareSelectorPatterns { name, count } + if name == "redactor" + && *count + == openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS + 1 + ))); + } + #[test] fn validate_rejects_malformed_middleware_selector_patterns() { let mut policy = restrictive_default_policy(); diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 72a27e0603..30f4aac182 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, HashSet}; +use openshell_core::middleware::{MAX_MIDDLEWARE_CONFIGS, MAX_MIDDLEWARE_SELECTOR_PATTERNS}; use openshell_core::proto::{ MiddlewareEndpointSelector, NetworkEndpoint, NetworkMiddlewareConfig, NetworkPolicyRule, SandboxPolicy, @@ -169,13 +170,15 @@ where ..Default::default() }; let mut violations = validate(&policy); - for middleware in &policy.network_middlewares { - let config = middleware.config.clone().unwrap_or_default(); - if let Err(reason) = validate_config(&middleware.middleware, &config) { - violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), - reason, - }); + if policy.network_middlewares.len() <= MAX_MIDDLEWARE_CONFIGS { + for middleware in &policy.network_middlewares { + let config = middleware.config.clone().unwrap_or_default(); + if let Err(reason) = validate_config(&middleware.middleware, &config) { + violations.push(PolicyViolation::InvalidMiddlewareConfig { + name: middleware.name.clone(), + reason, + }); + } } } Ok(violations) @@ -185,6 +188,12 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { let mut violations = Vec::new(); let mut names = HashSet::new(); + if policy.network_middlewares.len() > MAX_MIDDLEWARE_CONFIGS { + violations.push(PolicyViolation::TooManyMiddlewareConfigs { + count: policy.network_middlewares.len(), + }); + } + for middleware in &policy.network_middlewares { if middleware.name.is_empty() { violations.push(PolicyViolation::InvalidMiddlewareConfig { @@ -227,6 +236,17 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { reason: "endpoint selector must include at least one host pattern".to_string(), }); } + let selector_patterns = selector + .include + .len() + .saturating_add(selector.exclude.len()); + if selector_patterns > MAX_MIDDLEWARE_SELECTOR_PATTERNS { + violations.push(PolicyViolation::TooManyMiddlewareSelectorPatterns { + name: middleware.name.clone(), + count: selector_patterns, + }); + continue; + } let mut selector_valid = !selector.include.is_empty(); for pattern in selector.include.iter().chain(&selector.exclude) { if let Err(reason) = HostPattern::new(pattern) { diff --git a/crates/openshell-supervisor-middleware/src/headers.rs b/crates/openshell-supervisor-middleware/src/headers.rs index 1716a90486..655cf4790a 100644 --- a/crates/openshell-supervisor-middleware/src/headers.rs +++ b/crates/openshell-supervisor-middleware/src/headers.rs @@ -106,6 +106,7 @@ impl std::error::Error for HeaderMutationError {} /// preserved; comparisons are case-insensitive. pub fn apply( existing_headers: &[(String, String)], + connection_nominated_headers: &[String], mutations: &[HeaderMutation], ) -> Result, HeaderMutationError> { if mutations.len() > MAX_HEADER_MUTATIONS { @@ -120,7 +121,7 @@ pub fn apply( match mutation.operation.as_ref() { Some(header_mutation::Operation::Write(write)) => { let name = validate_name(&write.name)?; - if is_connection_nominated(&headers, &name) { + if is_connection_nominated(connection_nominated_headers, &name) { return Err(HeaderMutationError::HopByHop { name: write.name.clone(), }); @@ -159,7 +160,7 @@ pub fn apply( } Some(header_mutation::Operation::Remove(remove)) => { let name = validate_name(&remove.name)?; - if is_connection_nominated(&headers, &name) { + if is_connection_nominated(connection_nominated_headers, &name) { return Err(HeaderMutationError::HopByHop { name: remove.name.clone(), }); @@ -246,12 +247,10 @@ fn is_protected(name: &str) -> bool { || name.starts_with("x-openshell-credential") } -fn is_connection_nominated(headers: &[(String, String)], name: &str) -> bool { - headers +fn is_connection_nominated(connection_nominated_headers: &[String], name: &str) -> bool { + connection_nominated_headers .iter() - .filter(|(header, _)| header == "connection") - .flat_map(|(_, value)| value.split(',')) - .any(|token| token.trim().eq_ignore_ascii_case(name)) + .any(|nominated| nominated.eq_ignore_ascii_case(name)) } #[cfg(test)] @@ -280,6 +279,7 @@ mod tests { #[test] fn protected_header_write_is_rejected() { let error = apply( + &[], &[], &[write( "Authorization", @@ -298,6 +298,7 @@ mod tests { #[test] fn unsafe_header_value_is_rejected() { let error = apply( + &[], &[], &[write( "x-openshell-middleware-inject", @@ -317,6 +318,7 @@ mod tests { ]; let appended = apply( &existing, + &[], &[write( "X-OpenShell-Middleware-Tag", "two", @@ -335,6 +337,7 @@ mod tests { let overwritten = apply( &existing, + &[], &[write( "X-OpenShell-Middleware-Tag", "two", @@ -352,6 +355,7 @@ mod tests { let skipped = apply( &existing, + &[], &[write( "X-OpenShell-Middleware-Tag", "two", @@ -369,13 +373,13 @@ mod tests { ("accept".to_string(), "application/json".to_string()), ("x-trace".to_string(), "two".to_string()), ]; - let updated = apply(&existing, &[remove("X-Trace")]).expect("remove visible header"); + let updated = apply(&existing, &[], &[remove("X-Trace")]).expect("remove visible header"); assert_eq!(updated, vec![("accept".into(), "application/json".into())]); } #[test] fn protected_header_remove_is_rejected_even_when_not_visible() { - let error = apply(&[], &[remove("Authorization")]).expect_err("protected removal"); + let error = apply(&[], &[], &[remove("Authorization")]).expect_err("protected removal"); assert!( error .to_string() @@ -385,11 +389,29 @@ mod tests { #[test] fn connection_nominated_header_is_protected() { - let existing = [ - ("connection".to_string(), "keep-alive, x-hop".to_string()), - ("x-hop".to_string(), "value".to_string()), - ]; - let error = apply(&existing, &[remove("X-Hop")]).expect_err("hop-by-hop removal"); - assert!(error.to_string().contains("hop-by-hop header 'X-Hop'")); + let nominated = vec!["x-openshell-middleware-tag".to_string()]; + let write_error = apply( + &[], + &nominated, + &[write( + "X-OpenShell-Middleware-Tag", + "value", + ExistingHeaderAction::Append, + )], + ) + .expect_err("hop-by-hop write"); + assert!( + write_error + .to_string() + .contains("hop-by-hop header 'X-OpenShell-Middleware-Tag'") + ); + + let remove_error = apply(&[], &nominated, &[remove("X-OpenShell-Middleware-Tag")]) + .expect_err("hop-by-hop removal"); + assert!( + remove_error + .to_string() + .contains("hop-by-hop header 'X-OpenShell-Middleware-Tag'") + ); } } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 70e7e861f5..eb0cc71313 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -24,6 +24,11 @@ use openshell_core::proto::{ use tokio::sync::OnceCell; use tonic::Request; +pub use openshell_core::middleware::{ + MAX_MIDDLEWARE_CHAIN_FINDINGS, MAX_MIDDLEWARE_CHAIN_STAGES, MAX_MIDDLEWARE_CONFIGS, + MAX_MIDDLEWARE_FINDINGS_PER_STAGE, MAX_MIDDLEWARE_SELECTOR_PATTERNS, +}; + /// Largest request or replacement body accepted by the middleware platform. pub const MAX_MIDDLEWARE_BODY_BYTES: usize = 4 * 1024 * 1024; /// Largest encoded service-specific configuration attached to one evaluation. @@ -38,8 +43,6 @@ pub const MAX_MIDDLEWARE_HEADERS: usize = 128; pub const MAX_MIDDLEWARE_HEADER_BYTES: usize = 64 * 1024; /// Largest operator-provided reason accepted in one middleware result. pub const MAX_MIDDLEWARE_REASON_BYTES: usize = 4 * 1024; -/// Largest number of findings accepted from one middleware stage. -pub const MAX_MIDDLEWARE_FINDINGS: usize = 64; /// Largest encoded individual finding accepted from one middleware stage. pub const MAX_MIDDLEWARE_FINDING_BYTES: usize = 4 * 1024; /// Largest number of metadata entries accepted from one middleware stage. @@ -56,10 +59,11 @@ const MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_CONFIG_BYTES + MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES; const MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_REASON_BYTES + MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES - + MAX_MIDDLEWARE_FINDINGS * MAX_MIDDLEWARE_FINDING_BYTES + + MAX_MIDDLEWARE_FINDINGS_PER_STAGE * MAX_MIDDLEWARE_FINDING_BYTES + MAX_MIDDLEWARE_METADATA_BYTES + MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES; -const MAX_MIDDLEWARE_ENVELOPE_BYTES: usize = +/// gRPC envelope headroom derived from every bounded non-body component. +pub const MIDDLEWARE_GRPC_ENVELOPE_BYTES: usize = if MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES > MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES { MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES } else { @@ -67,7 +71,7 @@ const MAX_MIDDLEWARE_ENVELOPE_BYTES: usize = }; /// gRPC message limit derived from the body and bounded protobuf components. pub const MIDDLEWARE_GRPC_MESSAGE_BYTES: usize = - MAX_MIDDLEWARE_BODY_BYTES + MAX_MIDDLEWARE_ENVELOPE_BYTES; + MAX_MIDDLEWARE_BODY_BYTES + MIDDLEWARE_GRPC_ENVELOPE_BYTES; const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation = SupervisorMiddlewareOperation::HttpRequest; @@ -189,6 +193,10 @@ pub struct HttpRequestInput { /// preserved as separate entries so middleware inspects every value the /// upstream will receive. pub headers: Vec<(String, String)>, + /// Lowercased names nominated by the original request's `Connection` + /// headers. Their values are not exposed to middleware, but mutations must + /// still treat these dynamically hop-by-hop fields as protected. + pub connection_nominated_headers: Vec, pub body: Vec, } @@ -564,7 +572,7 @@ fn validate_response_envelope( if mutation_bytes > MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES { return Err("header_mutation_bytes_over_capacity"); } - if result.findings.len() > MAX_MIDDLEWARE_FINDINGS { + if result.findings.len() > MAX_MIDDLEWARE_FINDINGS_PER_STAGE { return Err("response_findings_over_capacity"); } if result @@ -695,6 +703,7 @@ impl MiddlewareRegistry { /// Validate implementation-owned configuration for every middleware entry. pub async fn validate_policy_configs(&self, policy: &SandboxPolicy) -> Result<()> { + ensure_config_capacity(policy.network_middlewares.len())?; let runner = ChainRunner::from_registry(self.clone()); for config in &policy.network_middlewares { runner @@ -808,6 +817,7 @@ impl ChainRunner { } pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result> { + ensure_chain_capacity(entries.len())?; let manifests = self.manifests().await?; let mut entries = entries.to_vec(); sort_chain_entries(&mut entries); @@ -925,6 +935,7 @@ impl ChainRunner { input: HttpRequestInput, transformed_body_policy: TransformedBodyPolicy<'_>, ) -> Result { + ensure_chain_capacity(entries.len())?; let mut headers = input.headers.clone(); let mut body = input.body.clone(); let mut header_mutations = Vec::new(); @@ -1102,7 +1113,11 @@ impl ChainRunner { // Validate and apply the entire stage atomically. Under fail-open, // one malformed mutation must not leave earlier mutations from the // same response visible to later middleware. - let updated_headers = match headers::apply(&headers, &result.header_mutations) { + let updated_headers = match headers::apply( + &headers, + &input.connection_nominated_headers, + &result.header_mutations, + ) { Ok(updated) => updated, Err(error) => { let reason = service @@ -1201,6 +1216,24 @@ pub fn sort_chain_entries(entries: &mut [ChainEntry]) { }); } +fn ensure_config_capacity(count: usize) -> Result<()> { + if count > MAX_MIDDLEWARE_CONFIGS { + return Err(miette!( + "middleware config count {count} exceeds platform maximum {MAX_MIDDLEWARE_CONFIGS}" + )); + } + Ok(()) +} + +fn ensure_chain_capacity(count: usize) -> Result<()> { + if count > MAX_MIDDLEWARE_CHAIN_STAGES { + return Err(miette!( + "selected middleware stage count {count} exceeds platform maximum {MAX_MIDDLEWARE_CHAIN_STAGES}" + )); + } + Ok(()) +} + fn build_evaluation( entry: &DescribedChainEntry, binding: &MiddlewareBinding, @@ -1292,6 +1325,7 @@ mod tests { path: "/v1".into(), query: String::new(), headers: Vec::new(), + connection_nominated_headers: Vec::new(), body: body.as_bytes().to_vec(), } } @@ -1387,6 +1421,37 @@ mod tests { assert_eq!(names, vec!["alpha", "beta", "later"]); } + #[tokio::test] + async fn describe_chain_accepts_maximum_selected_stages() { + let entries: Vec<_> = (0..MAX_MIDDLEWARE_CHAIN_STAGES) + .map(|index| entry(&format!("stage-{index}"), OnError::FailClosed)) + .collect(); + + let described = builtin_runner() + .describe_chain(&entries) + .await + .expect("maximum selected stage count"); + assert_eq!(described.len(), MAX_MIDDLEWARE_CHAIN_STAGES); + } + + #[tokio::test] + async fn describe_chain_rejects_selected_stages_over_capacity() { + let entries: Vec<_> = (0..=MAX_MIDDLEWARE_CHAIN_STAGES) + .map(|index| entry(&format!("stage-{index}"), OnError::FailClosed)) + .collect(); + + let error = builtin_runner() + .describe_chain(&entries) + .await + .err() + .expect("selected stage count over capacity"); + assert!( + error + .to_string() + .contains("selected middleware stage count 11 exceeds platform maximum 10") + ); + } + #[tokio::test] async fn fail_open_allows_unavailable_middleware() { let unavailable = ChainEntry { @@ -2405,7 +2470,7 @@ mod tests { .expect("bind test middleware"); let address = listener.local_addr().expect("test middleware address"); let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); - let response_findings = (0..MAX_MIDDLEWARE_FINDINGS) + let response_findings = (0..MAX_MIDDLEWARE_FINDINGS_PER_STAGE) .map(|_| Finding { r#type: "f".repeat(1024), label: "finding".into(), @@ -2487,7 +2552,7 @@ mod tests { assert!(outcome.allowed); assert_eq!(outcome.body.len(), MAX_MIDDLEWARE_BODY_BYTES); assert_eq!(outcome.header_mutations.len(), 1); - assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_FINDINGS); + assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_FINDINGS_PER_STAGE); let _ = shutdown_tx.send(()); server_task .await @@ -2495,6 +2560,15 @@ mod tests { .expect("serve"); } + #[test] + fn grpc_envelope_headroom_matches_bounded_components() { + assert_eq!(MIDDLEWARE_GRPC_ENVELOPE_BYTES, 292 * 1024); + assert_eq!( + MIDDLEWARE_GRPC_MESSAGE_BYTES, + MAX_MIDDLEWARE_BODY_BYTES + 292 * 1024 + ); + } + #[tokio::test] async fn external_diagnostics_are_normalized_before_reaching_logs() { let secret = "sk-secret-request-value"; @@ -2583,6 +2657,58 @@ mod tests { assert!(!format!("{outcome:?}").contains(secret)); } + #[tokio::test] + async fn connection_nominated_write_and_remove_are_rejected_after_filtering() { + let mutations = [ + write_header( + "x-openshell-middleware-tag", + "value", + ExistingHeaderAction::Append, + ), + HeaderMutation { + operation: Some(header_mutation::Operation::Remove( + openshell_core::proto::RemoveHeader { + name: "x-openshell-middleware-tag".into(), + }, + )), + }, + ]; + + for mutation in mutations { + let service = Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: openshell_core::proto::HttpRequestResult { + header_mutations: vec![mutation], + ..allow_result() + }, + }); + let registry = registry_with_external(service, external_registration(4096)).await; + let mut request = input("hello"); + request.connection_nominated_headers = vec!["x-openshell-middleware-tag".into()]; + + let outcome = ChainRunner::from_registry(registry) + .evaluate( + &[ChainEntry { + name: "guard".into(), + implementation: "example/content-guard".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }], + request, + ) + .await + .expect("evaluate external middleware"); + + assert!(!outcome.allowed); + assert_eq!( + outcome.reason, + "middleware_failed: header_mutation_hop_by_hop_header" + ); + } + } + #[tokio::test] async fn finding_overflow_is_an_invalid_response_governed_by_on_error() { let registration = external_registration(4096); @@ -2590,7 +2716,7 @@ mod tests { binding_id: "example/content-guard".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { - findings: vec![Finding::default(); MAX_MIDDLEWARE_FINDINGS + 1], + findings: vec![Finding::default(); MAX_MIDDLEWARE_FINDINGS_PER_STAGE + 1], ..allow_result() }, }); @@ -2625,6 +2751,56 @@ mod tests { } } + #[tokio::test] + async fn maximum_chain_retains_findings_from_every_stage() { + let runner = ChainRunner::new(Arc::new(ScriptedService { + binding_id: "example/content-guard".into(), + max_body_bytes: 4096, + result: openshell_core::proto::HttpRequestResult { + findings: vec![ + Finding { + r#type: "example.finding".into(), + label: "Example finding".into(), + count: 1, + confidence: String::new(), + severity: "medium".into(), + }; + MAX_MIDDLEWARE_FINDINGS_PER_STAGE + ], + ..allow_result() + }, + })); + let entries: Vec<_> = (0..MAX_MIDDLEWARE_CHAIN_STAGES) + .map(|index| ChainEntry { + name: format!("guard-{index}"), + implementation: "example/content-guard".into(), + order: i32::try_from(index).expect("bounded stage index"), + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }) + .collect(); + + let outcome = runner + .evaluate(&entries, input("hello")) + .await + .expect("evaluate maximum chain"); + + assert!(outcome.allowed); + assert_eq!(outcome.applied.len(), MAX_MIDDLEWARE_CHAIN_STAGES); + assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_CHAIN_FINDINGS); + for (stage, findings) in outcome + .findings + .chunks_exact(MAX_MIDDLEWARE_FINDINGS_PER_STAGE) + .enumerate() + { + assert!( + findings + .iter() + .all(|finding| finding.middleware == format!("guard-{stage}")) + ); + } + } + #[tokio::test] async fn deny_decision_short_circuits_chain() { let runner = ChainRunner::new(Arc::new(scripted_service( diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index a988ef89ee..b4502ea7ce 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -143,8 +143,15 @@ pub async fn apply_middleware_chain_for_scheme, + connection_nominated_headers: Vec, query: String, body: Vec, ) -> openshell_supervisor_middleware::HttpRequestInput { @@ -206,6 +222,7 @@ pub(super) fn middleware_request_input( path: req.target.clone(), query, headers, + connection_nominated_headers, body, } } @@ -272,8 +289,15 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { /// Parse the raw header block into middleware-visible headers, preserving /// wire order and repeated names so middleware inspects every value the -/// upstream will receive. Credential-bearing headers are omitted. -fn safe_middleware_headers(headers: &[u8]) -> Result> { +/// upstream will receive. Credential-bearing and hop-by-hop headers are +/// omitted, while dynamically nominated names are retained separately for +/// mutation validation. +struct SafeMiddlewareHeaders { + visible: Vec<(String, String)>, + connection_nominated: Vec, +} + +fn safe_middleware_headers(headers: &[u8]) -> Result { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; let parsed: Vec<(String, String)> = header_str @@ -292,7 +316,7 @@ fn safe_middleware_headers(headers: &[u8]) -> Result> { .filter(|token| !token.is_empty()) .collect(); - Ok(parsed + let visible = parsed .into_iter() .filter(|(name, _)| { !name.is_empty() @@ -316,7 +340,13 @@ fn safe_middleware_headers(headers: &[u8]) -> Result> { && !name.starts_with("x-openshell-credential") && !connection_nominated.contains(name) }) - .collect()) + .collect(); + let mut connection_nominated: Vec<_> = connection_nominated.into_iter().collect(); + connection_nominated.sort(); + Ok(SafeMiddlewareHeaders { + visible, + connection_nominated, + }) } pub fn middleware_network_input(ctx: &L7EvalContext) -> crate::opa::NetworkInput { @@ -434,13 +464,13 @@ pub(super) fn middleware_events( .build(); events.push(event); } - // The runner rejects stages above this cap through `on_error`. Keep the - // emission-side bound as defense in depth for manually constructed or - // future outcome producers. + // Each stage and the selected chain are independently bounded by the + // runner. Keep the derived chain-wide emission bound as defense in depth + // for manually constructed or future outcome producers. for finding in outcome .findings .iter() - .take(openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS) + .take(openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_FINDINGS) { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(match finding.finding.severity.as_str() { @@ -495,7 +525,7 @@ mod tests { .expect("headers should parse"); assert_eq!( - headers, + headers.visible, vec![("x-request-id".to_string(), "request-123".to_string())] ); } @@ -515,7 +545,7 @@ mod tests { .expect("headers should parse"); assert_eq!( - headers, + headers.visible, vec![ ("x-api-key".to_string(), "first-value".to_string()), ("accept".to_string(), "application/json".to_string()), @@ -539,8 +569,9 @@ mod tests { .expect("headers should parse"); assert_eq!( - headers, + headers.visible, vec![("x-visible".to_string(), "visible-value".to_string())] ); + assert_eq!(headers.connection_nominated, vec!["keep-alive", "x-hop"]); } } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 4f51077d23..a327c1d157 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -3959,8 +3959,15 @@ network_policies: token_grant_resolver: None, }; - let input = - middleware_request_input("http", &req, &ctx, Vec::new(), String::new(), Vec::new()); + let input = middleware_request_input( + "http", + &req, + &ctx, + Vec::new(), + Vec::new(), + String::new(), + Vec::new(), + ); assert_eq!(input.scheme, "http"); } @@ -4047,9 +4054,24 @@ network_policies: assert!(serialized.contains("secret.common")); let mut bounded_outcome = outcome; - bounded_outcome.findings = vec![ - NamespacedFinding { - middleware: "external-guard".into(), + bounded_outcome.findings = (0 + ..openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES) + .flat_map(|stage| { + (0..openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS_PER_STAGE).map( + move |_| NamespacedFinding { + middleware: format!("external-guard-{stage}"), + finding: openshell_core::proto::Finding { + r#type: "example/content-guard.finding".into(), + label: "External middleware finding".into(), + count: 1, + confidence: String::new(), + severity: "medium".into(), + }, + }, + ) + }) + .chain(std::iter::once(NamespacedFinding { + middleware: "over-capacity".into(), finding: openshell_core::proto::Finding { r#type: "example/content-guard.finding".into(), label: "External middleware finding".into(), @@ -4057,17 +4079,15 @@ network_policies: confidence: String::new(), severity: "medium".into(), }, - }; - openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS - + 10 - ]; + })) + .collect(); let bounded_events = middleware_events(&ctx, &req, &bounded_outcome); assert_eq!( bounded_events .iter() .filter(|event| event.class_uid() == 2004) .count(), - openshell_supervisor_middleware::MAX_MIDDLEWARE_FINDINGS, + openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_FINDINGS, "finding emission must remain bounded even if an invalid outcome bypasses the runner" ); diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index b295129e72..c5eaf3100a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -1042,9 +1042,14 @@ async fn collect_and_rewrite_request_body( Ok(PreparedRequestBody { headers, body }) } BodyLength::Chunked => { - let body = collect_chunked_body(client, already_read, generation_guard, None) - .await - .map_err(CollectChunkedError::into_report)?; + let body = collect_chunked_body( + client, + already_read, + generation_guard, + Some(MAX_REWRITE_BODY_BYTES), + ) + .await + .map_err(CollectChunkedError::into_report)?; let (mut headers, body) = rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; headers = set_content_length(&headers, body.len())?; @@ -1234,7 +1239,7 @@ impl CollectChunkedError { fn into_report(self) -> miette::Error { match self { Self::OverCapacity => { - miette!("chunked body wire representation exceeds middleware buffer limit") + miette!("chunked body wire representation exceeds configured buffer limit") } Self::Failed(error) => error, } @@ -3602,6 +3607,84 @@ mod tests { assert_eq!(body, b"hello world"); } + #[tokio::test] + async fn credential_rewrite_rejects_aggregate_chunk_extension_overflow() { + let mut wire = Vec::new(); + let chunk = format!("1;pad={}\r\nx\r\n", "a".repeat(64)); + while wire.len() <= MAX_REWRITE_BODY_BYTES { + wire.extend_from_slice(chunk.as_bytes()); + } + wire.extend_from_slice(b"0\r\n\r\n"); + + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: Vec::new(), + body_length: BodyLength::Chunked, + }; + let headers = + b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n"; + let result = collect_and_rewrite_request_body( + &req, + &mut tokio::io::empty(), + headers, + std::str::from_utf8(headers).expect("headers"), + &wire, + None, + None, + ) + .await; + let Err(error) = result else { + panic!("aggregate chunk extensions must be bounded") + }; + assert!( + error + .to_string() + .contains("chunked body wire representation exceeds configured buffer limit"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn credential_rewrite_rejects_aggregate_chunk_trailer_overflow() { + let mut wire = b"1\r\nx\r\n0\r\n".to_vec(); + let trailer = format!("x-pad: {}\r\n", "a".repeat(64)); + while wire.len() <= MAX_REWRITE_BODY_BYTES { + wire.extend_from_slice(trailer.as_bytes()); + } + wire.extend_from_slice(b"\r\n"); + + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: Vec::new(), + body_length: BodyLength::Chunked, + }; + let headers = + b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n"; + let result = collect_and_rewrite_request_body( + &req, + &mut tokio::io::empty(), + headers, + std::str::from_utf8(headers).expect("headers"), + &wire, + None, + None, + ) + .await; + let Err(error) = result else { + panic!("aggregate chunk trailers must be bounded") + }; + assert!( + error + .to_string() + .contains("chunked body wire representation exceeds configured buffer limit"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn collect_chunked_body_reads_payload_in_blocks() { let payload_len = 64 * 1024; @@ -3730,7 +3813,7 @@ mod tests { ); assert!( !err.to_string().contains("over_capacity") - && !err.to_string().contains("exceeds middleware buffer limit"), + && !err.to_string().contains("exceeds configured buffer limit"), "protocol errors must not be reported as over-capacity: {err}" ); } diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index cd5ab98c03..e462083f73 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -883,6 +883,12 @@ fn global_middleware_entries(configs: &[regorus::Value], host: &str) -> Result= openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES { + return Err(miette::miette!( + "selected middleware stage count exceeds platform maximum {}", + openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES + )); + } entries.push(chain_entry_from_value(config)?); } } @@ -7152,6 +7158,47 @@ network_policies: ); } + fn matching_middleware_configs(count: usize) -> Vec { + (0..count) + .map(|index| { + regorus::Value::from(serde_json::json!({ + "name": format!("stage-{index}"), + "middleware": "openshell/secrets", + "endpoints": {"include": ["api.example.com"]} + })) + }) + .collect() + } + + #[test] + fn middleware_chain_accepts_maximum_selected_stages() { + let configs = matching_middleware_configs( + openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES, + ); + + let chain = + global_middleware_entries(&configs, "api.example.com").expect("maximum selected chain"); + assert_eq!( + chain.len(), + openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES + ); + } + + #[test] + fn middleware_chain_rejects_selected_stages_over_capacity() { + let configs = matching_middleware_configs( + openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES + 1, + ); + + let error = global_middleware_entries(&configs, "api.example.com") + .expect_err("selected chain over capacity"); + assert!( + error + .to_string() + .contains("selected middleware stage count exceeds platform maximum 10") + ); + } + #[test] fn middleware_chain_uses_dns_label_glob_semantics() { let data = r#" diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 3cbf940072..0a1a46bea6 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -77,11 +77,11 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. +Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. A policy accepts at most 10 middleware configs. -`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. +`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Each config accepts at most 32 combined include and exclude patterns. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. -Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may reference the same binding and run as separate stages. Config names must be unique. +Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may reference the same binding and run as separate stages. Config names must be unique. Runtime selection defensively rejects chains with more than 10 stages. See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. @@ -109,7 +109,7 @@ Every middleware binding declares the largest request or replacement body it sup - A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. - The same per-stage limit applies to request bodies and replacement bodies. -The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-body protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB reason, 64 header mutations totaling at most 64 KiB encoded, 64 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 420 KiB so every platform-valid envelope fits. +The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-body protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB reason, 64 header mutations totaling at most 64 KiB encoded, 32 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 292 KiB so every platform-valid envelope fits. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. OpenShell can apply `fail_open` to an oversized `Content-Length` before consuming body bytes. A chunked body can cross the limit only after bytes have been consumed, so OpenShell denies that request because it cannot safely resume the original stream. @@ -147,7 +147,7 @@ Middleware activity is emitted through OpenShell's OCSF logging: - A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the validated binding ID instead of service-provided per-request text. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. -- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 64 findings. Exceeding the cap is an invalid response handled through `on_error`. +- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 32 findings. Exceeding the per-stage cap is an invalid response handled through `on_error`. A maximum 10-stage chain retains and emits up to 320 findings without silently dropping findings from later stages. - Registry reload success and failure are emitted as configuration state changes. See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 8b8ac82855..e3d8eede12 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -163,7 +163,7 @@ Each service implements the supervisor middleware gRPC contract and may expose m The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. -`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 64 KiB so a maximum-size body and its protobuf envelope fit on the transport. +`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 292 KiB so a maximum-size body and its protobuf envelope fit on the transport. The service `grpc_endpoint` currently supports plaintext `http://` and TLS `https://` using the platform trust store. Custom trust roots, client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index ea3cb68cf6..e0b8723960 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -166,7 +166,7 @@ Each endpoint defines a reachable destination and optional inspection rules. | `allowed_ips` | list of string | No | CIDR or IP allowlist for SSRF override. Exact user-declared hostname endpoints may resolve to RFC 1918 private addresses without this field, but wildcard, hostless, and policy-advisor-proposed endpoints still require `allowed_ips` for private resolved IPs. Entries overlapping loopback (`127.0.0.0/8`), link-local (`169.254.0.0/16`), or unspecified (`0.0.0.0`) are rejected at load time. | | `allow_encoded_slash` | bool | No | When `true`, L7 request parsing preserves `%2F` inside path segments instead of rejecting it. Use this for registries and APIs such as npm scoped packages (`/@scope%2Fname`). Defaults to `false`. | | `websocket_credential_rewrite` | bool | No | When `true` on a `protocol: rest` or `protocol: websocket` endpoint, OpenShell rewrites credential placeholders in client-to-server WebSocket text messages after an allowed HTTP `101` upgrade. Binary frames are relayed but not rewritten. Defaults to `false`. | -| `request_body_credential_rewrite` | bool | No | When `true` on a `protocol: rest` endpoint, OpenShell rewrites credential placeholders in UTF-8 `application/json`, `application/x-www-form-urlencoded`, and `text/*` request bodies before forwarding upstream. The proxy buffers at most 256 KiB and updates `Content-Length` after rewriting. Defaults to `false`. Mutually exclusive with `credential_signing`. | +| `request_body_credential_rewrite` | bool | No | When `true` on a `protocol: rest` endpoint, OpenShell rewrites credential placeholders in UTF-8 `application/json`, `application/x-www-form-urlencoded`, and `text/*` request bodies before forwarding upstream. The proxy buffers at most 256 KiB and updates `Content-Length` after rewriting. For chunked requests, the limit counts framing, extensions, and trailers. Defaults to `false`. Mutually exclusive with `credential_signing`. | | `credential_signing` | string | No | Proxy-side credential signing mode. When set, the proxy strips the sandbox client's `Authorization` header and re-signs with real provider credentials. Values: `sigv4` (auto-detect payload mode from client headers), `sigv4:body` (buffer and hash body, max 10 MiB), `sigv4:no_body` (unsigned payload, stream body). Mutually exclusive with `request_body_credential_rewrite`. See [AWS SigV4](/providers/aws-sigv4). | | `signing_service` | string | No | AWS service name for SigV4 signing (e.g. `bedrock`, `s3`, `sts`). Required when `credential_signing` is set. | | `signing_region` | string | No | AWS region override for SigV4 signing (e.g. `us-east-1`). When omitted, the region is extracted from the endpoint hostname. Required for non-standard AWS endpoints where the region cannot be inferred. | @@ -478,7 +478,7 @@ Identifies an executable that is permitted to use the associated endpoints. **Category:** Dynamic -An ordered list of middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once by ascending `order`, with the policy-local name breaking ties, before provider credential injection. +An ordered list of up to 10 middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once by ascending `order`, with the policy-local name breaking ties, before provider credential injection. Runtime selection also enforces the 10-stage maximum. ```yaml showLineNumbers={false} network_middlewares: @@ -500,7 +500,7 @@ network_middlewares: | `order` | integer | No | Execution priority. Lower values run first; names break ties. Defaults to `0`. | | `config` | object | No | Implementation-owned configuration validated by the selected middleware. | | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | -| `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists. Exclusions take precedence. | +| `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists, limited to 32 combined patterns. Exclusions take precedence. | Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints: `*` matches exactly one DNS label and `**` matches one or more labels, so `**.example.com` covers subdomains but not `example.com` itself. Brace alternates are rejected at validation. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index a49e89c19b..45d04da7d7 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -285,7 +285,7 @@ If you set `protocol: rest` or `protocol: websocket`, you also need an allow sha Use the `websocket-credential-rewrite` endpoint option with `protocol: websocket` when the sandbox should send credential placeholders in client text frames and have OpenShell resolve them after the allowed upgrade. The option can also be used with `protocol: rest` compatibility endpoints that perform a WebSocket upgrade. It is rejected for plain L4 or `protocol: sql` endpoints. -Use the `request-body-credential-rewrite` endpoint option with `protocol: rest` when an API expects OpenShell-managed credentials in UTF-8 JSON, form, or text request bodies. OpenShell buffers up to 256 KiB, rewrites recognized credential placeholders, updates `Content-Length`, and rejects unresolved placeholders instead of forwarding them. The option is rejected for WebSocket, GraphQL, SQL, and plain L4 endpoints. +Use the `request-body-credential-rewrite` endpoint option with `protocol: rest` when an API expects OpenShell-managed credentials in UTF-8 JSON, form, or text request bodies. OpenShell buffers up to 256 KiB, rewrites recognized credential placeholders, updates `Content-Length`, and rejects unresolved placeholders instead of forwarding them. For chunked requests, the 256 KiB limit counts the complete wire representation, including framing, extensions, and trailers. The option is rejected for WebSocket, GraphQL, SQL, and plain L4 endpoints. Credential rewrite recognizes the canonical `openshell:resolve:env:KEY` placeholder form and whole-token provider-shaped aliases such as `provider-OPENSHELL-RESOLVE-ENV-API_TOKEN` when the referenced environment key exists in the configured provider credentials. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 4e4d4e053b..d7afe68e40 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -27,7 +27,8 @@ message SandboxPolicy { ProcessPolicy process = 4; // Network access policies keyed by name (e.g. "claude_code", "gitlab"). map network_policies = 5; - // Reusable supervisor middleware configs for network egress. + // Reusable supervisor middleware configs for network egress. At most 10 + // configs are accepted, and at most 10 stages can be selected per request. repeated NetworkMiddlewareConfig network_middlewares = 6; } @@ -83,7 +84,8 @@ message NetworkMiddlewareConfig { // Host selector controlling which admitted destinations use a middleware config. message MiddlewareEndpointSelector { - // Exact host or DNS glob patterns included in the selection. + // Exact host or DNS glob patterns included in the selection. Include and + // exclude accept at most 32 combined patterns. repeated string include = 1; // Exact host or DNS glob patterns removed from the selection. // Exclusions take precedence over inclusions. diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index e7a260f770..de880aa756 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -223,7 +223,8 @@ message HttpRequestResult { // Audit-safe findings produced during evaluation. For operator-run services, // OpenShell logs platform-owned fields derived from the validated binding id // rather than service-provided type, label, confidence, or metadata text. - // At most 64 findings of at most 4 KiB encoded each are accepted. + // At most 32 findings of at most 4 KiB encoded each are accepted per stage. + // A policy selects at most 10 stages, so one chain retains at most 320. repeated Finding findings = 6; // Non-secret service-defined metadata included in diagnostics. At most 64 // entries and 32 KiB of combined key/value data are accepted. From 3646c8b3a7f48c309da83411c4665b9c8a9da4e0 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 16:34:55 -0700 Subject: [PATCH 45/59] feat(middleware): configure RPC timeouts [skip ci] Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 6 + crates/openshell-core/src/middleware.rs | 93 ++++- crates/openshell-sandbox/src/lib.rs | 1 + crates/openshell-server/src/config_file.rs | 8 + crates/openshell-server/src/grpc/policy.rs | 1 + .../src/secrets.rs | 1 + .../src/lib.rs | 378 +++++++++++++++--- .../src/remote.rs | 27 +- .../src/l7/relay.rs | 2 + docs/extensibility/supervisor-middleware.mdx | 4 + docs/reference/gateway-config.mdx | 4 + proto/sandbox.proto | 4 + proto/supervisor_middleware.proto | 5 + 13 files changed, 451 insertions(+), 83 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index e20d07b7f5..58a0a15027 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -81,6 +81,12 @@ reasons, findings, and metadata each have explicit size or cardinality limits. Policies contain at most 10 middleware configs with 32 combined host selector patterns each. Runtime selection defensively allows at most 10 stages, each of which can return 32 findings, for a derived chain maximum of 320 findings. +Middleware RPCs default to a 500 ms deadline. Operators can set a service-wide +`timeout` in gateway configuration, and a service can advertise a more specific +timeout for each binding in its `Describe` manifest. Both use integer `ms` or +`s` values bounded from 10 ms through 30 s. Binding values take precedence over +the service value; `Describe` uses the service value because bindings have not +been discovered yet. Registration and manifest body limits above the platform boundary fail before request-body allocation. External per-request reason, finding text, mutation errors, and diagnostic metadata are untrusted: the supervisor maps logged diff --git a/crates/openshell-core/src/middleware.rs b/crates/openshell-core/src/middleware.rs index cc201d63e3..75ed66003b 100644 --- a/crates/openshell-core/src/middleware.rs +++ b/crates/openshell-core/src/middleware.rs @@ -1,7 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Platform-wide supervisor middleware cardinality limits. +//! Platform-wide supervisor middleware limits. + +use std::time::Duration; + +/// Default timeout for one supervisor middleware RPC. +pub const DEFAULT_MIDDLEWARE_TIMEOUT: Duration = Duration::from_millis(500); +/// Smallest operator-configured supervisor middleware RPC timeout. +pub const MIN_MIDDLEWARE_TIMEOUT: Duration = Duration::from_millis(10); +/// Largest operator-configured supervisor middleware RPC timeout. +pub const MAX_MIDDLEWARE_TIMEOUT: Duration = Duration::from_secs(30); /// Largest number of middleware configurations accepted in one sandbox policy. pub const MAX_MIDDLEWARE_CONFIGS: usize = 10; @@ -14,3 +23,85 @@ pub const MAX_MIDDLEWARE_FINDINGS_PER_STAGE: usize = 32; /// Largest number of findings retained and emitted for one complete chain. pub const MAX_MIDDLEWARE_CHAIN_FINDINGS: usize = MAX_MIDDLEWARE_CHAIN_STAGES * MAX_MIDDLEWARE_FINDINGS_PER_STAGE; + +/// Parse the middleware timeout syntax shared by gateway configuration and +/// supervisor runtime registrations. +/// +/// Values use the same compact duration form as gateway interceptors: an +/// integer followed by `ms` or `s`. +pub fn parse_middleware_timeout(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err("timeout must not be empty".to_string()); + } + let timeout = if let Some(milliseconds) = value.strip_suffix("ms") { + let milliseconds = milliseconds + .parse::() + .map_err(|_| format!("invalid timeout '{value}'"))?; + Duration::from_millis(milliseconds) + } else if let Some(seconds) = value.strip_suffix('s') { + let seconds = seconds + .parse::() + .map_err(|_| format!("invalid timeout '{value}'"))?; + Duration::from_secs(seconds) + } else { + return Err(format!( + "invalid timeout '{value}'; expected suffix ms or s" + )); + }; + + if timeout < MIN_MIDDLEWARE_TIMEOUT || timeout > MAX_MIDDLEWARE_TIMEOUT { + return Err(format!("timeout '{value}' must be between 10ms and 30s")); + } + Ok(timeout) +} + +/// Resolve an optional wire/config timeout, using the platform default when +/// the value is empty. +pub fn middleware_timeout_or_default(value: &str) -> Result { + if value.trim().is_empty() { + Ok(DEFAULT_MIDDLEWARE_TIMEOUT) + } else { + parse_middleware_timeout(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timeout_parser_matches_gateway_interceptor_duration_syntax() { + assert_eq!( + parse_middleware_timeout("500ms").unwrap(), + Duration::from_millis(500) + ); + assert_eq!( + parse_middleware_timeout("2s").unwrap(), + Duration::from_secs(2) + ); + assert!(parse_middleware_timeout("2").is_err()); + } + + #[test] + fn timeout_parser_enforces_inclusive_platform_bounds() { + assert_eq!( + parse_middleware_timeout("10ms").unwrap(), + MIN_MIDDLEWARE_TIMEOUT + ); + assert_eq!( + parse_middleware_timeout("30s").unwrap(), + MAX_MIDDLEWARE_TIMEOUT + ); + assert!(parse_middleware_timeout("9ms").is_err()); + assert!(parse_middleware_timeout("30001ms").is_err()); + } + + #[test] + fn empty_wire_timeout_uses_platform_default() { + assert_eq!( + middleware_timeout_or_default(""), + Ok(DEFAULT_MIDDLEWARE_TIMEOUT) + ); + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index e9d9f0fe57..78e5027953 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -3289,6 +3289,7 @@ filesystem_policy: name: "unavailable-guard".into(), grpc_endpoint: "http://127.0.0.1:1".into(), max_body_bytes: 1024, + ..Default::default() }; connect_middleware_registry(&[invalid_external]) .await diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 7306c80e7e..519a7e594b 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -177,6 +177,9 @@ pub struct MiddlewareServiceFileConfig { pub grpc_endpoint: String, /// Operator-owned body limit for every binding exposed by this service. pub max_body_bytes: u64, + /// Default RPC timeout using an integer with an `ms` or `s` suffix. + #[serde(default)] + pub timeout: Option, } impl From<&MiddlewareServiceFileConfig> for SupervisorMiddlewareService { @@ -185,6 +188,7 @@ impl From<&MiddlewareServiceFileConfig> for SupervisorMiddlewareService { name: config.name.clone(), grpc_endpoint: config.grpc_endpoint.clone(), max_body_bytes: config.max_body_bytes, + timeout: config.timeout.clone().unwrap_or_default(), } } } @@ -437,6 +441,7 @@ allow_unauthenticated_users = true name = "local-guard" grpc_endpoint = "http://127.0.0.1:50051" max_body_bytes = 262144 +timeout = "2s" "#; let tmp = write_tmp(toml); let file = load(tmp.path()).expect("valid middleware registration parses"); @@ -446,8 +451,11 @@ max_body_bytes = 262144 name: "local-guard".into(), grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes: 262_144, + timeout: Some("2s".into()), }] ); + let registration = SupervisorMiddlewareService::from(&file.openshell.gateway.middleware[0]); + assert_eq!(registration.timeout, "2s"); } #[test] diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 7c0a580c56..deb505a4b6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -9658,6 +9658,7 @@ mod tests { name: "local-guard".into(), grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes: 1024, + ..Default::default() }; let without = compute_config_revision(Some(&policy), &settings, PolicySource::Sandbox, &[]); diff --git a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs index b9c41f02f8..5a6d30b36b 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/secrets.rs @@ -47,6 +47,7 @@ pub fn describe() -> MiddlewareBinding { operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: MAX_BODY_BYTES, + timeout: String::new(), } } diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index eb0cc71313..1085c11c06 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -9,7 +9,9 @@ mod remote; #[cfg(test)] use std::collections::HashMap; use std::collections::{BTreeMap, HashSet}; +use std::future::Future; use std::sync::Arc; +use std::time::Duration; use miette::{Result, miette}; use prost::Message; @@ -25,8 +27,10 @@ use tokio::sync::OnceCell; use tonic::Request; pub use openshell_core::middleware::{ - MAX_MIDDLEWARE_CHAIN_FINDINGS, MAX_MIDDLEWARE_CHAIN_STAGES, MAX_MIDDLEWARE_CONFIGS, - MAX_MIDDLEWARE_FINDINGS_PER_STAGE, MAX_MIDDLEWARE_SELECTOR_PATTERNS, + DEFAULT_MIDDLEWARE_TIMEOUT, MAX_MIDDLEWARE_CHAIN_FINDINGS, MAX_MIDDLEWARE_CHAIN_STAGES, + MAX_MIDDLEWARE_CONFIGS, MAX_MIDDLEWARE_FINDINGS_PER_STAGE, MAX_MIDDLEWARE_SELECTOR_PATTERNS, + MAX_MIDDLEWARE_TIMEOUT, MIN_MIDDLEWARE_TIMEOUT, middleware_timeout_or_default, + parse_middleware_timeout, }; /// Largest request or replacement body accepted by the middleware platform. @@ -137,6 +141,7 @@ pub struct DescribedChainEntry { service: Option>, binding: Option, max_body_bytes: usize, + timeout: Duration, } impl DescribedChainEntry { @@ -148,6 +153,10 @@ impl DescribedChainEntry { self.entry.on_error } + pub fn timeout(&self) -> Duration { + self.timeout + } + /// True when this entry resolved to a registered binding and will be /// evaluated. When false, the binding is absent from the current registry /// and the entry is handled entirely by its `on_error` policy, so it @@ -278,6 +287,32 @@ struct MiddlewareServiceState { manifest: OnceCell, diagnostic_policy: MiddlewareDiagnosticPolicy, operator_max_body_bytes: Option, + operator_timeout: Duration, +} + +impl MiddlewareServiceState { + fn timeout_for_binding(&self, binding: &MiddlewareBinding) -> Result { + if binding.timeout.trim().is_empty() { + Ok(self.operator_timeout) + } else { + parse_middleware_timeout(&binding.timeout).map_err(|reason| { + miette!( + "middleware binding '{}' has invalid timeout: {reason}", + binding.id + ) + }) + } + } +} + +async fn call_with_timeout( + timeout: Duration, + operation: &'static str, + future: impl Future, tonic::Status>>, +) -> std::result::Result, tonic::Status> { + tokio::time::timeout(timeout, future).await.map_err(|_| { + tonic::Status::deadline_exceeded(format!("middleware {operation} timed out")) + })? } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -352,7 +387,7 @@ impl Default for MiddlewareRegistry { } } -fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<()> { +fn validate_registration(registration: &SupervisorMiddlewareService) -> Result { if registration.name.trim().is_empty() { return Err(miette!( "supervisor middleware registration name cannot be empty" @@ -378,7 +413,12 @@ fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<( registration.name )); } - Ok(()) + middleware_timeout_or_default(®istration.timeout).map_err(|reason| { + miette!( + "middleware registration '{}' has invalid timeout: {reason}", + registration.name + ) + }) } fn is_stable_identifier(value: &str) -> bool { @@ -443,6 +483,14 @@ fn validate_manifest_bindings( )); } let advertised = validate_body_limit(source, binding)?; + if !binding.timeout.trim().is_empty() { + parse_middleware_timeout(&binding.timeout).map_err(|reason| { + miette!( + "{source} binding '{}' has invalid timeout: {reason}", + binding.id + ) + })?; + } if operator_max_body_bytes.is_some_and(|limit| limit > advertised) { return Err(miette!( "{source} max_body_bytes ({}) exceeds binding '{}' capability ({advertised})", @@ -610,16 +658,19 @@ impl MiddlewareRegistry { let mut binding_ids = HashSet::new(); for service in in_process_services { - let manifest = service - .describe(Request::new(())) - .await - .map(tonic::Response::into_inner) - .map_err(|error| { - miette!( - "in-process middleware Describe failed: {}", - safe_reason(&error.to_string()) - ) - })?; + let manifest = call_with_timeout( + DEFAULT_MIDDLEWARE_TIMEOUT, + "Describe", + service.describe(Request::new(())), + ) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "in-process middleware Describe failed: {}", + safe_reason(&error.to_string()) + ) + })?; let source = if manifest.name.trim().is_empty() { "in-process middleware service".to_string() } else { @@ -635,11 +686,12 @@ impl MiddlewareRegistry { manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, + operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT, })); } for registration in registrations { - validate_registration(®istration)?; + let operator_timeout = validate_registration(®istration)?; if !registration_names.insert(registration.name.clone()) { return Err(miette!( "duplicate supervisor middleware registration name '{}'", @@ -661,17 +713,20 @@ impl MiddlewareRegistry { ) .await?, ); - let manifest = service - .describe(Request::new(())) - .await - .map(tonic::Response::into_inner) - .map_err(|error| { - miette!( - "middleware registration '{}' Describe failed: {}", - registration.name, - safe_reason(&error.to_string()) - ) - })?; + let manifest = call_with_timeout( + operator_timeout, + "Describe", + service.describe(Request::new(())), + ) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware registration '{}' Describe failed: {}", + registration.name, + safe_reason(&error.to_string()) + ) + })?; let described_ids = validate_external_manifest( ®istration, &manifest, @@ -687,6 +742,7 @@ impl MiddlewareRegistry { manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, operator_max_body_bytes: Some(operator_max_body_bytes), + operator_timeout, })); registered_services.push(RegisteredMiddlewareService { registration, @@ -779,6 +835,7 @@ impl ChainRunner { manifest: OnceCell::new(), diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, + operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT, })]), registered_services: Arc::new(Vec::new()), binding_ids: Arc::new(HashSet::new()), @@ -798,17 +855,19 @@ impl ChainRunner { let manifest = state .manifest .get_or_try_init(|| async { - state - .service - .describe(Request::new(())) - .await - .map(tonic::Response::into_inner) - .map_err(|error| { - miette!( - "middleware Describe failed: {}", - safe_reason(&error.to_string()) - ) - }) + call_with_timeout( + state.operator_timeout, + "Describe", + state.service.describe(Request::new(())), + ) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware Describe failed: {}", + safe_reason(&error.to_string()) + ) + }) }) .await?; manifests.push((Arc::clone(state), manifest.clone())); @@ -848,11 +907,18 @@ impl ChainRunner { }) .transpose()? .unwrap_or(0); + let timeout = service + .as_ref() + .zip(binding.as_ref()) + .map(|(state, binding)| state.timeout_for_binding(binding)) + .transpose()? + .unwrap_or(DEFAULT_MIDDLEWARE_TIMEOUT); Ok(DescribedChainEntry { entry: entry.clone(), service, binding, max_body_bytes, + timeout, }) }) .collect() @@ -869,30 +935,35 @@ impl ChainRunner { )); } let manifests = self.manifests().await?; - let Some((state, _)) = manifests.iter().find(|(_, manifest)| { + let Some((state, binding)) = manifests.iter().find_map(|(state, manifest)| { manifest .bindings .iter() - .any(|binding| binding.id == implementation) + .find(|binding| binding.id == implementation) + .map(|binding| (state, binding)) }) else { return Err(miette!( "middleware binding '{implementation}' is not registered" )); }; - let response = state - .service - .validate_config(Request::new(ValidateConfigRequest { - binding_id: implementation.into(), - config: Some(config), - })) - .await - .map(tonic::Response::into_inner) - .map_err(|error| { - miette!( - "middleware ValidateConfig failed: {}", - safe_reason(&error.to_string()) - ) - })?; + let response = call_with_timeout( + state.timeout_for_binding(binding)?, + "ValidateConfig", + state + .service + .validate_config(Request::new(ValidateConfigRequest { + binding_id: implementation.into(), + config: Some(config), + })), + ) + .await + .map(tonic::Response::into_inner) + .map_err(|error| { + miette!( + "middleware ValidateConfig failed: {}", + safe_reason(&error.to_string()) + ) + })?; if response.valid { Ok(()) } else { @@ -996,14 +1067,22 @@ impl ChainRunner { let Some(service) = entry.service.as_ref() else { unreachable!("described binding always has a service") }; - let mut result = match service - .service - .evaluate_http_request(Request::new(evaluation)) - .await + let mut result = match call_with_timeout( + entry.timeout, + "EvaluateHttpRequest", + service + .service + .evaluate_http_request(Request::new(evaluation)), + ) + .await { Ok(result) => result.into_inner(), Err(err) => { - let reason = service.diagnostic_policy.error_reason(&err); + let reason = if err.code() == tonic::Code::DeadlineExceeded { + "middleware_timeout".to_string() + } else { + service.diagnostic_policy.error_reason(&err) + }; match apply_on_error(entry, &reason, &mut applied) { OnErrorAction::FailOpen => continue, OnErrorAction::FailClosed(reason) => { @@ -1560,6 +1639,7 @@ mod tests { operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: self.max_body_bytes, + timeout: String::new(), }], })) } @@ -1590,6 +1670,57 @@ mod tests { } } + struct SlowService { + delay: Duration, + binding_timeout: String, + } + + #[tonic::async_trait] + impl SupervisorMiddleware for SlowService { + async fn describe( + &self, + _request: Request<()>, + ) -> std::result::Result, tonic::Status> { + Ok(tonic::Response::new(MiddlewareManifest { + name: "test/slow".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/slow".into(), + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: 4096, + timeout: self.binding_timeout.clone(), + }], + })) + } + + async fn validate_config( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + Ok(tonic::Response::new( + openshell_core::proto::ValidateConfigResponse { + valid: true, + reason: String::new(), + }, + )) + } + + async fn evaluate_http_request( + &self, + _request: Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + tokio::time::sleep(self.delay).await; + Ok(tonic::Response::new(allow_result())) + } + } + /// A two-binding service for exercising per-stage validation: `test/transform` /// replaces the body, `test/second` records that it ran and allows. struct TwoStageService { @@ -1607,6 +1738,7 @@ mod tests { operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 256 * 1024, + timeout: String::new(), }; Ok(tonic::Response::new(MiddlewareManifest { name: "test/two-stage".into(), @@ -1842,6 +1974,7 @@ mod tests { operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 4096, + timeout: String::new(), }], })) } @@ -1897,6 +2030,7 @@ mod tests { operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 4096, + timeout: String::new(), }], })) } @@ -2053,6 +2187,7 @@ mod tests { name: "local-guard-service".into(), grpc_endpoint: "http://127.0.0.1:50051".into(), max_body_bytes, + ..Default::default() } } @@ -2089,6 +2224,7 @@ mod tests { .expect("describe test service") .into_inner(); let operator_max_body_bytes = usize::try_from(registration.max_body_bytes).unwrap(); + let operator_timeout = validate_registration(®istration).expect("valid registration"); let binding_ids = validate_external_manifest( ®istration, &manifest, @@ -2105,12 +2241,14 @@ mod tests { manifest: builtin_manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, operator_max_body_bytes: None, + operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT, }), Arc::new(MiddlewareServiceState { service, manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, operator_max_body_bytes: Some(operator_max_body_bytes), + operator_timeout, }), ]), registered_services: Arc::new(vec![RegisteredMiddlewareService { @@ -2308,6 +2446,7 @@ mod tests { operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, + timeout: String::new(), }], }; let error = validate_external_manifest(®istration, &manifest, 4097, &mut HashSet::new()) @@ -2334,6 +2473,7 @@ mod tests { operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: u64::MAX, + timeout: String::new(), }], }; let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) @@ -2352,6 +2492,7 @@ mod tests { operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, + timeout: String::new(), }], }; let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) @@ -2370,6 +2511,7 @@ mod tests { operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, + timeout: String::new(), }], }; let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) @@ -2397,6 +2539,126 @@ mod tests { assert!(error.to_string().contains("http:// or https://")); } + #[test] + fn registration_timeout_uses_default_and_operator_override() { + let registration = external_registration(4096); + let timeout = validate_registration(®istration).expect("default timeout"); + assert_eq!(timeout, DEFAULT_MIDDLEWARE_TIMEOUT); + + let mut registration = external_registration(4096); + registration.timeout = "2s".into(); + let timeout = validate_registration(®istration).expect("operator timeout"); + assert_eq!(timeout, Duration::from_secs(2)); + } + + #[test] + fn registration_timeout_enforces_bounds() { + for timeout in ["9ms", "31s"] { + let mut registration = external_registration(4096); + registration.timeout = timeout.into(); + assert!(validate_registration(®istration).is_err()); + } + } + + #[test] + fn manifest_binding_timeout_enforces_bounds() { + let registration = external_registration(4096); + for timeout in ["9ms", "31s"] { + let manifest = MiddlewareManifest { + name: "example/service".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + id: "example/content-guard".into(), + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, + max_body_bytes: 4096, + timeout: timeout.into(), + }], + }; + let error = + validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) + .expect_err("out-of-bounds binding timeout must be rejected"); + assert!(error.to_string().contains("invalid timeout")); + } + } + + #[tokio::test] + async fn binding_timeout_override_controls_evaluation_and_on_error() { + let mut registration = external_registration(4096); + registration.timeout = "2s".into(); + let registry = registry_with_external( + Arc::new(SlowService { + delay: Duration::from_millis(50), + binding_timeout: "10ms".into(), + }), + registration, + ) + .await; + let runner = ChainRunner::from_registry(registry); + let slow_entry = |on_error| ChainEntry { + name: "slow".into(), + implementation: "example/slow".into(), + order: 0, + config: prost_types::Struct::default(), + on_error, + }; + + let described = runner + .describe_chain(&[slow_entry(OnError::FailClosed)]) + .await + .expect("describe slow binding"); + assert_eq!(described[0].timeout(), Duration::from_millis(10)); + + let closed = runner + .evaluate(&[slow_entry(OnError::FailClosed)], input("payload")) + .await + .expect("fail-closed timeout outcome"); + assert!(!closed.allowed); + assert_eq!(closed.reason, "middleware_failed: middleware_timeout"); + + let open = runner + .evaluate(&[slow_entry(OnError::FailOpen)], input("payload")) + .await + .expect("fail-open timeout outcome"); + assert!(open.allowed); + assert!(open.applied[0].failed); + } + + #[tokio::test] + async fn operator_timeout_controls_binding_without_manifest_override() { + let mut registration = external_registration(4096); + registration.timeout = "10ms".into(); + let registry = registry_with_external( + Arc::new(SlowService { + delay: Duration::from_millis(50), + binding_timeout: String::new(), + }), + registration, + ) + .await; + let runner = ChainRunner::from_registry(registry); + let slow_entry = ChainEntry { + name: "slow".into(), + implementation: "example/slow".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + + let described = runner + .describe_chain(std::slice::from_ref(&slow_entry)) + .await + .expect("describe slow binding"); + assert_eq!(described[0].timeout(), Duration::from_millis(10)); + + let outcome = runner + .evaluate(&[slow_entry], input("payload")) + .await + .expect("operator timeout outcome"); + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "middleware_failed: middleware_timeout"); + } + #[tokio::test] async fn external_registry_invokes_remote_service_over_grpc() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") diff --git a/crates/openshell-supervisor-middleware/src/remote.rs b/crates/openshell-supervisor-middleware/src/remote.rs index b95e8d9f0f..378dac3ec4 100644 --- a/crates/openshell-supervisor-middleware/src/remote.rs +++ b/crates/openshell-supervisor-middleware/src/remote.rs @@ -16,11 +16,9 @@ use tonic::{Request, Response, Status}; use crate::MIDDLEWARE_GRPC_MESSAGE_BYTES; const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); -const RPC_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Clone)] pub struct RemoteMiddlewareService { - registration_name: String, client: SupervisorMiddlewareClient, } @@ -52,27 +50,11 @@ impl RemoteMiddlewareService { ) })?; Ok(Self { - registration_name: registration_name.to_string(), client: SupervisorMiddlewareClient::new(channel) .max_decoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES) .max_encoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES), }) } - - async fn with_timeout( - &self, - operation: &'static str, - future: impl Future, Status>>, - ) -> std::result::Result, Status> { - tokio::time::timeout(RPC_TIMEOUT, future) - .await - .map_err(|_| { - Status::deadline_exceeded(format!( - "middleware '{}' {operation} timed out", - self.registration_name - )) - })? - } } #[tonic::async_trait] @@ -82,8 +64,7 @@ impl SupervisorMiddleware for RemoteMiddlewareService { request: Request<()>, ) -> std::result::Result, Status> { let mut client = self.client.clone(); - self.with_timeout("Describe", client.describe(request)) - .await + client.describe(request).await } async fn validate_config( @@ -91,8 +72,7 @@ impl SupervisorMiddleware for RemoteMiddlewareService { request: Request, ) -> std::result::Result, Status> { let mut client = self.client.clone(); - self.with_timeout("ValidateConfig", client.validate_config(request)) - .await + client.validate_config(request).await } async fn evaluate_http_request( @@ -100,7 +80,6 @@ impl SupervisorMiddleware for RemoteMiddlewareService { request: Request, ) -> std::result::Result, Status> { let mut client = self.client.clone(); - self.with_timeout("EvaluateHttpRequest", client.evaluate_http_request(request)) - .await + client.evaluate_http_request(request).await } } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index a327c1d157..682efe0037 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -3250,6 +3250,7 @@ network_policies: phase: openshell_core::proto::SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 8192, + timeout: String::new(), }], }, )) @@ -3750,6 +3751,7 @@ network_policies: operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes, + timeout: String::new(), }; Ok(tonic::Response::new(MiddlewareManifest { name: "test/two-limits".into(), diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 0a1a46bea6..c5a47b0b03 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -48,6 +48,7 @@ Start an operator-run service before starting the gateway, then add a registrati name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 +timeout = "500ms" ``` | Field | Description | @@ -55,6 +56,9 @@ max_body_bytes = 262144 | `name` | Operator-facing registration name used in diagnostics. Policies do not reference this value. | | `grpc_endpoint` | Service address reachable from both the gateway and sandbox supervisors. Supports plaintext `http://` and TLS `https://` with platform trust roots. | | `max_body_bytes` | Operator limit applied to every binding exposed by the service, up to the 4 MiB platform maximum. | +| `timeout` | Optional service-wide RPC timeout using an integer with an `ms` or `s` suffix. Defaults to `500ms`; valid values range from `10ms` through `30s`. | + +Each binding returned by `Describe` may advertise its own `timeout` using the same syntax and bounds. A binding timeout takes precedence over the gateway service setting; an omitted binding timeout inherits the service setting, and an omitted service setting uses the 500 ms platform default. OpenShell rejects an invalid timeout before accepting the manifest. The service timeout applies to `Describe`, while the effective binding timeout applies to `ValidateConfig` and `EvaluateHttpRequest`. The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e3d8eede12..ccfe140e6c 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -110,6 +110,7 @@ grpc_rate_limit_window_seconds = 60 name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 +timeout = "500ms" # Gateway listener TLS (distinct from the per-driver guest_tls_*). [openshell.gateway.tls] @@ -157,6 +158,7 @@ Register operator-run supervisor middleware services with one or more `[[openshe name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 +timeout = "500ms" ``` Each service implements the supervisor middleware gRPC contract and may expose multiple binding IDs through `Describe`. Policies reference those binding IDs, not the registration `name`. The gateway rejects duplicate binding IDs across services and prevents operator-run services from claiming the reserved `openshell/` namespace. @@ -165,6 +167,8 @@ The gateway connects to every registered service and validates `Describe` before `max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 292 KiB so a maximum-size body and its protobuf envelope fit on the transport. +`timeout` is the operator-configured service-wide RPC timeout. It accepts the same compact duration syntax as gateway interceptors: an integer followed by `ms` or `s`, such as `500ms` or `2s`. Values must be between `10ms` and `30s`, inclusive. Omit the field to use the 500 ms platform default. A binding may advertise its own `timeout` in the `Describe` manifest; that binding-specific value takes precedence over the service setting. OpenShell validates both levels before accepting the service. The effective timeout covers `ValidateConfig` and `EvaluateHttpRequest`; `Describe` uses the service timeout because binding metadata is not available yet. + The service `grpc_endpoint` currently supports plaintext `http://` and TLS `https://` using the platform trust store. Custom trust roots, client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. See [Supervisor Middleware](/extensibility/supervisor-middleware) for selection, failure, body-limit, and operational guidance. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index d7afe68e40..8236556b63 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -374,4 +374,8 @@ message SupervisorMiddlewareService { string grpc_endpoint = 2; // Operator-owned body limit applied to every binding exposed by the service. uint64 max_body_bytes = 3; + // Default RPC timeout for this service. Empty uses the platform default of + // 500ms. Values use an integer with an `ms` or `s` suffix and must be + // between 10ms and 30s. + string timeout = 4; } diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index de880aa756..8cec67805a 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -45,6 +45,11 @@ message MiddlewareBinding { SupervisorMiddlewarePhase phase = 3; // Maximum request or replacement body this binding can process. uint64 max_body_bytes = 4; + // Optional binding-specific RPC timeout. Empty uses the operator-configured + // service timeout, or the 500ms platform default when that is also omitted. + // Values use an integer with an `ms` or `s` suffix and must be between 10ms + // and 30s. + string timeout = 5; } // ValidateConfigRequest contains one policy configuration to validate. From 24fd229277b0c4b548bc0a1c868c31d67b27de88 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 17:05:35 -0700 Subject: [PATCH 46/59] fix(middleware): address review findings Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 5 +- crates/openshell-policy/src/lib.rs | 97 ++++++++++++++----- crates/openshell-policy/src/middleware.rs | 4 +- crates/openshell-server/src/grpc/policy.rs | 4 +- .../openshell-server/src/grpc/validation.rs | 2 +- crates/openshell-server/src/middleware.rs | 6 +- .../src/lib.rs | 42 ++++---- .../src/{secrets.rs => regex.rs} | 50 ++++++---- .../src/lib.rs | 20 ++-- .../src/l7/relay.rs | 28 +++--- .../src/l7/rest.rs | 48 ++++++++- .../openshell-supervisor-network/src/opa.rs | 52 +++++----- .../openshell-supervisor-network/src/proxy.rs | 2 +- docs/extensibility/supervisor-middleware.mdx | 10 +- docs/reference/policy-schema.mdx | 8 +- docs/sandboxes/policies.mdx | 16 +-- 16 files changed, 251 insertions(+), 143 deletions(-) rename crates/openshell-supervisor-middleware-builtins/src/{secrets.rs => regex.rs} (68%) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 58a0a15027..e16113392c 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -104,7 +104,10 @@ hash and registrations unchanged so the snapshot is retried. The generic registry and chain runner live in `openshell-supervisor-middleware`; first-party implementations and their config schemas live in -`openshell-supervisor-middleware-builtins`. The gateway and supervisor inject +`openshell-supervisor-middleware-builtins`. The initial `openshell/regex` +implementation is only a best-effort example with fixed regular expressions; +it is not a parser-aware secret scanner and makes no redaction guarantee. The +gateway and supervisor inject those services explicitly and discover their binding IDs through the same `Describe` contract used by external services. Reusable compiled DNS host patterns and selectors remain in `openshell-core::host_pattern`. diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 75163364df..5dae917b66 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1509,19 +1509,16 @@ network_policies: version: 1 network_middlewares: - name: global-redactor - middleware: openshell/secrets + middleware: openshell/regex order: 20 on_error: fail_open endpoints: include: ["api.example.com", "*.service.test"] exclude: ["internal.example.com"] config: - secrets: ["api_key", "authorization"] - service: - mode: redact - max_matches: 2 + mode: redact - name: secondary-redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api.example.com"] network_policies: @@ -1537,7 +1534,7 @@ network_policies: let proto = parse_sandbox_policy(yaml).expect("parse failed"); assert_eq!(proto.network_middlewares.len(), 2); assert_eq!(proto.network_middlewares[0].name, "global-redactor"); - assert_eq!(proto.network_middlewares[0].middleware, "openshell/secrets"); + assert_eq!(proto.network_middlewares[0].middleware, "openshell/regex"); assert_eq!(proto.network_middlewares[0].order, 20); assert_eq!(proto.network_middlewares[0].on_error, "fail_open"); assert_eq!( @@ -1556,13 +1553,15 @@ network_policies: .exclude, vec!["internal.example.com"] ); - assert!( + assert_eq!( proto.network_middlewares[0] .config .as_ref() .expect("config") .fields - .contains_key("service") + .get("mode") + .and_then(|value| value.kind.as_ref()), + Some(&prost_types::value::Kind::StringValue("redact".into())) ); let yaml_out = serialize_sandbox_policy(&proto).expect("serialize failed"); let reparsed = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); @@ -1859,7 +1858,7 @@ network_policies: .map(|index| { serde_json::json!({ "name": format!("middleware-{index}"), - "middleware": "openshell/secrets", + "middleware": "openshell/regex", "endpoints": {"include": ["api.example.com"]} }) }) @@ -1913,7 +1912,7 @@ network_policies: fn validate_rejects_invalid_middleware_control_fields() { let cases = [ ( - middleware_config("", "openshell/secrets"), + middleware_config("", "openshell/regex"), "name must not be empty", ), ( @@ -1922,7 +1921,7 @@ network_policies: ), ( { - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); middleware.on_error = "maybe".into(); middleware }, @@ -1930,7 +1929,7 @@ network_policies: ), ( { - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); middleware.endpoints = None; middleware }, @@ -1938,7 +1937,7 @@ network_policies: ), ( { - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); middleware.endpoints.as_mut().unwrap().include.clear(); middleware }, @@ -1967,10 +1966,10 @@ network_policies: let mut policy = restrictive_default_policy(); policy .network_middlewares - .push(middleware_config("redactor", "openshell/secrets")); + .push(middleware_config("redactor", "openshell/regex")); policy .network_middlewares - .push(middleware_config("redactor", "openshell/secrets")); + .push(middleware_config("redactor", "openshell/regex")); let violations = validate_sandbox_policy(&policy).expect_err("duplicate name"); assert!(violations.iter().any(|violation| matches!( @@ -1985,7 +1984,7 @@ network_policies: for index in 0..openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { policy.network_middlewares.push(middleware_config( &format!("middleware-{index}"), - "openshell/secrets", + "openshell/regex", )); } @@ -1998,7 +1997,7 @@ network_policies: for index in 0..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { policy.network_middlewares.push(middleware_config( &format!("middleware-{index}"), - "openshell/secrets", + "openshell/regex", )); } @@ -2013,7 +2012,7 @@ network_policies: #[test] fn validate_accepts_maximum_middleware_selector_patterns() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); let selector = middleware.endpoints.as_mut().expect("selector"); selector.exclude = vec![ "excluded.example.com".into(); @@ -2027,7 +2026,7 @@ network_policies: #[test] fn validate_rejects_middleware_selector_patterns_over_capacity() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); let selector = middleware.endpoints.as_mut().expect("selector"); selector.exclude = vec![ "excluded.example.com".into(); @@ -2049,7 +2048,7 @@ network_policies: #[test] fn validate_rejects_malformed_middleware_selector_patterns() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); middleware.endpoints.as_mut().unwrap().include = vec!["api[.example.com".into()]; policy.network_middlewares.push(middleware); @@ -2076,7 +2075,7 @@ network_policies: let mut policy = restrictive_default_policy(); policy .network_middlewares - .push(middleware_config("redactor", "openshell/secrets")); + .push(middleware_config("redactor", "openshell/regex")); policy.network_policies.insert( "api".into(), NetworkPolicyRule { @@ -2102,10 +2101,62 @@ network_policies: ))); } + #[test] + fn validate_accepts_fail_open_middleware_selector_matching_tls_skip_endpoint() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/regex"); + middleware.on_error = "fail_open".into(); + policy.network_middlewares.push(middleware); + policy.network_policies.insert( + "api".into(), + NetworkPolicyRule { + name: "api".into(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".into(), + port: 443, + tls: "skip".into(), + ..Default::default() + }], + binaries: Vec::new(), + }, + ); + + validate_sandbox_policy(&policy) + .expect("fail-open middleware may select uninspectable tls: skip traffic"); + } + + #[test] + fn validate_rejects_explicit_fail_closed_middleware_on_tls_skip_endpoint() { + let mut policy = restrictive_default_policy(); + let mut middleware = middleware_config("redactor", "openshell/regex"); + middleware.on_error = "fail_closed".into(); + policy.network_middlewares.push(middleware); + policy.network_policies.insert( + "api".into(), + NetworkPolicyRule { + name: "api".into(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".into(), + port: 443, + tls: "skip".into(), + ..Default::default() + }], + binaries: Vec::new(), + }, + ); + + let violations = validate_sandbox_policy(&policy).expect_err("tls skip conflict"); + assert!(violations.iter().any(|violation| matches!( + violation, + PolicyViolation::MiddlewareTlsSkipConflict { middleware_name, .. } + if middleware_name == "redactor" + ))); + } + #[test] fn validate_rejects_concrete_selector_overlapping_tls_skip_wildcard() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/secrets"); + let mut middleware = middleware_config("redactor", "openshell/regex"); middleware.endpoints.as_mut().unwrap().include = vec!["api.example.com".into()]; policy.network_middlewares.push(middleware); policy.network_policies.insert( diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 30f4aac182..7c294d8abe 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -263,6 +263,7 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { None }; + let requires_inspection = matches!(middleware.on_error.as_str(), "" | "fail_closed"); for (key, rule) in &policy.network_policies { let policy_name = if rule.name.is_empty() { key @@ -270,7 +271,8 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { &rule.name }; for endpoint in &rule.endpoints { - let overlaps_tls_skip = endpoint.tls == "skip" + let overlaps_tls_skip = requires_inspection + && endpoint.tls == "skip" && compiled_selector.as_ref().is_some_and(|selector| { HostPattern::new(&endpoint.host) .is_ok_and(|endpoint| selector.may_match_pattern(&endpoint)) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index deb505a4b6..bfe1dd5a2e 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -9625,8 +9625,8 @@ mod tests { let policy_a = ProtoSandboxPolicy::default(); let policy_b = ProtoSandboxPolicy { network_middlewares: vec![openshell_core::proto::NetworkMiddlewareConfig { - name: "redact-secrets".into(), - middleware: "openshell/secrets".into(), + name: "regex-redactor".into(), + middleware: "openshell/regex".into(), on_error: "fail_closed".into(), ..Default::default() }], diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 4885d62334..8c5912a2f9 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -1634,7 +1634,7 @@ mod tests { let mut policy = openshell_policy::restrictive_default_policy(); policy.network_middlewares.push(NetworkMiddlewareConfig { name: "redactor".into(), - middleware: "openshell/secrets".into(), + middleware: "openshell/regex".into(), on_error: "maybe".into(), endpoints: Some(MiddlewareEndpointSelector { include: vec!["api[.example.com".into()], diff --git a/crates/openshell-server/src/middleware.rs b/crates/openshell-server/src/middleware.rs index 603a086131..806b8dcc4a 100644 --- a/crates/openshell-server/src/middleware.rs +++ b/crates/openshell-server/src/middleware.rs @@ -52,10 +52,10 @@ mod tests { let policy = SandboxPolicy { network_middlewares: vec![NetworkMiddlewareConfig { name: "redactor".into(), - middleware: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), config: Some(prost_types::Struct { fields: std::iter::once(( - "secrets".into(), + "mode".into(), prost_types::Value { kind: Some(prost_types::value::Kind::StringValue("allow".into())), }, @@ -71,6 +71,6 @@ mod tests { .await .expect_err("invalid built-in config must fail admission"); assert_eq!(error.code(), tonic::Code::InvalidArgument); - assert!(error.message().contains("supports only secrets: redact")); + assert!(error.message().contains("supports only mode: redact")); } } diff --git a/crates/openshell-supervisor-middleware-builtins/src/lib.rs b/crates/openshell-supervisor-middleware-builtins/src/lib.rs index ca040c3783..8e414fa6e5 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/lib.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/lib.rs @@ -3,7 +3,7 @@ //! First-party in-process supervisor middleware implementations. -mod secrets; +mod regex; use std::sync::Arc; @@ -15,7 +15,7 @@ use openshell_core::proto::{ }; use tonic::{Request, Response, Status}; -pub use secrets::{BINDING_ID as BUILTIN_SECRETS, SecretsConfig, SecretsMode}; +pub use regex::{BINDING_ID as BUILTIN_REGEX, RegexConfig, RegexMode}; /// Return the first-party services that the gateway and supervisor install. pub fn services() -> Vec> { @@ -25,7 +25,7 @@ pub fn services() -> Vec> { /// Validate configuration for a first-party binding. pub fn validate_config(implementation: &str, config: &prost_types::Struct) -> Result<()> { match implementation { - BUILTIN_SECRETS => secrets::validate_config(config), + BUILTIN_REGEX => regex::validate_config(config), other => Err(miette!( "middleware implementation '{other}' is not a registered OpenShell built-in" )), @@ -34,7 +34,7 @@ pub fn validate_config(implementation: &str, config: &prost_types::Struct) -> Re fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { match evaluation.binding_id.as_str() { - BUILTIN_SECRETS => secrets::evaluate_http_request(evaluation), + BUILTIN_REGEX => regex::evaluate_http_request(evaluation), other => Err(miette!( "middleware implementation '{other}' is not a registered OpenShell built-in" )), @@ -55,7 +55,7 @@ impl SupervisorMiddleware for BuiltinMiddlewareService { Ok(Response::new(MiddlewareManifest { name: "openshell/builtins".into(), service_version: env!("CARGO_PKG_VERSION").into(), - bindings: vec![secrets::describe()], + bindings: vec![regex::describe()], })) } @@ -109,14 +109,14 @@ mod tests { } #[tokio::test] - async fn service_describes_secrets_binding() { + async fn service_describes_regex_binding() { let manifest = BuiltinMiddlewareService .describe(Request::new(())) .await .expect("describe") .into_inner(); assert_eq!(manifest.bindings.len(), 1); - assert_eq!(manifest.bindings[0].id, BUILTIN_SECRETS); + assert_eq!(manifest.bindings[0].id, BUILTIN_REGEX); assert_eq!( manifest.bindings[0].operation, SupervisorMiddlewareOperation::HttpRequest as i32 @@ -129,25 +129,25 @@ mod tests { } #[test] - fn secrets_config_defaults_to_redact() { - let config = SecretsConfig::from_struct(&prost_types::Struct::default()).unwrap(); - assert_eq!(config.secrets, SecretsMode::Redact); + fn regex_config_defaults_to_redact() { + let config = RegexConfig::from_struct(&prost_types::Struct::default()).unwrap(); + assert_eq!(config.mode, RegexMode::Redact); } #[test] - fn secrets_config_accepts_explicit_redact() { - let config = SecretsConfig::from_struct(&string_config("secrets", "redact")).unwrap(); - assert_eq!(config.secrets, SecretsMode::Redact); + fn regex_config_accepts_explicit_redact() { + let config = RegexConfig::from_struct(&string_config("mode", "redact")).unwrap(); + assert_eq!(config.mode, RegexMode::Redact); } #[test] - fn secrets_config_rejects_unsupported_or_malformed_values() { + fn regex_config_rejects_unsupported_or_malformed_values() { for config in [ - string_config("secrets", "allow"), - string_config("secret", "redact"), + string_config("mode", "allow"), + string_config("patterns", "password"), prost_types::Struct { fields: std::iter::once(( - "secrets".into(), + "mode".into(), prost_types::Value { kind: Some(prost_types::value::Kind::NumberValue(42.0)), }, @@ -155,19 +155,19 @@ mod tests { .collect(), }, ] { - assert!(validate_config(BUILTIN_SECRETS, &config).is_err()); + assert!(validate_config(BUILTIN_REGEX, &config).is_err()); } } #[test] - fn secrets_redaction_evaluates_through_binding() { + fn regex_replacement_evaluates_through_binding() { let result = evaluate_http_request(&HttpRequestEvaluation { - binding_id: BUILTIN_SECRETS.into(), + binding_id: BUILTIN_REGEX.into(), body: br#"{"password":"top-secret","token":"sk-ABCDEFGHIJKLMNOP"}"#.to_vec(), config: Some(prost_types::Struct::default()), ..Default::default() }) - .expect("evaluate secrets binding"); + .expect("evaluate regex binding"); assert_eq!(result.decision, Decision::Allow as i32); assert!(result.has_body); diff --git a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs b/crates/openshell-supervisor-middleware-builtins/src/regex.rs similarity index 68% rename from crates/openshell-supervisor-middleware-builtins/src/secrets.rs rename to crates/openshell-supervisor-middleware-builtins/src/regex.rs index 5a6d30b36b..ca5cdfb87d 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/secrets.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/regex.rs @@ -1,6 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +//! Example built-in middleware that applies a fixed set of regular-expression +//! replacements to UTF-8 request bodies. +//! +//! This is intentionally a best-effort text transformation, not a secret +//! scanner or a parser-aware redactor. It provides no guarantee that sensitive +//! values will be detected or fully removed. + use std::collections::HashMap; use std::sync::LazyLock; @@ -12,29 +19,29 @@ use openshell_core::proto::{ use regex::Regex; use serde::Deserialize; -pub const BINDING_ID: &str = "openshell/secrets"; +pub const BINDING_ID: &str = "openshell/regex"; const MAX_BODY_BYTES: u64 = 256 * 1024; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(default, deny_unknown_fields)] -pub struct SecretsConfig { - /// Redaction mode. Omitting the field selects [`SecretsMode::Redact`]. - pub secrets: SecretsMode, +pub struct RegexConfig { + /// Replacement mode. Omitting the field selects [`RegexMode::Redact`]. + pub mode: RegexMode, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "lowercase")] -pub enum SecretsMode { +pub enum RegexMode { #[default] Redact, } -impl SecretsConfig { +impl RegexConfig { pub fn from_struct(config: &prost_types::Struct) -> Result { serde_json::from_value(openshell_core::proto_struct::struct_to_json_value(config)).map_err( |error| { miette!( - "invalid {BINDING_ID} config: {error}; phase 1 supports only secrets: redact" + "invalid {BINDING_ID} config: {error}; this example supports only mode: redact" ) }, ) @@ -51,32 +58,35 @@ pub fn describe() -> MiddlewareBinding { } } -struct SecretPattern { +struct ReplacementPattern { kind: &'static str, regex: Regex, } -impl SecretPattern { +impl ReplacementPattern { fn new(kind: &'static str, pattern: &str) -> Self { Self { kind, - regex: Regex::new(pattern).expect("valid built-in secret redaction pattern"), + regex: Regex::new(pattern).expect("valid built-in replacement pattern"), } } } -static SECRET_PATTERNS: LazyLock<[SecretPattern; 2]> = LazyLock::new(|| { +// TODO: Allow policies to supply custom replacement expressions after the +// configuration contract, validation limits, and replacement semantics are +// designed. The initial example deliberately exposes only these fixed patterns. +static REPLACEMENT_PATTERNS: LazyLock<[ReplacementPattern; 2]> = LazyLock::new(|| { [ - SecretPattern::new( + ReplacementPattern::new( "keyword", r#"(?i)(api[_-]?key|access[_-]?token|secret|password)(["']?\s*[:=]\s*["'])[^"',\s}]+(["']?)"#, ), - SecretPattern::new("openai", r"(sk-[A-Za-z0-9_-]{16,})"), + ReplacementPattern::new("openai", r"(sk-[A-Za-z0-9_-]{16,})"), ] }); pub fn validate_config(config: &prost_types::Struct) -> Result<()> { - SecretsConfig::from_struct(config).map(|_| ()) + RegexConfig::from_struct(config).map(|_| ()) } pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { @@ -84,7 +94,7 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result Result Result (String, Vec<(&'static str, u32)>) { +fn apply_replacements(input: &str) -> (String, Vec<(&'static str, u32)>) { let mut output = input.to_string(); let mut matches = Vec::new(); - for pattern in SECRET_PATTERNS.iter() { + for pattern in REPLACEMENT_PATTERNS.iter() { let count = u32::try_from(pattern.regex.find_iter(&output).count()).unwrap_or(u32::MAX); if count > 0 { matches.push((pattern.kind, count)); diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 1085c11c06..92a9970a8a 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -1363,7 +1363,7 @@ mod tests { SupervisorMiddleware, SupervisorMiddlewareServer, }; use openshell_core::proto::{ExistingHeaderAction, header_mutation}; - use openshell_supervisor_middleware_builtins::{BUILTIN_SECRETS, services}; + use openshell_supervisor_middleware_builtins::{BUILTIN_REGEX, services}; use tokio_stream::wrappers::TcpListenerStream; fn builtin_runner() -> ChainRunner { @@ -1378,11 +1378,11 @@ mod tests { fn entry(name: &str, on_error: OnError) -> ChainEntry { ChainEntry { name: name.into(), - implementation: BUILTIN_SECRETS.into(), + implementation: BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct { fields: std::iter::once(( - "secrets".into(), + "mode".into(), prost_types::Value { kind: Some(prost_types::value::Kind::StringValue("redact".into())), }, @@ -1446,7 +1446,7 @@ mod tests { } #[tokio::test] - async fn redacts_common_secret_patterns() { + async fn applies_fixed_regex_replacements() { let outcome = builtin_runner() .evaluate( &[entry("redact", OnError::FailClosed)], @@ -1573,7 +1573,7 @@ mod tests { let policy = SandboxPolicy { network_middlewares: vec![NetworkMiddlewareConfig { name: "redactor".into(), - middleware: BUILTIN_SECRETS.into(), + middleware: BUILTIN_REGEX.into(), ..Default::default() }], ..Default::default() @@ -1936,7 +1936,7 @@ mod tests { fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService { ScriptedService { - binding_id: BUILTIN_SECRETS.into(), + binding_id: BUILTIN_REGEX.into(), max_body_bytes: 256 * 1024, result, } @@ -2507,7 +2507,7 @@ mod tests { name: "example/service".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "openshell/secrets".into(), + id: "openshell/regex".into(), operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, @@ -3121,7 +3121,7 @@ mod tests { #[tokio::test] async fn deny_decision_ignores_oversized_replacement_under_fail_open() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_SECRETS.into(), + binding_id: BUILTIN_REGEX.into(), max_body_bytes: 4, result: openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, @@ -3242,7 +3242,7 @@ mod tests { #[tokio::test] async fn oversized_replacement_body_honors_on_error() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_SECRETS.into(), + binding_id: BUILTIN_REGEX.into(), max_body_bytes: 4, result: openshell_core::proto::HttpRequestResult { body: b"too large".to_vec(), @@ -3277,7 +3277,7 @@ mod tests { #[tokio::test] async fn oversized_request_body_honors_on_error() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_SECRETS.into(), + binding_id: BUILTIN_REGEX.into(), max_body_bytes: 4, result: allow_result(), })); diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 682efe0037..9bc2e4a0e9 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2677,7 +2677,7 @@ network_policies: #[tokio::test] async fn l7_rest_middleware_redacts_body_before_upstream() { let (config, tunnel_engine, ctx) = - middleware_relay_context("openshell/secrets", "fail_closed"); + middleware_relay_context("openshell/regex", "fail_closed"); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -2735,7 +2735,7 @@ network_policies: #[tokio::test] async fn l7_rest_middleware_acknowledges_expect_continue_before_reading_body() { let (config, tunnel_engine, ctx) = - middleware_relay_context("openshell/secrets", "fail_closed"); + middleware_relay_context("openshell/regex", "fail_closed"); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -2855,7 +2855,7 @@ network_policies: // still forwarded on an `enforcement: audit` endpoint when the // middleware chain is healthy and allows it. let (config, tunnel_engine, ctx) = - middleware_relay_context_with_enforcement("openshell/secrets", "fail_closed", "audit"); + middleware_relay_context_with_enforcement("openshell/regex", "fail_closed", "audit"); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -3066,7 +3066,7 @@ network_policies: #[tokio::test] async fn l7_rest_middleware_over_capacity_fails_closed() { let (config, tunnel_engine, ctx) = - middleware_relay_context("openshell/secrets", "fail_closed"); + middleware_relay_context("openshell/regex", "fail_closed"); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -3141,7 +3141,7 @@ network_policies: }; let fail_open = ChainEntry { name: "m".into(), - implementation: "openshell/secrets".into(), + implementation: "openshell/regex".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailOpen, @@ -3186,7 +3186,7 @@ network_policies: let resolved = ChainEntry { name: "redact".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -3805,7 +3805,7 @@ network_policies: // guard is skipped through its own on_error, instead of the whole // chain taking the unbuffered over-capacity path. let (_config, tunnel_engine, ctx) = - middleware_relay_context("openshell/secrets", "fail_closed"); + middleware_relay_context("openshell/regex", "fail_closed"); let runner = ChainRunner::new(Arc::new(TwoLimitService)); let chain = vec![ ChainEntry { @@ -4009,10 +4009,10 @@ network_policies: body: format!(r#"{{"api_key":"{RAW_SECRET}"}}"#).into_bytes(), header_mutations: Vec::new(), findings: vec![NamespacedFinding { - middleware: "redact-secrets".into(), + middleware: "regex-redactor".into(), finding: openshell_core::proto::Finding { - r#type: "secret.common".into(), - label: "common secret pattern".into(), + r#type: "regex.keyword".into(), + label: "keyword regex match".into(), count: 1, confidence: "medium".into(), severity: "medium".into(), @@ -4020,8 +4020,8 @@ network_policies: }], metadata: BTreeMap::new(), applied: vec![MiddlewareInvocation { - name: "redact-secrets".into(), - implementation: "openshell/secrets".into(), + name: "regex-redactor".into(), + implementation: "openshell/regex".into(), decision: openshell_core::proto::Decision::Allow, transformed: true, failed: false, @@ -4053,7 +4053,7 @@ network_policies: "raw secret leaked into OCSF events: {serialized}" ); // Safe finding metadata is still present. - assert!(serialized.contains("secret.common")); + assert!(serialized.contains("regex.keyword")); let mut bounded_outcome = outcome; bounded_outcome.findings = (0 @@ -4148,7 +4148,7 @@ network_policies: let data = r#" network_middlewares: - name: request-middleware - middleware: openshell/secrets + middleware: openshell/regex on_error: fail_closed endpoints: include: ["api.example.test"] diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index c5eaf3100a..48c89609e0 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -38,7 +38,7 @@ async fn max_middleware_body_bytes() -> usize { ) .describe_chain(&[openshell_supervisor_middleware::ChainEntry { name: "test".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, @@ -978,6 +978,9 @@ pub(crate) fn rebuild_request_with_buffered_body( let mut header_bytes = set_content_length(headers, body.len())?; header_bytes = strip_header(&header_bytes, "transfer-encoding")?; + if matches!(req.body_length, BodyLength::Chunked) { + header_bytes = strip_header(&header_bytes, "trailer")?; + } header_bytes = apply_header_mutations(&header_bytes, header_mutations)?; header_bytes.extend_from_slice(body); Ok(L7Request { @@ -1054,6 +1057,7 @@ async fn collect_and_rewrite_request_body( rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; headers = set_content_length(&headers, body.len())?; headers = strip_header(&headers, "transfer-encoding")?; + headers = strip_header(&headers, "trailer")?; Ok(PreparedRequestBody { headers, body }) } } @@ -3607,6 +3611,41 @@ mod tests { assert_eq!(body, b"hello world"); } + #[tokio::test] + async fn middleware_chunked_normalization_discards_trailers_and_declaration() { + let mut raw = b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\nTrailer: X-Checksum\r\n\r\n".to_vec(); + raw.extend_from_slice(b"5\r\nhello\r\n0\r\nX-Checksum: ignored\r\n\r\n"); + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw, + body_length: BodyLength::Chunked, + }; + + let buffered = + buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, 64 * 1024) + .await + .expect("chunked request should buffer"); + let BufferResult::Buffered(buffered) = buffered else { + panic!("chunked request should fit in middleware buffer") + }; + let rebuilt = + rebuild_request_with_buffered_body(&req, &buffered.headers, &buffered.body, &[]) + .expect("chunked request should normalize"); + let forwarded = String::from_utf8(rebuilt.raw_header).expect("normalized request is UTF-8"); + + assert!(forwarded.contains("Content-Length: 5\r\n")); + assert!( + !forwarded + .to_ascii_lowercase() + .contains("transfer-encoding:") + ); + assert!(!forwarded.to_ascii_lowercase().contains("trailer:")); + assert!(!forwarded.contains("X-Checksum: ignored")); + assert!(forwarded.ends_with("hello")); + } + #[tokio::test] async fn credential_rewrite_rejects_aggregate_chunk_extension_overflow() { let mut wire = Vec::new(); @@ -6016,8 +6055,9 @@ mod tests { "POST /api/messages HTTP/1.1\r\n\ Host: api.example.com\r\n\ Authorization: Bearer {alias}\r\n\ - Transfer-Encoding: chunked\r\n\r\n\ - 5\r\nhello\r\n0\r\n\r\n", + Transfer-Encoding: chunked\r\n\ + Trailer: X-Checksum\r\n\r\n\ + 5\r\nhello\r\n0\r\nX-Checksum: ignored\r\n\r\n", ); let forwarded = relay_and_capture_with_options( @@ -6032,6 +6072,8 @@ mod tests { assert!(forwarded.contains("Authorization: Bearer provider-real-token\r\n")); assert!(forwarded.contains("Content-Length: 5\r\n")); assert!(!forwarded.contains("Transfer-Encoding: chunked\r\n")); + assert!(!forwarded.contains("Trailer: X-Checksum\r\n")); + assert!(!forwarded.contains("X-Checksum: ignored\r\n")); assert!(forwarded.ends_with("hello")); } diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index e462083f73..c47b655483 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -6637,8 +6637,8 @@ network_policies: assert!(engine.evaluate_network(&python_input).unwrap().allowed); let entry = ChainEntry { - name: "secrets".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + name: "regex".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, @@ -6699,8 +6699,8 @@ network_policies: assert!(engine.evaluate_network(&python_input).unwrap().allowed); let entry = ChainEntry { - name: "secrets".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + name: "regex".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, @@ -6731,7 +6731,7 @@ network_policies: let mut invalid = proto; invalid.network_middlewares.push(NetworkMiddlewareConfig { name: String::new(), - middleware: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), ..Default::default() }); let empty_registry = MiddlewareRegistry::connect_services(Vec::new(), Vec::new()) @@ -6754,8 +6754,8 @@ network_policies: assert!(engine.evaluate_network(&claude_input).unwrap().allowed); let entry = ChainEntry { - name: "secrets".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + name: "regex".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, @@ -7112,17 +7112,17 @@ network_policies: let data = r#" network_middlewares: - name: global-redactor - middleware: openshell/secrets + middleware: openshell/regex order: 20 endpoints: include: ["api.example.com"] - name: policy-redactor - middleware: openshell/secrets + middleware: openshell/regex order: 10 endpoints: include: ["api.example.com"] - name: endpoint-redactor - middleware: openshell/secrets + middleware: openshell/regex order: 10 endpoints: include: ["api.example.com"] @@ -7163,7 +7163,7 @@ network_policies: .map(|index| { regorus::Value::from(serde_json::json!({ "name": format!("stage-{index}"), - "middleware": "openshell/secrets", + "middleware": "openshell/regex", "endpoints": {"include": ["api.example.com"]} })) }) @@ -7204,18 +7204,18 @@ network_policies: let data = r#" network_middlewares: - name: single-label - middleware: openshell/secrets + middleware: openshell/regex order: 10 endpoints: include: ["*.Example.COM"] exclude: ["trusted.example.com"] - name: recursive - middleware: openshell/secrets + middleware: openshell/regex order: 20 endpoints: include: ["**.example.com"] - name: intra-label - middleware: openshell/secrets + middleware: openshell/regex order: 30 endpoints: include: ["*-api.example.com"] @@ -7325,7 +7325,7 @@ host_match if { r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex on_error: maybe endpoints: include: ["api.example.com"] @@ -7337,11 +7337,11 @@ network_middlewares: r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api.example.com"] - name: redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api.example.com"] "#, @@ -7352,7 +7352,7 @@ network_middlewares: r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex "#, "endpoint selector is required", ), @@ -7361,7 +7361,7 @@ network_middlewares: r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api[.example.com"] "#, @@ -7372,7 +7372,7 @@ network_middlewares: r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api.example.com"] network_policies: @@ -7391,7 +7391,7 @@ network_policies: r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex endpoints: include: ["api.example.com"] network_policies: @@ -7438,17 +7438,17 @@ network_middlewares: "not a registered OpenShell built-in", ), ( - "invalid secrets config", + "invalid regex config", r#" network_middlewares: - name: redactor - middleware: openshell/secrets + middleware: openshell/regex config: - secrets: allow + mode: allow endpoints: include: ["api.example.com"] "#, - "supports only secrets: redact", + "supports only mode: redact", ), ] { let error = @@ -7468,7 +7468,7 @@ network_middlewares: let mut policy = openshell_policy::restrictive_default_policy(); policy.network_middlewares.push(NetworkMiddlewareConfig { name: "redactor".into(), - middleware: "openshell/secrets".into(), + middleware: "openshell/regex".into(), endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { include: vec!["api[.example.com".into()], exclude: Vec::new(), diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 3e8dd32048..a45a50052a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5257,7 +5257,7 @@ network_policies: }; let chain = vec![openshell_supervisor_middleware::ChainEntry { name: "redactor".into(), - implementation: openshell_supervisor_middleware_builtins::BUILTIN_SECRETS.into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), order: 0, config: prost_types::Struct::default(), on_error: openshell_supervisor_middleware::OnError::FailClosed, diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index c5a47b0b03..dd77c2af44 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -35,7 +35,7 @@ Middleware receives the request before credential injection. Operator-run servic | Built-in | None | Defined by OpenShell | Runs inside the supervisor | | Operator-run service | Required in gateway TOML | Set by the operator, up to the service capability | Runs as a separate service reachable by the gateway and supervisors | -`openshell/secrets` is the built-in middleware currently available. It identifies common secret patterns in UTF-8 request bodies and replaces matched values before the request leaves the sandbox. Its `config` accepts one field, `secrets: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. +`openshell/regex` is an example built-in middleware. It applies a small, fixed set of regular-expression replacements to UTF-8 request bodies. This best-effort text transformation is not parser-aware and does not guarantee that it will detect or fully remove sensitive values. Its `config` accepts one field, `mode: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Custom expressions are not configurable yet. Operator-run services expose one or more binding IDs. Policies reference a binding ID, such as `example/content-guard`, rather than the gateway registration name. @@ -70,11 +70,11 @@ Add middleware configs to the top-level `network_middlewares` list: ```yaml network_middlewares: - - name: redact-secrets - middleware: openshell/secrets + - name: regex-redactor + middleware: openshell/regex order: 10 config: - secrets: redact + mode: redact on_error: fail_closed endpoints: include: ["*.example.com"] @@ -161,6 +161,6 @@ See [Logging](/observability/logging) for log access and [OCSF JSON Export](/obs - Middleware applies only to HTTP requests parsed by the supervisor. Traffic the supervisor cannot inspect, such as HTTP/2 prior knowledge or non-HTTP TCP protocols, is denied on hosts matched by a fail-closed middleware, and relayed with a detection finding when every matching middleware is `fail_open`. - The typed operation and phase are `HTTP_REQUEST/PRE_CREDENTIALS`. - Selection uses destination host include and exclude patterns. -- Required middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. +- A fail-closed middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. An all-`fail_open` match may cover the endpoint; OpenShell bypasses the middleware and emits a detection finding. - Operator-run services support plaintext `http://` and TLS `https://` endpoints. HTTPS certificates must chain to a CA in the platform trust store. - Custom trust roots, client authentication, health checks, and runtime registration are not available. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index e0b8723960..e7f065e84b 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -482,11 +482,11 @@ An ordered list of up to 10 middleware configs selected after network and L7 pol ```yaml showLineNumbers={false} network_middlewares: - - name: redact-secrets - middleware: openshell/secrets + - name: regex-redactor + middleware: openshell/regex order: 10 config: - secrets: redact + mode: redact on_error: fail_closed endpoints: include: ["*.example.com"] @@ -502,7 +502,7 @@ network_middlewares: | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | | `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists, limited to 32 combined patterns. Exclusions take precedence. | -Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints: `*` matches exactly one DNS label and `**` matches one or more labels, so `**.example.com` covers subdomains but not `example.com` itself. Brace alternates are rejected at validation. Middleware runs only on HTTP requests the supervisor parses. A selector that can require middleware on a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. +Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints: `*` matches exactly one DNS label and `**` matches one or more labels, so `**.example.com` covers subdomains but not `example.com` itself. Brace alternates are rejected at validation. Middleware runs only on HTTP requests the supervisor parses. A fail-closed selector that can cover a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic. An all-`fail_open` match may cover the endpoint; the supervisor bypasses the middleware and emits a detection finding. See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, failure behavior, body limits, and operational guidance. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 45d04da7d7..09075889c2 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -46,11 +46,11 @@ network_policies: # Dynamic: ordered middleware selected independently by admitted host. network_middlewares: - - name: redact-secrets - middleware: openshell/secrets + - name: regex-redactor + middleware: openshell/regex order: 10 config: - secrets: redact + mode: redact on_error: fail_closed endpoints: include: ["api.example.com"] @@ -77,11 +77,11 @@ Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies ```yaml network_middlewares: - - name: redact-secrets - middleware: openshell/secrets + - name: regex-redactor + middleware: openshell/regex order: 10 config: - secrets: redact + mode: redact on_error: fail_closed endpoints: include: ["*.example.com"] @@ -90,9 +90,9 @@ network_middlewares: Matching entries run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. -`openshell/secrets` is built into the supervisor. Operator-provided binding IDs must be registered before a policy can reference them. The gateway validates implementation-owned config before accepting the policy. +`openshell/regex` is an example built into the supervisor. It applies fixed regular expressions as a best-effort UTF-8 text transformation, without guarantees that sensitive values will be detected or fully removed. Custom expressions are not configurable yet. Operator-provided binding IDs must be registered before a policy can reference them. The gateway validates implementation-owned config before accepting the policy. -`on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect; policy validation rejects a required selector that can cover a `tls: skip` endpoint. +`on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect. Policy validation rejects a fail-closed selector that can cover a `tls: skip` endpoint. An all-`fail_open` match may cover the endpoint; the supervisor bypasses the middleware and emits a detection finding. See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, chain ordering, body limits, failure behavior, and operations. From fbf08d0aa76ee2b3bea79fb6ebe8ee00cf9a13c5 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 17:47:08 -0700 Subject: [PATCH 47/59] fix(config): move middleware under supervisor Signed-off-by: Piotr Mlocek --- architecture/gateway.md | 7 ++--- crates/openshell-server/src/config_file.rs | 28 +++++++++++++------- crates/openshell-server/src/lib.rs | 2 +- docs/extensibility/supervisor-middleware.mdx | 2 +- docs/reference/gateway-config.mdx | 6 ++--- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index ba8437b7f3..6cd4a89fd5 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -271,9 +271,10 @@ config path. A gateway-global policy can override sandbox-scoped policy. The sandbox supervisor polls for config revisions and hot-reloads dynamic policy when the policy engine accepts the update. -External supervisor middleware registration is operator-owned gateway -configuration. At startup the gateway connects to each service, validates its -described bindings and operator body limit, and rejects duplicate binding IDs. +External supervisor middleware registration is operator-owned configuration +under `[[openshell.supervisor.middleware]]`. At startup the gateway connects to +each service, validates its described bindings and operator body limit, and +rejects duplicate binding IDs. Before persisting a policy, the gateway asks each selected implementation to validate its config. The effective sandbox config contains only the registered services required by that policy; supervisors invoke those services directly on diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 519a7e594b..6a6659e2a7 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -56,6 +56,9 @@ pub struct OpenShellRoot { #[serde(default)] pub gateway: GatewayFileSection, + #[serde(default)] + pub supervisor: SupervisorFileSection, + /// `[openshell.drivers.]` tables — passed verbatim to each driver /// crate's `Deserialize` impl after the gateway-side inheritance merge. /// Stored as raw [`toml::Value`] so each driver can evolve its schema @@ -152,12 +155,6 @@ pub struct GatewayFileSection { #[serde(default)] pub gateway_jwt: Option, - // ── Supervisor middleware ───────────────────────────────────────────── - /// Statically registered supervisor middleware services. Registration is - /// operator-owned and changes require a gateway restart. - #[serde(default)] - pub middleware: Vec, - // ── Disallowed-in-file fields ──────────────────────────────────────── // // Captured so we can produce a friendly "set this via env/CLI instead" @@ -167,7 +164,17 @@ pub struct GatewayFileSection { pub database_url: Option, } -/// One `[[openshell.gateway.middleware]]` supervisor middleware registration. +/// `[openshell.supervisor]` section. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SupervisorFileSection { + /// Statically registered supervisor middleware services. Registration is + /// operator-owned and changes require a gateway restart. + #[serde(default)] + pub middleware: Vec, +} + +/// One `[[openshell.supervisor.middleware]]` supervisor middleware registration. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MiddlewareServiceFileConfig { @@ -437,7 +444,7 @@ allow_unauthenticated_users = true #[test] fn parses_supervisor_middleware_registration() { let toml = r#" -[[openshell.gateway.middleware]] +[[openshell.supervisor.middleware]] name = "local-guard" grpc_endpoint = "http://127.0.0.1:50051" max_body_bytes = 262144 @@ -446,7 +453,7 @@ timeout = "2s" let tmp = write_tmp(toml); let file = load(tmp.path()).expect("valid middleware registration parses"); assert_eq!( - file.openshell.gateway.middleware, + file.openshell.supervisor.middleware, vec![MiddlewareServiceFileConfig { name: "local-guard".into(), grpc_endpoint: "http://127.0.0.1:50051".into(), @@ -454,7 +461,8 @@ timeout = "2s" timeout: Some("2s".into()), }] ); - let registration = SupervisorMiddlewareService::from(&file.openshell.gateway.middleware[0]); + let registration = + SupervisorMiddlewareService::from(&file.openshell.supervisor.middleware[0]); assert_eq!(registration.timeout, "2s"); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 81b0fb666b..a534f5acca 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -233,7 +233,7 @@ pub(crate) async fn run_server( .as_ref() .map(|file| { file.openshell - .gateway + .supervisor .middleware .iter() .map(Into::into) diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index dd77c2af44..d29ff751b0 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -44,7 +44,7 @@ Operator-run services expose one or more binding IDs. Policies reference a bindi Start an operator-run service before starting the gateway, then add a registration to the local gateway TOML: ```toml -[[openshell.gateway.middleware]] +[[openshell.supervisor.middleware]] name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ccfe140e6c..2b2d9aaf1f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -106,7 +106,7 @@ grpc_rate_limit_window_seconds = 60 # Operator-run supervisor middleware. The gRPC endpoint must be reachable from # both the gateway and sandbox supervisors. -[[openshell.gateway.middleware]] +[[openshell.supervisor.middleware]] name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 @@ -151,10 +151,10 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth ## Supervisor Middleware Services -Register operator-run supervisor middleware services with one or more `[[openshell.gateway.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. +Register operator-run supervisor middleware services with one or more `[[openshell.supervisor.middleware]]` entries. Registration is static and operator-owned; changing it requires restarting the gateway. ```toml -[[openshell.gateway.middleware]] +[[openshell.supervisor.middleware]] name = "local-content-guard" grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 262144 From ba5200e773ecf4198db6eba9cc0c3313b3d18998 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 18:06:53 -0700 Subject: [PATCH 48/59] fix(middleware): address security review findings Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 16 +- .../src/lib.rs | 28 ++- .../src/regex.rs | 19 +- .../src/lib.rs | 75 +++++-- .../src/l7/middleware.rs | 31 ++- .../src/l7/rest.rs | 200 +++++++++++++----- docs/extensibility/supervisor-middleware.mdx | 4 +- docs/reference/gateway-config.mdx | 2 +- proto/supervisor_middleware.proto | 5 +- 9 files changed, 283 insertions(+), 97 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index e16113392c..60c081ce72 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -82,11 +82,11 @@ Policies contain at most 10 middleware configs with 32 combined host selector patterns each. Runtime selection defensively allows at most 10 stages, each of which can return 32 findings, for a derived chain maximum of 320 findings. Middleware RPCs default to a 500 ms deadline. Operators can set a service-wide -`timeout` in gateway configuration, and a service can advertise a more specific +`timeout` in gateway configuration, and a service can advertise a shorter timeout for each binding in its `Describe` manifest. Both use integer `ms` or -`s` values bounded from 10 ms through 30 s. Binding values take precedence over -the service value; `Describe` uses the service value because bindings have not -been discovered yet. +`s` values bounded from 10 ms through 30 s. The operator value is a ceiling, +so binding calls use the smaller value; `Describe` uses the service value +because bindings have not been discovered yet. Registration and manifest body limits above the platform boundary fail before request-body allocation. External per-request reason, finding text, mutation errors, and diagnostic metadata are untrusted: the supervisor maps logged @@ -105,8 +105,9 @@ registry and chain runner live in `openshell-supervisor-middleware`; first-party implementations and their config schemas live in `openshell-supervisor-middleware-builtins`. The initial `openshell/regex` -implementation is only a best-effort example with fixed regular expressions; -it is not a parser-aware secret scanner and makes no redaction guarantee. The +implementation is only a best-effort example for simple, self-contained token +patterns; it does not infer values from keyword assignments, is not a +parser-aware secret scanner, and makes no redaction guarantee. The gateway and supervisor inject those services explicitly and discover their binding IDs through the same `Describe` contract used by external services. Reusable compiled DNS host @@ -127,6 +128,9 @@ policy generation. Each replacement is reclassified and evaluated before the next stage runs. Hard-deny classification is shared by CONNECT relays, route-selected relays, and forward proxying; evaluator failures become structured fail-closed outcomes instead of escaping the middleware pipeline. +The shared HTTP/1 request parser rejects obsolete folded continuations, +colonless fields, whitespace before field-name colons, and invalid field-name +tokens before policy evaluation, middleware projection, or forwarding. Each middleware stage can also return ordered header write and remove operations. Rust validates and applies a stage atomically to the logical header state before the next stage runs, then replays the validated operations against diff --git a/crates/openshell-supervisor-middleware-builtins/src/lib.rs b/crates/openshell-supervisor-middleware-builtins/src/lib.rs index 8e414fa6e5..38b2018270 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/lib.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/lib.rs @@ -172,7 +172,33 @@ mod tests { assert_eq!(result.decision, Decision::Allow as i32); assert!(result.has_body); let body = String::from_utf8(result.body).unwrap(); - assert!(!body.contains("top-secret")); + assert!(body.contains("top-secret")); assert!(!body.contains("sk-ABCDEFGHIJKLMNOP")); + assert!( + result + .findings + .iter() + .all(|finding| finding.r#type != "regex.keyword") + ); + } + + #[test] + fn regex_replacement_does_not_parse_keyword_assignments() { + let body = concat!( + r#"{"password":"alpha beta","secret":"alpha,beta","api_key":"alpha\"beta"}"#, + "\npassword=alpha\nnotpassword=omega" + ); + let result = evaluate_http_request(&HttpRequestEvaluation { + binding_id: BUILTIN_REGEX.into(), + body: body.as_bytes().to_vec(), + config: Some(prost_types::Struct::default()), + ..Default::default() + }) + .expect("evaluate regex binding"); + + assert_eq!(result.decision, Decision::Allow as i32); + assert!(!result.has_body); + assert_eq!(result.body, body.as_bytes()); + assert!(result.findings.is_empty()); } } diff --git a/crates/openshell-supervisor-middleware-builtins/src/regex.rs b/crates/openshell-supervisor-middleware-builtins/src/regex.rs index ca5cdfb87d..b8872ce66f 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/regex.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/regex.rs @@ -75,15 +75,8 @@ impl ReplacementPattern { // TODO: Allow policies to supply custom replacement expressions after the // configuration contract, validation limits, and replacement semantics are // designed. The initial example deliberately exposes only these fixed patterns. -static REPLACEMENT_PATTERNS: LazyLock<[ReplacementPattern; 2]> = LazyLock::new(|| { - [ - ReplacementPattern::new( - "keyword", - r#"(?i)(api[_-]?key|access[_-]?token|secret|password)(["']?\s*[:=]\s*["'])[^"',\s}]+(["']?)"#, - ), - ReplacementPattern::new("openai", r"(sk-[A-Za-z0-9_-]{16,})"), - ] -}); +static REPLACEMENT_PATTERNS: LazyLock<[ReplacementPattern; 1]> = + LazyLock::new(|| [ReplacementPattern::new("openai", r"sk-[A-Za-z0-9_-]{16,}")]); pub fn validate_config(config: &prost_types::Struct) -> Result<()> { RegexConfig::from_struct(config).map(|_| ()) @@ -134,13 +127,7 @@ fn apply_replacements(input: &str) -> (String, Vec<(&'static str, u32)>) { } output = pattern .regex - .replace_all(&output, |captures: ®ex::Captures<'_>| { - if captures.len() >= 4 { - format!("{}{}[REDACTED]{}", &captures[1], &captures[2], &captures[3]) - } else { - "[REDACTED]".to_string() - } - }) + .replace_all(&output, "[REDACTED]") .into_owned(); } (output, matches) diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 92a9970a8a..5300422d1c 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -295,12 +295,14 @@ impl MiddlewareServiceState { if binding.timeout.trim().is_empty() { Ok(self.operator_timeout) } else { - parse_middleware_timeout(&binding.timeout).map_err(|reason| { - miette!( - "middleware binding '{}' has invalid timeout: {reason}", - binding.id - ) - }) + parse_middleware_timeout(&binding.timeout) + .map(|binding_timeout| binding_timeout.min(self.operator_timeout)) + .map_err(|reason| { + miette!( + "middleware binding '{}' has invalid timeout: {reason}", + binding.id + ) + }) } } } @@ -1469,13 +1471,13 @@ mod tests { entry("second", OnError::FailClosed), ]; let outcome = builtin_runner() - .evaluate(&entries, input(r#"password="top-secret""#)) + .evaluate(&entries, input(r#"token="sk-ABCDEFGHIJKLMNOP""#)) .await .expect("evaluate"); assert!(outcome.allowed); assert_eq!( String::from_utf8(outcome.body).expect("utf8"), - r#"password="[REDACTED]""# + r#"token="[REDACTED]""# ); assert_eq!(outcome.applied.len(), 2); } @@ -1701,6 +1703,7 @@ mod tests { tonic::Response, tonic::Status, > { + tokio::time::sleep(self.delay).await; Ok(tonic::Response::new( openshell_core::proto::ValidateConfigResponse { valid: true, @@ -2341,14 +2344,14 @@ mod tests { assert_eq!(described[1].max_body_bytes(), 1024); let outcome = runner - .evaluate_described(&described, input(r#"password="top-secret""#)) + .evaluate_described(&described, input(r#"token="sk-ABCDEFGHIJKLMNOP""#)) .await .expect("evaluate mixed chain"); assert!(outcome.allowed); assert_eq!(outcome.applied.len(), 2); assert_eq!( String::from_utf8(outcome.body).expect("utf8"), - r#"password="[REDACTED]""# + r#"token="[REDACTED]""# ); } @@ -2375,7 +2378,7 @@ mod tests { redact_entry.order = 10; let entries = [guard_entry, redact_entry]; - let body = format!("{}password=\"top-secret\"", "x".repeat(1500)); + let body = format!("{}token=\"sk-ABCDEFGHIJKLMNOP\"", "x".repeat(1500)); let outcome = runner .evaluate(&entries, input(&body)) .await @@ -2392,7 +2395,7 @@ mod tests { assert!(outcome.applied[1].transformed); let body = String::from_utf8(outcome.body).expect("utf8"); assert!(body.contains("[REDACTED]")); - assert!(!body.contains("top-secret")); + assert!(!body.contains("sk-ABCDEFGHIJKLMNOP")); } #[tokio::test] @@ -2416,7 +2419,7 @@ mod tests { }; let entries = [entry("redact", OnError::FailClosed), guard_entry]; - let body = format!("{}password=\"top-secret\"", "x".repeat(1500)); + let body = format!("{}token=\"sk-ABCDEFGHIJKLMNOP\"", "x".repeat(1500)); let outcome = runner .evaluate(&entries, input(&body)) .await @@ -2659,6 +2662,52 @@ mod tests { assert_eq!(outcome.reason, "middleware_failed: middleware_timeout"); } + #[tokio::test] + async fn operator_timeout_caps_longer_binding_timeout_for_validation_and_evaluation() { + let mut registration = external_registration(4096); + registration.timeout = "10ms".into(); + let registry = registry_with_external( + Arc::new(SlowService { + delay: Duration::from_millis(50), + binding_timeout: "2s".into(), + }), + registration, + ) + .await; + let runner = ChainRunner::from_registry(registry); + let slow_entry = ChainEntry { + name: "slow".into(), + implementation: "example/slow".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }; + + let described = runner + .describe_chain(std::slice::from_ref(&slow_entry)) + .await + .expect("describe slow binding"); + assert_eq!(described[0].timeout(), Duration::from_millis(10)); + + let validation_error = runner + .validate_config("example/slow", prost_types::Struct::default()) + .await + .expect_err("operator timeout must cap ValidateConfig"); + assert!( + validation_error + .to_string() + .contains("ValidateConfig failed") + ); + assert!(validation_error.to_string().contains("timed out")); + + let outcome = runner + .evaluate(&[slow_entry], input("payload")) + .await + .expect("operator-capped evaluation outcome"); + assert!(!outcome.allowed); + assert_eq!(outcome.reason, "middleware_failed: middleware_timeout"); + } + #[tokio::test] async fn external_registry_invokes_remote_service_over_grpc() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index b4502ea7ce..27e86c94ac 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -300,12 +300,18 @@ struct SafeMiddlewareHeaders { fn safe_middleware_headers(headers: &[u8]) -> Result { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - let parsed: Vec<(String, String)> = header_str - .lines() + crate::l7::rest::validate_http_request_header_block(header_str)?; + let header_block = header_str + .strip_suffix("\r\n\r\n") + .expect("validated header block has terminator"); + let parsed: Vec<(String, String)> = header_block + .split("\r\n") .skip(1) - .filter_map(|line| { - let (name, value) = line.split_once(':')?; - Some((name.trim().to_ascii_lowercase(), value.trim().to_string())) + .map(|line| { + let (name, value) = line + .split_once(':') + .expect("validated header field contains colon"); + (name.to_ascii_lowercase(), value.trim().to_string()) }) .collect(); let connection_nominated: HashSet = parsed @@ -574,4 +580,19 @@ mod tests { ); assert_eq!(headers.connection_nominated, vec!["keep-alive", "x-hop"]); } + + #[test] + fn middleware_headers_reject_malformed_fields_instead_of_dropping_them() { + for headers in [ + b"GET /v1 HTTP/1.1\r\nX-Test: first\r\n continued\r\n\r\n".as_slice(), + b"GET /v1 HTTP/1.1\r\nX-Test value\r\n\r\n".as_slice(), + b"GET /v1 HTTP/1.1\r\nX-Test : value\r\n\r\n".as_slice(), + b"GET /v1 HTTP/1.1\r\nX@Test: value\r\n\r\n".as_slice(), + ] { + assert!( + safe_middleware_headers(headers).is_err(), + "middleware must reject malformed header fields" + ); + } + } } diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 48c89609e0..21b1921d79 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -205,6 +205,7 @@ async fn parse_http_request( // raw bytes). This gap enables request smuggling via mutated header names. let header_str = std::str::from_utf8(&buf[..header_end]) .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + validate_http_request_header_block(header_str)?; let request_line = header_str .lines() @@ -283,6 +284,7 @@ pub(crate) fn request_from_buffered_http( + 4; let header_str = std::str::from_utf8(&raw_header[..header_end]) .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + validate_http_request_header_block(header_str)?; let body_length = parse_body_length(header_str)?; let (_, query_params) = parse_target_query(query_target)?; @@ -295,6 +297,72 @@ pub(crate) fn request_from_buffered_http( }) } +/// Validate HTTP/1 request header fields before policy evaluation or forwarding. +/// +/// This parser deliberately rejects obsolete line folding and malformed field +/// names rather than allowing downstream middleware and the upstream server to +/// interpret the same wire bytes differently. +pub(crate) fn validate_http_request_header_block(headers: &str) -> Result<()> { + let header_block = headers + .strip_suffix("\r\n\r\n") + .ok_or_else(|| miette!("HTTP request headers are missing the CRLF terminator"))?; + let mut lines = header_block.split("\r\n"); + lines + .next() + .ok_or_else(|| miette!("HTTP request is missing a request line"))?; + + for line in lines { + if line + .as_bytes() + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + return Err(miette!( + "HTTP request header continuation lines are not supported" + )); + } + let (name, _) = line + .split_once(':') + .ok_or_else(|| miette!("HTTP request header field is missing ':'"))?; + if name + .as_bytes() + .last() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + return Err(miette!( + "HTTP request header field contains whitespace before ':'" + )); + } + if name.is_empty() || !name.bytes().all(is_http_field_name_byte) { + return Err(miette!( + "HTTP request header field name is not a valid HTTP token" + )); + } + } + Ok(()) +} + +fn is_http_field_name_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) +} + /// Rebuild the request line in a raw HTTP header block with a canonicalized /// target. Called when the canonical path differs from what the client sent, /// so the upstream dispatches on the exact bytes the policy engine evaluated. @@ -1283,19 +1351,20 @@ async fn collect_chunked_body( })?; if chunk_size == 0 { - loop { - let trailer_line = - read_chunked_line(client, already_read, &mut read_state, generation_guard) - .await - .map_err(|error| { - error.context(|e| { - miette!("Chunked body ended before trailer terminator: {e}") - }) - })?; - if trailer_line.is_empty() { - return Ok(body); - } + let trailer_line = + read_chunked_line(client, already_read, &mut read_state, generation_guard) + .await + .map_err(|error| { + error.context(|e| { + miette!("Chunked body ended before trailer terminator: {e}") + }) + })?; + if trailer_line.is_empty() { + return Ok(body); } + return Err(CollectChunkedError::Failed(miette!( + "chunked request trailers are not supported when buffering or transforming request bodies" + ))); } if body.len().saturating_add(chunk_size) > max_decoded_bytes { @@ -3417,6 +3486,19 @@ mod tests { assert!(err.to_string().contains("missing the CRLF terminator")); } + #[test] + fn buffered_request_parser_rejects_malformed_header_fields() { + for raw in [ + b"GET /v1/items HTTP/1.1\r\nX-Test: first\r\n continued\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX-Test value\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX-Test : value\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX@Test: value\r\n\r\n".as_slice(), + ] { + request_from_buffered_http("GET", "/v1/items", "/v1/items", raw.to_vec()) + .expect_err("malformed buffered header fields must be rejected"); + } + } + #[test] fn parse_chunked() { let headers = @@ -3601,7 +3683,7 @@ mod tests { let mut client = tokio::io::empty(); let body = collect_chunked_body( &mut client, - b"5\r\nhello\r\n6;ext=value\r\n world\r\n0\r\nx-checksum: abc\r\n\r\n", + b"5\r\nhello\r\n6;ext=value\r\n world\r\n0\r\n\r\n", None, None, ) @@ -3612,9 +3694,9 @@ mod tests { } #[tokio::test] - async fn middleware_chunked_normalization_discards_trailers_and_declaration() { + async fn middleware_chunked_request_with_trailers_is_rejected() { let mut raw = b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\nTrailer: X-Checksum\r\n\r\n".to_vec(); - raw.extend_from_slice(b"5\r\nhello\r\n0\r\nX-Checksum: ignored\r\n\r\n"); + raw.extend_from_slice(b"5\r\nhello\r\n0\r\nX-Checksum: digest\r\n\r\n"); let req = L7Request { action: "POST".into(), target: "/api".into(), @@ -3623,27 +3705,16 @@ mod tests { body_length: BodyLength::Chunked, }; - let buffered = + let error = buffer_request_body_for_middleware(&req, &mut tokio::io::empty(), None, 64 * 1024) .await - .expect("chunked request should buffer"); - let BufferResult::Buffered(buffered) = buffered else { - panic!("chunked request should fit in middleware buffer") - }; - let rebuilt = - rebuild_request_with_buffered_body(&req, &buffered.headers, &buffered.body, &[]) - .expect("chunked request should normalize"); - let forwarded = String::from_utf8(rebuilt.raw_header).expect("normalized request is UTF-8"); - - assert!(forwarded.contains("Content-Length: 5\r\n")); + .expect_err("middleware must reject non-empty chunked trailers"); assert!( - !forwarded - .to_ascii_lowercase() - .contains("transfer-encoding:") + error.to_string().contains( + "chunked request trailers are not supported when buffering or transforming request bodies" + ), + "unexpected error: {error}" ); - assert!(!forwarded.to_ascii_lowercase().contains("trailer:")); - assert!(!forwarded.contains("X-Checksum: ignored")); - assert!(forwarded.ends_with("hello")); } #[tokio::test] @@ -3686,14 +3757,7 @@ mod tests { } #[tokio::test] - async fn credential_rewrite_rejects_aggregate_chunk_trailer_overflow() { - let mut wire = b"1\r\nx\r\n0\r\n".to_vec(); - let trailer = format!("x-pad: {}\r\n", "a".repeat(64)); - while wire.len() <= MAX_REWRITE_BODY_BYTES { - wire.extend_from_slice(trailer.as_bytes()); - } - wire.extend_from_slice(b"\r\n"); - + async fn credential_rewrite_chunked_request_with_trailers_is_rejected() { let req = L7Request { action: "POST".into(), target: "/api".into(), @@ -3702,24 +3766,24 @@ mod tests { body_length: BodyLength::Chunked, }; let headers = - b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n"; + b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\nTrailer: Digest\r\n\r\n"; let result = collect_and_rewrite_request_body( &req, &mut tokio::io::empty(), headers, std::str::from_utf8(headers).expect("headers"), - &wire, + b"1\r\nx\r\n0\r\nDigest: sha-256=:abc123:\r\n\r\n", None, None, ) .await; let Err(error) = result else { - panic!("aggregate chunk trailers must be bounded") + panic!("credential rewriting must reject non-empty chunked trailers") }; assert!( - error - .to_string() - .contains("chunked body wire representation exceeds configured buffer limit"), + error.to_string().contains( + "chunked request trailers are not supported when buffering or transforming request bodies" + ), "unexpected error: {error}" ); } @@ -3987,6 +4051,43 @@ mod tests { assert!(result.is_err(), "Must reject headers with invalid UTF-8"); } + #[tokio::test] + async fn reject_malformed_header_fields_before_forwarding() { + let cases = [ + ( + "space continuation", + b"GET /api HTTP/1.1\r\nX-Test: first\r\n continued\r\nHost: x\r\n\r\n".as_slice(), + ), + ( + "tab continuation", + b"GET /api HTTP/1.1\r\nX-Test: first\r\n\tcontinued\r\nHost: x\r\n\r\n".as_slice(), + ), + ( + "missing colon", + b"GET /api HTTP/1.1\r\nX-Test value\r\nHost: x\r\n\r\n".as_slice(), + ), + ( + "whitespace before colon", + b"GET /api HTTP/1.1\r\nX-Test : value\r\nHost: x\r\n\r\n".as_slice(), + ), + ( + "invalid field-name token", + b"GET /api HTTP/1.1\r\nX@Test: value\r\nHost: x\r\n\r\n".as_slice(), + ), + ]; + + for (case, raw) in cases { + let (mut client, mut writer) = tokio::io::duplex(4096); + writer.write_all(raw).await.unwrap(); + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; + assert!(result.is_err(), "{case} must be rejected before forwarding"); + } + } + /// SEC-009: Reject unsupported HTTP versions. #[tokio::test] async fn reject_invalid_http_version() { @@ -6055,9 +6156,8 @@ mod tests { "POST /api/messages HTTP/1.1\r\n\ Host: api.example.com\r\n\ Authorization: Bearer {alias}\r\n\ - Transfer-Encoding: chunked\r\n\ - Trailer: X-Checksum\r\n\r\n\ - 5\r\nhello\r\n0\r\nX-Checksum: ignored\r\n\r\n", + Transfer-Encoding: chunked\r\n\r\n\ + 5\r\nhello\r\n0\r\n\r\n", ); let forwarded = relay_and_capture_with_options( @@ -6072,8 +6172,6 @@ mod tests { assert!(forwarded.contains("Authorization: Bearer provider-real-token\r\n")); assert!(forwarded.contains("Content-Length: 5\r\n")); assert!(!forwarded.contains("Transfer-Encoding: chunked\r\n")); - assert!(!forwarded.contains("Trailer: X-Checksum\r\n")); - assert!(!forwarded.contains("X-Checksum: ignored\r\n")); assert!(forwarded.ends_with("hello")); } diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index d29ff751b0..1c060ecb6b 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -35,7 +35,7 @@ Middleware receives the request before credential injection. Operator-run servic | Built-in | None | Defined by OpenShell | Runs inside the supervisor | | Operator-run service | Required in gateway TOML | Set by the operator, up to the service capability | Runs as a separate service reachable by the gateway and supervisors | -`openshell/regex` is an example built-in middleware. It applies a small, fixed set of regular-expression replacements to UTF-8 request bodies. This best-effort text transformation is not parser-aware and does not guarantee that it will detect or fully remove sensitive values. Its `config` accepts one field, `mode: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Custom expressions are not configurable yet. +`openshell/regex` is an example built-in middleware. It replaces only simple, self-contained token patterns in UTF-8 request bodies; the initial pattern recognizes `sk-` tokens. It does not infer values from keyword assignments such as JSON `password` fields. This best-effort text transformation is not parser-aware and does not guarantee that it will detect or fully remove sensitive values. Its `config` accepts one field, `mode: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Custom expressions are not configurable yet. Operator-run services expose one or more binding IDs. Policies reference a binding ID, such as `example/content-guard`, rather than the gateway registration name. @@ -58,7 +58,7 @@ timeout = "500ms" | `max_body_bytes` | Operator limit applied to every binding exposed by the service, up to the 4 MiB platform maximum. | | `timeout` | Optional service-wide RPC timeout using an integer with an `ms` or `s` suffix. Defaults to `500ms`; valid values range from `10ms` through `30s`. | -Each binding returned by `Describe` may advertise its own `timeout` using the same syntax and bounds. A binding timeout takes precedence over the gateway service setting; an omitted binding timeout inherits the service setting, and an omitted service setting uses the 500 ms platform default. OpenShell rejects an invalid timeout before accepting the manifest. The service timeout applies to `Describe`, while the effective binding timeout applies to `ValidateConfig` and `EvaluateHttpRequest`. +Each binding returned by `Describe` may advertise a shorter `timeout` using the same syntax and bounds. The operator-configured service timeout is a ceiling: OpenShell uses the smaller of the binding and service values. An omitted binding timeout inherits the service setting, and an omitted service setting uses the 500 ms platform default. OpenShell rejects an invalid timeout before accepting the manifest. The service timeout applies to `Describe`, while the effective binding timeout applies to `ValidateConfig` and `EvaluateHttpRequest`. The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 2b2d9aaf1f..d2bd322c32 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -167,7 +167,7 @@ The gateway connects to every registered service and validates `Describe` before `max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 292 KiB so a maximum-size body and its protobuf envelope fit on the transport. -`timeout` is the operator-configured service-wide RPC timeout. It accepts the same compact duration syntax as gateway interceptors: an integer followed by `ms` or `s`, such as `500ms` or `2s`. Values must be between `10ms` and `30s`, inclusive. Omit the field to use the 500 ms platform default. A binding may advertise its own `timeout` in the `Describe` manifest; that binding-specific value takes precedence over the service setting. OpenShell validates both levels before accepting the service. The effective timeout covers `ValidateConfig` and `EvaluateHttpRequest`; `Describe` uses the service timeout because binding metadata is not available yet. +`timeout` is the operator-configured service-wide RPC timeout. It accepts the same compact duration syntax as gateway interceptors: an integer followed by `ms` or `s`, such as `500ms` or `2s`. Values must be between `10ms` and `30s`, inclusive. Omit the field to use the 500 ms platform default. A binding may advertise a shorter `timeout` in the `Describe` manifest, but it cannot extend the operator-configured deadline; OpenShell uses the smaller value. OpenShell validates both levels before accepting the service. The effective timeout covers `ValidateConfig` and `EvaluateHttpRequest`; `Describe` uses the service timeout because binding metadata is not available yet. The service `grpc_endpoint` currently supports plaintext `http://` and TLS `https://` using the platform trust store. Custom trust roots, client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index 8cec67805a..ca63b2dfe5 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -47,8 +47,9 @@ message MiddlewareBinding { uint64 max_body_bytes = 4; // Optional binding-specific RPC timeout. Empty uses the operator-configured // service timeout, or the 500ms platform default when that is also omitted. - // Values use an integer with an `ms` or `s` suffix and must be between 10ms - // and 30s. + // A non-empty value may shorten but cannot extend the operator timeout. + // Values use an integer with an `ms` or `s` suffix and must be between + // 10ms and 30s. string timeout = 5; } From 19043041c0d1ffe46fe9039afb6f55c64d8709df Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 19:33:29 -0700 Subject: [PATCH 49/59] fix(middleware): close remaining inspection gaps Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 16 +- crates/openshell-server/src/grpc/policy.rs | 82 ++++++++ .../src/l7/middleware.rs | 18 +- .../src/l7/rest.rs | 196 ++++++++++++++++-- .../openshell-supervisor-network/src/proxy.rs | 132 +++++++++++- docs/extensibility/supervisor-middleware.mdx | 2 +- 6 files changed, 394 insertions(+), 52 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 60c081ce72..3f197dae0e 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -119,7 +119,9 @@ implementation-owned config before gateway admission. The supervisor's local-file path supplies its built-in catalog to the same JSON projection so it retains early config validation. Rego exposes the middleware list as policy data, but Rust performs selector validation, overlap detection, matching, chain -ordering, implementation discovery, and config validation. +ordering, implementation discovery, and config validation. When provider layers +are composed just in time, the gateway reruns structural validation on the +complete effective policy before delivering it to a supervisor. The Rust HTTP pipeline makes transformed-body handling explicit for every middleware invocation: body-independent protocols select a no-recheck mode, @@ -128,9 +130,12 @@ policy generation. Each replacement is reclassified and evaluated before the next stage runs. Hard-deny classification is shared by CONNECT relays, route-selected relays, and forward proxying; evaluator failures become structured fail-closed outcomes instead of escaping the middleware pipeline. -The shared HTTP/1 request parser rejects obsolete folded continuations, -colonless fields, whitespace before field-name colons, and invalid field-name -tokens before policy evaluation, middleware projection, or forwarding. +The shared HTTP/1 request parser validates raw headers at ingress and rejects +obsolete folded continuations, colonless fields, whitespace before field-name +colons, invalid field-name tokens, invalid UTF-8, and control bytes in field +values before policy evaluation, middleware projection, or forwarding. Request +framing accepts either `Content-Length` or exactly one `chunked` transfer coding; +unsupported transfer codings and every CL/TE combination fail closed. Each middleware stage can also return ordered header write and remove operations. Rust validates and applies a stage atomically to the logical header state before the next stage runs, then replays the validated operations against @@ -138,7 +143,8 @@ the raw request before credential injection. Credential, routing, framing, and hop-by-hop headers remain supervisor-owned and cannot be mutated. Names dynamically nominated by the original request's `Connection` header are carried separately after header filtering so write and remove mutations cannot -reintroduce them. +reintroduce them. The proxy also removes nominated fields before forwarding; +only a validated WebSocket handshake preserves the required `Upgrade` pair. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index bfe1dd5a2e..2292e99dbd 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1241,6 +1241,12 @@ pub(super) async fn handle_get_sandbox_config( profile_provider_policy_layers(state.store.as_ref(), &sandbox_provider_names).await?; if !provider_layers.is_empty() { let effective_policy = compose_effective_policy(source_policy, &provider_layers); + validate_policy_safety(&effective_policy).map_err(|error| { + Status::failed_precondition(format!( + "provider composition produced an invalid effective policy: {}", + error.message() + )) + })?; policy_hash = deterministic_policy_hash(&effective_policy); policy = Some(effective_policy); } @@ -4818,6 +4824,82 @@ mod tests { ); } + #[tokio::test] + async fn sandbox_config_rejects_invalid_provider_composed_policy() { + use openshell_core::proto::{ + MiddlewareEndpointSelector, NetworkMiddlewareConfig, ProviderProfile, + ProviderProfileCategory, StoredProviderProfile, + }; + + let state = test_server_state().await; + enable_providers_v2(&state).await; + + let profile = StoredProviderProfile { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "profile-tls-skip".to_string(), + name: "tls-skip".to_string(), + created_at_ms: 1_000_000, + labels: HashMap::new(), + resource_version: 0, + }), + profile: Some(ProviderProfile { + id: "tls-skip".to_string(), + display_name: "TLS skip".to_string(), + category: ProviderProfileCategory::Other as i32, + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }], + ..Default::default() + }), + }; + state.store.put_message(&profile).await.unwrap(); + state + .store + .put_message(&test_provider("work-tls-skip", "tls-skip")) + .await + .unwrap(); + + let policy = ProtoSandboxPolicy { + network_middlewares: vec![NetworkMiddlewareConfig { + name: "redactor".to_string(), + middleware: "openshell/regex".to_string(), + on_error: "fail_closed".to_string(), + endpoints: Some(MiddlewareEndpointSelector { + include: vec!["api.example.com".to_string()], + exclude: Vec::new(), + }), + ..Default::default() + }], + ..Default::default() + }; + state + .store + .put_message(&test_sandbox( + "sb-invalid-composed-policy", + "invalid-composed-policy", + policy, + vec!["work-tls-skip".to_string()], + )) + .await + .unwrap(); + + let error = handle_get_sandbox_config( + &state, + with_user(Request::new(GetSandboxConfigRequest { + sandbox_id: "sb-invalid-composed-policy".to_string(), + })), + ) + .await + .expect_err("invalid composed policy must not be delivered"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("provider composition")); + assert!(error.message().contains("tls: skip")); + } + #[tokio::test] async fn sandbox_config_skips_profileless_provider_types_when_v2_enabled() { let state = test_server_state().await; diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 27e86c94ac..3998460db0 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -10,7 +10,6 @@ use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; -use std::collections::HashSet; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncWrite}; @@ -298,13 +297,15 @@ struct SafeMiddlewareHeaders { } fn safe_middleware_headers(headers: &[u8]) -> Result { + crate::l7::rest::validate_http_request_header_block(headers)?; let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - crate::l7::rest::validate_http_request_header_block(header_str)?; let header_block = header_str .strip_suffix("\r\n\r\n") .expect("validated header block has terminator"); - let parsed: Vec<(String, String)> = header_block + let connection_nominated = crate::l7::rest::connection_nominated_header_names(headers)?; + + let visible = header_block .split("\r\n") .skip(1) .map(|line| { @@ -313,17 +314,6 @@ fn safe_middleware_headers(headers: &[u8]) -> Result { .expect("validated header field contains colon"); (name.to_ascii_lowercase(), value.trim().to_string()) }) - .collect(); - let connection_nominated: HashSet = parsed - .iter() - .filter(|(name, _)| name == "connection") - .flat_map(|(_, value)| value.split(',')) - .map(|token| token.trim().to_ascii_lowercase()) - .filter(|token| !token.is_empty()) - .collect(); - - let visible = parsed - .into_iter() .filter(|(name, _)| { !name.is_empty() && !matches!( diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 21b1921d79..8524eb957a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -203,9 +203,9 @@ async fn parse_http_request( // bytes with U+FFFD, creating an interpretation gap between this proxy // (which parses the lossy string) and upstream servers (which receive the // raw bytes). This gap enables request smuggling via mutated header names. + validate_http_request_header_block(&buf[..header_end])?; let header_str = std::str::from_utf8(&buf[..header_end]) .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - validate_http_request_header_block(header_str)?; let request_line = header_str .lines() @@ -282,9 +282,9 @@ pub(crate) fn request_from_buffered_http( .position(|window| window == b"\r\n\r\n") .ok_or_else(|| miette!("HTTP request headers are missing the CRLF terminator"))? + 4; + validate_http_request_header_block(&raw_header[..header_end])?; let header_str = std::str::from_utf8(&raw_header[..header_end]) .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - validate_http_request_header_block(header_str)?; let body_length = parse_body_length(header_str)?; let (_, query_params) = parse_target_query(query_target)?; @@ -302,7 +302,9 @@ pub(crate) fn request_from_buffered_http( /// This parser deliberately rejects obsolete line folding and malformed field /// names rather than allowing downstream middleware and the upstream server to /// interpret the same wire bytes differently. -pub(crate) fn validate_http_request_header_block(headers: &str) -> Result<()> { +pub(crate) fn validate_http_request_header_block(headers: &[u8]) -> Result<()> { + let headers = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; let header_block = headers .strip_suffix("\r\n\r\n") .ok_or_else(|| miette!("HTTP request headers are missing the CRLF terminator"))?; @@ -310,6 +312,7 @@ pub(crate) fn validate_http_request_header_block(headers: &str) -> Result<()> { lines .next() .ok_or_else(|| miette!("HTTP request is missing a request line"))?; + let mut connection_nominated = HashSet::new(); for line in lines { if line @@ -321,7 +324,7 @@ pub(crate) fn validate_http_request_header_block(headers: &str) -> Result<()> { "HTTP request header continuation lines are not supported" )); } - let (name, _) = line + let (name, value) = line .split_once(':') .ok_or_else(|| miette!("HTTP request header field is missing ':'"))?; if name @@ -338,6 +341,32 @@ pub(crate) fn validate_http_request_header_block(headers: &str) -> Result<()> { "HTTP request header field name is not a valid HTTP token" )); } + if !value.bytes().all(is_http_field_value_byte) { + return Err(miette!( + "HTTP request header field value contains an invalid control byte" + )); + } + if name.eq_ignore_ascii_case("connection") { + for token in value.split(',') { + let token = token.trim(); + if token.is_empty() || !token.bytes().all(is_http_field_name_byte) { + return Err(miette!( + "HTTP Connection header contains an invalid option token" + )); + } + connection_nominated.insert(token.to_ascii_lowercase()); + } + } + } + if connection_nominated.iter().any(|name| { + matches!( + name.as_str(), + "host" | "content-length" | "transfer-encoding" + ) + }) { + return Err(miette!( + "HTTP Connection header nominates a request framing or routing field" + )); } Ok(()) } @@ -363,6 +392,10 @@ fn is_http_field_name_byte(byte: u8) -> bool { ) } +fn is_http_field_value_byte(byte: u8) -> bool { + byte == b'\t' || (b' '..=b'~').contains(&byte) || byte >= 0x80 +} + /// Rebuild the request line in a raw HTTP header block with a canonicalized /// target. Called when the canonical path differs from what the client sent, /// so the upstream dispatches on the exact bytes the policy engine evaluated. @@ -565,6 +598,7 @@ where let header_str = std::str::from_utf8(&req.raw_header[..header_end]) .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; let client_requested_upgrade = client_requested_upgrade(header_str); + let is_websocket_request = request_is_websocket_upgrade(&req.raw_header[..header_end]); let websocket_request = if options.websocket_extensions == WebSocketExtensionMode::Preserve { None } else { @@ -587,6 +621,7 @@ where options.websocket_extensions, websocket_request.is_some(), )?; + let header_bytes = strip_connection_nominated_headers(&header_bytes, is_websocket_request)?; let websocket_response = websocket_request .as_ref() @@ -1608,6 +1643,44 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { Ok(out.into_bytes()) } +pub(crate) fn connection_nominated_header_names(headers: &[u8]) -> Result> { + validate_http_request_header_block(headers)?; + let header_str = + std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + Ok(header_str + .strip_suffix("\r\n\r\n") + .expect("validated header block has terminator") + .split("\r\n") + .skip(1) + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("connection")) + .flat_map(|(_, value)| value.split(',')) + .map(|token| token.trim().to_ascii_lowercase()) + .filter(|token| !token.is_empty()) + .collect()) +} + +fn strip_connection_nominated_headers( + headers: &[u8], + preserve_websocket_upgrade: bool, +) -> Result> { + let nominated = connection_nominated_header_names(headers)?; + let mut out = headers.to_vec(); + for name in nominated { + if preserve_websocket_upgrade && name == "upgrade" { + continue; + } + out = strip_header(&out, &name)?; + } + out = strip_header(&out, "connection")?; + if preserve_websocket_upgrade { + out = append_header(&out, "Connection", "Upgrade"); + } else { + out = strip_header(&out, "upgrade")?; + } + Ok(out) +} + fn append_header(headers: &[u8], name: &str, value: &str) -> Vec { let split = headers .windows(4) @@ -2192,20 +2265,25 @@ fn detect_payload_mode(headers: &str) -> Result { /// `Content-Length` and `Transfer-Encoding` headers to prevent request /// smuggling via CL/TE ambiguity. pub(crate) fn parse_body_length(headers: &str) -> Result { - let mut has_te_chunked = false; + let mut transfer_codings = Vec::new(); let mut cl_value: Option = None; for line in headers.lines().skip(1) { - let lower = line.to_ascii_lowercase(); - if lower.starts_with("transfer-encoding:") { - let val = lower.split_once(':').map_or("", |(_, v)| v.trim()); - if val.split(',').any(|enc| enc.trim() == "chunked") { - has_te_chunked = true; + let Some((name, value)) = line.split_once(':') else { + continue; + }; + if name.eq_ignore_ascii_case("transfer-encoding") { + for coding in value.split(',') { + let coding = coding.trim(); + if coding.is_empty() { + return Err(miette!("Request contains an empty Transfer-Encoding value")); + } + transfer_codings.push(coding.to_ascii_lowercase()); } } - if lower.starts_with("content-length:") { - let val = lower.split_once(':').map_or("", |(_, v)| v.trim()); - let len: u64 = val + if name.eq_ignore_ascii_case("content-length") { + let len: u64 = value + .trim() .parse() .map_err(|_| miette!("Request contains invalid Content-Length value"))?; if let Some(prev) = cl_value @@ -2219,14 +2297,19 @@ pub(crate) fn parse_body_length(headers: &str) -> Result { } } - if has_te_chunked && cl_value.is_some() { + if !transfer_codings.is_empty() && cl_value.is_some() { return Err(miette!( "Request contains both Transfer-Encoding and Content-Length headers" )); } - if has_te_chunked { - return Ok(BodyLength::Chunked); + if !transfer_codings.is_empty() { + if transfer_codings.len() == 1 && transfer_codings[0] == "chunked" { + return Ok(BodyLength::Chunked); + } + return Err(miette!( + "Request contains an unsupported Transfer-Encoding sequence" + )); } if let Some(len) = cl_value { return Ok(BodyLength::ContentLength(len)); @@ -3493,6 +3576,12 @@ mod tests { b"GET /v1/items HTTP/1.1\r\nX-Test value\r\n\r\n".as_slice(), b"GET /v1/items HTTP/1.1\r\nX-Test : value\r\n\r\n".as_slice(), b"GET /v1/items HTTP/1.1\r\nX@Test: value\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX-Test: before\0after\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX-Test: before\x7fafter\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nX-Test: before\rafter\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nConnection: x guard\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nConnection: content-length\r\nContent-Length: 0\r\n\r\n" + .as_slice(), ] { request_from_buffered_http("GET", "/v1/items", "/v1/items", raw.to_vec()) .expect_err("malformed buffered header fields must be rejected"); @@ -3668,16 +3757,65 @@ mod tests { ); } - /// SEC: Transfer-Encoding substring match must not match partial tokens. + /// SEC: Unsupported transfer codings must not be silently treated as no body. #[test] - fn te_substring_not_chunked() { - let headers = "POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunkedx\r\n\r\n"; - match parse_body_length(headers).unwrap() { - BodyLength::None => {} - other => panic!("Expected None for non-matching TE, got {other:?}"), + fn reject_unsupported_transfer_encoding_sequences() { + for value in [ + "gzip", + "gzip, chunked", + "chunked, gzip", + "chunked, chunked", + "chunkedx", + "chunked,", + ] { + let headers = + format!("POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: {value}\r\n\r\n"); + assert!( + parse_body_length(&headers).is_err(), + "unsupported transfer coding must be rejected: {value}" + ); } } + #[test] + fn reject_multiple_chunked_transfer_encoding_fields() { + let headers = "POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nTransfer-Encoding: chunked\r\n\r\n"; + assert!(parse_body_length(headers).is_err()); + } + + #[test] + fn reject_content_length_with_unsupported_transfer_encoding() { + let headers = + "POST /api HTTP/1.1\r\nHost: x\r\nContent-Length: 4\r\nTransfer-Encoding: gzip\r\n\r\n"; + assert!(parse_body_length(headers).is_err()); + } + + #[test] + fn strip_connection_nominated_headers_before_forwarding() { + let raw = b"GET /api HTTP/1.1\r\nHost: x\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; + let sanitized = + strip_connection_nominated_headers(raw, false).expect("sanitize request headers"); + let sanitized = String::from_utf8(sanitized).unwrap(); + + assert!(!sanitized.to_ascii_lowercase().contains("x-guard:")); + assert!(!sanitized.to_ascii_lowercase().contains("keep-alive:")); + assert!(!sanitized.to_ascii_lowercase().contains("connection:")); + assert!(sanitized.contains("X-Visible: yes\r\n")); + } + + #[test] + fn connection_sanitization_preserves_only_websocket_upgrade_exception() { + let raw = b"GET /ws HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: keep-alive, Upgrade, x-guard\r\nX-Guard: hidden\r\n\r\n"; + let sanitized = + strip_connection_nominated_headers(raw, true).expect("sanitize websocket headers"); + let sanitized = String::from_utf8(sanitized).unwrap(); + + assert!(sanitized.contains("Upgrade: websocket\r\n")); + assert!(sanitized.contains("Connection: Upgrade\r\n")); + assert!(!sanitized.to_ascii_lowercase().contains("x-guard:")); + assert!(!sanitized.contains("keep-alive")); + } + #[tokio::test] async fn collect_chunked_body_decodes_payload_bytes() { let mut client = tokio::io::empty(); @@ -6107,6 +6245,20 @@ mod tests { .map_err(|e| miette!("upstream task failed: {e}")) } + #[tokio::test] + async fn connect_rest_relay_strips_connection_nominated_headers_upstream() { + let raw = b"GET /api HTTP/1.1\r\nHost: api.example.com\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n".to_vec(); + let forwarded = relay_and_capture_with_options(raw, BodyLength::None, None, false) + .await + .expect("relay request"); + let lower = forwarded.to_ascii_lowercase(); + + assert!(!lower.contains("x-guard:")); + assert!(!lower.contains("keep-alive:")); + assert!(!lower.contains("connection:")); + assert!(forwarded.contains("X-Visible: yes\r\n")); + } + #[tokio::test] async fn relay_request_body_rewrites_provider_alias_header_and_urlencoded_token() { let (_, resolver) = SecretResolver::from_provider_env( diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a45a50052a..10401a6691 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -662,7 +662,21 @@ async fn handle_tcp_connection( } } - let request = String::from_utf8_lossy(&buf[..used]); + let header_end = buf[..used] + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("header terminator was observed") + + 4; + if crate::l7::rest::validate_http_request_header_block(&buf[..header_end]).is_err() { + respond(&mut client, b"HTTP/1.1 400 Bad Request\r\n\r\n").await?; + return Ok(()); + } + let request = + std::str::from_utf8(&buf[..header_end]).expect("validated HTTP request headers are UTF-8"); + if crate::l7::rest::parse_body_length(request).is_err() { + respond(&mut client, b"HTTP/1.1 400 Bad Request\r\n\r\n").await?; + return Ok(()); + } let mut lines = request.split("\r\n"); let request_line = lines.next().unwrap_or(""); let mut parts = request_line.split_whitespace(); @@ -3240,6 +3254,15 @@ fn rewrite_forward_request( let header_str = String::from_utf8_lossy(&raw[..header_end]); let lines = header_str.split("\r\n").collect::>(); + let connection_nominated: std::collections::HashSet = lines + .iter() + .skip(1) + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("connection")) + .flat_map(|(_, value)| value.split(',')) + .map(|token| token.trim().to_ascii_lowercase()) + .filter(|token| !token.is_empty()) + .collect(); // Rebuild headers, stripping hop-by-hop and adding proxy headers let mut output = Vec::with_capacity(header_end + 128); @@ -3267,27 +3290,41 @@ fn rewrite_forward_request( break; } - let lower = line.to_ascii_lowercase(); + let (field_name, _) = line + .split_once(':') + .expect("forward request passed strict ingress header validation"); + let field_name = field_name.to_ascii_lowercase(); // Strip proxy hop-by-hop headers - if lower.starts_with("proxy-connection:") - || lower.starts_with("proxy-authorization:") - || lower.starts_with("proxy-authenticate:") + if matches!( + field_name.as_str(), + "proxy-connection" | "proxy-authorization" | "proxy-authenticate" + ) { + continue; + } + + if connection_nominated.contains(&field_name) + && !(websocket_upgrade && field_name == "upgrade") { continue; } // Replace Connection header - if lower.starts_with("connection:") { + if field_name == "connection" { + if has_connection { + continue; + } has_connection = true; if websocket_upgrade { - output.extend_from_slice(line.as_bytes()); - output.extend_from_slice(b"\r\n"); + output.extend_from_slice(b"Connection: Upgrade\r\n"); continue; } output.extend_from_slice(b"Connection: close\r\n"); continue; } + if field_name == "upgrade" && !websocket_upgrade { + continue; + } let rewritten_line = match secret_resolver { Some(resolver) => rewrite_header_line_checked(line, resolver)?, @@ -3297,7 +3334,7 @@ fn rewrite_forward_request( output.extend_from_slice(rewritten_line.as_bytes()); output.extend_from_slice(b"\r\n"); - if lower.starts_with("via:") { + if field_name == "via" { has_via = true; } } @@ -4843,6 +4880,62 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + async fn drive_raw_request_through_handler(raw: Vec) -> Vec { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_middlewares: + - name: guard + middleware: openshell/regex + endpoints: + include: ["api.example.com"] +network_policies: {} +"#; + let engine = Arc::new(OpaEngine::from_strings(policy, data).expect("load policy")); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let client = tokio::spawn(async move { + let mut socket = TcpStream::connect(address).await.unwrap(); + socket.write_all(&raw).await.unwrap(); + let mut response = Vec::new(); + socket.read_to_end(&mut response).await.unwrap(); + response + }); + let (server, _) = listener.accept().await.unwrap(); + + Box::pin(handle_tcp_connection( + server, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(std::process::id())), + None, + None, + None, + Arc::new(None), + None, + None, + None, + None, + )) + .await + .expect("malformed request should be handled"); + client.await.unwrap() + } + + #[tokio::test] + async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { + for host in ["api.example.com", "unmatched.example.com"] { + let raw = format!( + "GET http://{host}/ HTTP/1.1\r\nHost: {host}\r\nX-Guard: before\0after\r\n\r\n" + ) + .into_bytes(); + let response = Box::pin(drive_raw_request_through_handler(raw)).await; + assert!( + response.starts_with(b"HTTP/1.1 400 Bad Request"), + "malformed request for {host} must fail at ingress" + ); + } + } + #[test] fn middleware_denial_response_contains_only_platform_mutation_reason() { const RAW_SECRET: &str = "sk-secret-request-value"; @@ -8050,6 +8143,20 @@ network_policies: assert!(!result_str.contains("keep-alive")); } + #[test] + fn test_rewrite_strips_connection_nominated_headers() { + let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; + let result = + rewrite_forward_request(raw, raw.len(), "/p", None, false).expect("should succeed"); + let result_str = String::from_utf8_lossy(&result); + let lower = result_str.to_ascii_lowercase(); + + assert!(!lower.contains("x-guard:")); + assert!(!lower.contains("keep-alive:")); + assert!(result_str.contains("Connection: close\r\n")); + assert!(result_str.contains("X-Visible: yes\r\n")); + } + #[test] fn test_rewrite_preserves_body_overflow() { let raw = b"POST http://host/api HTTP/1.1\r\nHost: host\r\nContent-Length: 13\r\n\r\n{\"key\":\"val\"}"; @@ -8279,6 +8386,9 @@ network_policies: Host: gateway.example.test\r\n\ Upgrade: websocket\r\n\ Connection: keep-alive, Upgrade\r\n\ + Keep-Alive: timeout=5\r\n\ + X-Guard: hidden\r\n\ + Connection: x-guard\r\n\ Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\ Sec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover\r\n\ Sec-WebSocket-Version: 13\r\n\r\n"; @@ -8288,7 +8398,9 @@ network_policies: let result_str = String::from_utf8_lossy(&result); assert!(result_str.starts_with("GET /ws HTTP/1.1\r\n")); - assert!(result_str.contains("Connection: keep-alive, Upgrade\r\n")); + assert!(result_str.contains("Connection: Upgrade\r\n")); + assert!(!result_str.contains("keep-alive")); + assert!(!result_str.to_ascii_lowercase().contains("x-guard:")); assert!( !result_str.contains("Connection: close\r\n"), "websocket forward proxy must not strip the upgrade token" diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 1c060ecb6b..0398bf8e75 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -26,7 +26,7 @@ Because each transformed body is re-checked before the next stage runs, a middle If post-transformation policy evaluation itself fails, OpenShell denies the request and emits a high-severity detection finding. This failure is separate from middleware `on_error` because the middleware completed successfully; the sandbox policy could not validate its output. -Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Middleware-visible request headers are delivered in wire order and repeated header names are preserved as separate entries. OpenShell filters credential, routing, framing, and hop-by-hop headers before invoking middleware. +Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Middleware-visible request headers are delivered in wire order and repeated header names are preserved as separate entries. OpenShell filters credential, routing, framing, and hop-by-hop headers before invoking middleware. It rejects malformed request headers and unsupported transfer-coding sequences before middleware or policy dispatch. Headers named by a request's `Connection` field are omitted from middleware input and removed before forwarding, except for the validated WebSocket upgrade pair. ## Choose a Middleware Type From 9923194aa71d5ccba8281e5cf2666444ca0130b2 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 14 Jul 2026 20:29:17 -0700 Subject: [PATCH 50/59] fix(proxy): harden middleware request forwarding Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 5 +- architecture/security-policy.md | 4 + .../src/l7/rest.rs | 189 +++++++++++++++--- .../openshell-supervisor-network/src/proxy.rs | 103 +++++----- 4 files changed, 231 insertions(+), 70 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 3f197dae0e..ec6dc8ab9d 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -144,7 +144,10 @@ hop-by-hop headers remain supervisor-owned and cannot be mutated. Names dynamically nominated by the original request's `Connection` header are carried separately after header filtering so write and remove mutations cannot reintroduce them. The proxy also removes nominated fields before forwarding; -only a validated WebSocket handshake preserves the required `Upgrade` pair. +only a validated WebSocket handshake preserves a canonical `Connection: +Upgrade` and `Upgrade: websocket` pair. When middleware buffers a request body, +the proxy consumes `Expect: 100-continue` locally, acknowledges it only if more +client bytes are required, and never forwards it with the normalized body. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/architecture/security-policy.md b/architecture/security-policy.md index c4eaa2328a..26470711ca 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -191,6 +191,10 @@ Sandbox events that represent observable behavior use OCSF structured logs: Use plain tracing for internal plumbing such as retries, debug state, and intermediate steps where the final observable event is logged separately. +Forward-proxy success is a final observable event: emit it only after +middleware, token grants, credential rewriting, policy-generation checks, and +the HTTP relay have succeeded so a later denial cannot coexist with an allowed +record for the same request. Never log secrets, credentials, bearer tokens, or query parameters in OCSF messages. OCSF JSONL output may be shipped to external systems. diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 8524eb957a..a9f865fc50 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -966,6 +966,7 @@ pub(crate) async fn buffer_request_body_for_middleware 0 && already_read.is_empty() { - acknowledge_expect_continue(client, &mut headers).await?; - } + handle_buffered_expect_continue(client, &mut headers, remaining > 0).await?; let mut buf = [0u8; RELAY_BUF_SIZE]; while remaining > 0 { let to_read = remaining.min(buf.len()); @@ -1018,9 +1017,8 @@ pub(crate) async fn buffer_request_body_for_middleware( +async fn handle_buffered_expect_continue( client: &mut C, headers: &mut Vec, + needs_client_read: bool, ) -> Result<()> { let header_str = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; - if !has_expect_continue(header_str) { - return Ok(()); + if needs_client_read && has_expect_continue(header_str) { + client + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .await + .into_diagnostic()?; + client.flush().await.into_diagnostic()?; } - - client - .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; + // Middleware has taken responsibility for collecting the body. Never + // forward Expect upstream alongside the completed, normalized request. *headers = strip_header(headers, "expect")?; Ok(()) } +fn chunked_body_is_fully_buffered(bytes: &[u8]) -> bool { + let mut pos = 0usize; + loop { + let Some(line_end) = bytes[pos..].windows(2).position(|window| window == b"\r\n") else { + return false; + }; + let line_end = pos + line_end; + let Ok(size_line) = std::str::from_utf8(&bytes[pos..line_end]) else { + return false; + }; + let size_token = size_line + .split(';') + .next() + .map(str::trim) + .unwrap_or_default(); + let Ok(chunk_size) = usize::from_str_radix(size_token, 16) else { + return false; + }; + pos = line_end + 2; + if chunk_size == 0 { + return bytes.get(pos..pos.saturating_add(2)) == Some(b"\r\n"); + } + let Some(chunk_end) = pos.checked_add(chunk_size) else { + return false; + }; + let Some(terminator_end) = chunk_end.checked_add(2) else { + return false; + }; + if bytes.get(chunk_end..terminator_end) != Some(b"\r\n") { + return false; + } + pos = terminator_end; + } +} + pub(crate) fn rebuild_request_with_buffered_body( req: &L7Request, headers: &[u8], @@ -1667,16 +1701,13 @@ fn strip_connection_nominated_headers( let nominated = connection_nominated_header_names(headers)?; let mut out = headers.to_vec(); for name in nominated { - if preserve_websocket_upgrade && name == "upgrade" { - continue; - } out = strip_header(&out, &name)?; } out = strip_header(&out, "connection")?; + out = strip_header(&out, "upgrade")?; if preserve_websocket_upgrade { out = append_header(&out, "Connection", "Upgrade"); - } else { - out = strip_header(&out, "upgrade")?; + out = append_header(&out, "Upgrade", "websocket"); } Ok(out) } @@ -3805,17 +3836,116 @@ mod tests { #[test] fn connection_sanitization_preserves_only_websocket_upgrade_exception() { - let raw = b"GET /ws HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: keep-alive, Upgrade, x-guard\r\nX-Guard: hidden\r\n\r\n"; + let raw = b"GET /ws HTTP/1.1\r\nHost: x\r\nUpgrade: h2c\r\nUpgrade: h2c, websocket\r\nConnection: keep-alive, Upgrade, x-guard\r\nX-Guard: hidden\r\n\r\n"; let sanitized = strip_connection_nominated_headers(raw, true).expect("sanitize websocket headers"); let sanitized = String::from_utf8(sanitized).unwrap(); - assert!(sanitized.contains("Upgrade: websocket\r\n")); - assert!(sanitized.contains("Connection: Upgrade\r\n")); + assert_eq!(sanitized.matches("Upgrade: websocket\r\n").count(), 1); + assert_eq!(sanitized.matches("Connection: Upgrade\r\n").count(), 1); + assert!(!sanitized.to_ascii_lowercase().contains("upgrade: h2c")); assert!(!sanitized.to_ascii_lowercase().contains("x-guard:")); assert!(!sanitized.contains("keep-alive")); } + #[tokio::test] + async fn middleware_fixed_read_ahead_consumes_expect_continue() { + for (already_read, remaining, should_acknowledge) in [ + (b"hello".as_slice(), b"".as_slice(), false), + (b"he".as_slice(), b"llo".as_slice(), true), + ] { + let mut raw = b"POST /api HTTP/1.1\r\nHost: example.com\r\nContent-Length: 5\r\nExpect: 100-continue\r\n\r\n".to_vec(); + raw.extend_from_slice(already_read); + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw, + body_length: BodyLength::ContentLength(5), + }; + let (mut client, mut peer) = tokio::io::duplex(128); + peer.write_all(remaining).await.unwrap(); + + let result = buffer_request_body_for_middleware(&req, &mut client, None, 1024) + .await + .expect("fixed body should buffer"); + let BufferResult::Buffered(buffered) = result else { + panic!("fixed body unexpectedly exceeded capacity") + }; + assert_eq!(buffered.body, b"hello"); + assert!( + !String::from_utf8_lossy(&buffered.headers) + .to_ascii_lowercase() + .contains("expect:") + ); + + let mut response = [0u8; 64]; + let read = tokio::time::timeout( + std::time::Duration::from_millis(20), + peer.read(&mut response), + ) + .await; + if should_acknowledge { + let count = read.expect("partial body should be acknowledged").unwrap(); + assert_eq!(&response[..count], b"HTTP/1.1 100 Continue\r\n\r\n"); + } else { + assert!( + read.is_err(), + "complete read-ahead should not be acknowledged" + ); + } + } + } + + #[tokio::test] + async fn middleware_chunked_read_ahead_consumes_expect_continue() { + for (already_read, remaining, should_acknowledge) in [ + (b"5\r\nhello\r\n0\r\n\r\n".as_slice(), b"".as_slice(), false), + (b"5\r\nhe".as_slice(), b"llo\r\n0\r\n\r\n".as_slice(), true), + ] { + let mut raw = b"POST /api HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n".to_vec(); + raw.extend_from_slice(already_read); + let req = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header: raw, + body_length: BodyLength::Chunked, + }; + let (mut client, mut peer) = tokio::io::duplex(128); + peer.write_all(remaining).await.unwrap(); + + let result = buffer_request_body_for_middleware(&req, &mut client, None, 1024) + .await + .expect("chunked body should buffer"); + let BufferResult::Buffered(buffered) = result else { + panic!("chunked body unexpectedly exceeded capacity") + }; + assert_eq!(buffered.body, b"hello"); + assert!( + !String::from_utf8_lossy(&buffered.headers) + .to_ascii_lowercase() + .contains("expect:") + ); + + let mut response = [0u8; 64]; + let read = tokio::time::timeout( + std::time::Duration::from_millis(20), + peer.read(&mut response), + ) + .await; + if should_acknowledge { + let count = read.expect("partial body should be acknowledged").unwrap(); + assert_eq!(&response[..count], b"HTTP/1.1 100 Continue\r\n\r\n"); + } else { + assert!( + read.is_err(), + "complete read-ahead should not be acknowledged" + ); + } + } + } + #[tokio::test] async fn collect_chunked_body_decodes_payload_bytes() { let mut client = tokio::io::empty(); @@ -6259,6 +6389,19 @@ mod tests { assert!(forwarded.contains("X-Visible: yes\r\n")); } + #[tokio::test] + async fn connect_rest_relay_canonicalizes_websocket_upgrade_headers() { + let raw = b"GET /ws HTTP/1.1\r\nHost: api.example.com\r\nUpgrade: h2c\r\nUpgrade: h2c, websocket\r\nConnection: keep-alive, Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n".to_vec(); + let forwarded = relay_and_capture_with_options(raw, BodyLength::None, None, false) + .await + .expect("relay websocket request"); + let lower = forwarded.to_ascii_lowercase(); + + assert_eq!(lower.matches("connection: upgrade\r\n").count(), 1); + assert_eq!(lower.matches("upgrade: websocket\r\n").count(), 1); + assert!(!lower.contains("upgrade: h2c")); + } + #[tokio::test] async fn relay_request_body_rewrites_provider_alias_header_and_urlencoded_token() { let (_, resolver) = SecretResolver::from_provider_env( diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 10401a6691..d0d6dd89fa 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3266,7 +3266,6 @@ fn rewrite_forward_request( // Rebuild headers, stripping hop-by-hop and adding proxy headers let mut output = Vec::with_capacity(header_end + 128); - let mut has_connection = false; let mut has_via = false; for (i, line) in lines.iter().enumerate() { @@ -3303,26 +3302,12 @@ fn rewrite_forward_request( continue; } - if connection_nominated.contains(&field_name) - && !(websocket_upgrade && field_name == "upgrade") - { + if connection_nominated.contains(&field_name) { continue; } - // Replace Connection header - if field_name == "connection" { - if has_connection { - continue; - } - has_connection = true; - if websocket_upgrade { - output.extend_from_slice(b"Connection: Upgrade\r\n"); - continue; - } - output.extend_from_slice(b"Connection: close\r\n"); - continue; - } - if field_name == "upgrade" && !websocket_upgrade { + // Reconstruct hop-by-hop upgrade fields after processing the originals. + if field_name == "connection" || field_name == "upgrade" { continue; } @@ -3340,7 +3325,10 @@ fn rewrite_forward_request( } // Inject missing headers - if !has_connection && !websocket_upgrade { + if websocket_upgrade { + output.extend_from_slice(b"Connection: Upgrade\r\n"); + output.extend_from_slice(b"Upgrade: websocket\r\n"); + } else { output.extend_from_slice(b"Connection: close\r\n"); } if !has_via { @@ -4531,31 +4519,6 @@ async fn handle_forward_proxy( } }; - // Log success - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "opa") - .message(format!("FORWARD allowed {method} {host_lc}:{port}{path}")) - .build(); - ocsf_emit!(event); - } - emit_forward_success_activity(activity_tx, l7_activity_pending); - let middleware_path = path.split_once('?').map_or(path.as_str(), |(path, _)| path); let middleware_input = crate::opa::NetworkInput { host: host_lc.clone(), @@ -4727,6 +4690,34 @@ async fn handle_forward_proxy( }, ) .await?; + + // The request has now survived middleware, token grant, credential + // rewriting, generation checks, and the HTTP relay. Only now record the + // final allowed outcome. + { + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", &host_lc, &path, port), + )) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule(policy_str, "opa") + .message(format!("FORWARD allowed {method} {host_lc}:{port}{path}")) + .build(); + ocsf_emit!(event); + } + emit_forward_success_activity(activity_tx, l7_activity_pending); + if let crate::l7::provider::RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -5403,6 +5394,23 @@ network_policies: assert_eq!(event.deny_group, "l7_policy"); } + #[test] + fn forward_middleware_denial_does_not_emit_success_activity() { + let (tx, mut rx) = mpsc::channel(4); + let activity_tx = Some(tx); + + emit_activity_simple(activity_tx.as_ref(), true, "middleware"); + let event = rx + .try_recv() + .expect("middleware denial should emit the request activity"); + assert!(event.denied); + assert_eq!(event.deny_group, "middleware"); + assert!( + rx.try_recv().is_err(), + "middleware-denied forward request must not emit success activity" + ); + } + #[test] fn forward_success_activity_uses_unknown_without_l7() { let (tx, mut rx) = mpsc::channel(4); @@ -8384,7 +8392,8 @@ network_policies: fn test_forward_rewrite_preserves_websocket_upgrade_connection_header() { let raw = "GET http://gateway.example.test/ws HTTP/1.1\r\n\ Host: gateway.example.test\r\n\ - Upgrade: websocket\r\n\ + Upgrade: h2c\r\n\ + Upgrade: h2c, websocket\r\n\ Connection: keep-alive, Upgrade\r\n\ Keep-Alive: timeout=5\r\n\ X-Guard: hidden\r\n\ @@ -8398,7 +8407,9 @@ network_policies: let result_str = String::from_utf8_lossy(&result); assert!(result_str.starts_with("GET /ws HTTP/1.1\r\n")); - assert!(result_str.contains("Connection: Upgrade\r\n")); + assert_eq!(result_str.matches("Connection: Upgrade\r\n").count(), 1); + assert_eq!(result_str.matches("Upgrade: websocket\r\n").count(), 1); + assert!(!result_str.to_ascii_lowercase().contains("upgrade: h2c")); assert!(!result_str.contains("keep-alive")); assert!(!result_str.to_ascii_lowercase().contains("x-guard:")); assert!( From be9e5b7f0246f47d395f16fbeff71fa4063d9baa Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 15 Jul 2026 10:17:51 -0700 Subject: [PATCH 51/59] fix(network): harden HTTP proxy request parsing Signed-off-by: Piotr Mlocek --- .../src/l7/rest.rs | 59 +++++++++++- .../openshell-supervisor-network/src/proxy.rs | 92 +++++++++++++------ 2 files changed, 120 insertions(+), 31 deletions(-) diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index a9f865fc50..2542e4ddc3 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -297,11 +297,11 @@ pub(crate) fn request_from_buffered_http( }) } -/// Validate HTTP/1 request header fields before policy evaluation or forwarding. +/// Validate an HTTP/1 request line and header fields before policy evaluation or forwarding. /// -/// This parser deliberately rejects obsolete line folding and malformed field -/// names rather than allowing downstream middleware and the upstream server to -/// interpret the same wire bytes differently. +/// This parser deliberately rejects malformed request lines, obsolete line +/// folding, and malformed field names rather than allowing policy, middleware, +/// and the upstream server to interpret the same wire bytes differently. pub(crate) fn validate_http_request_header_block(headers: &[u8]) -> Result<()> { let headers = std::str::from_utf8(headers).map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; @@ -309,9 +309,10 @@ pub(crate) fn validate_http_request_header_block(headers: &[u8]) -> Result<()> { .strip_suffix("\r\n\r\n") .ok_or_else(|| miette!("HTTP request headers are missing the CRLF terminator"))?; let mut lines = header_block.split("\r\n"); - lines + let request_line = lines .next() .ok_or_else(|| miette!("HTTP request is missing a request line"))?; + validate_http_request_line(request_line)?; let mut connection_nominated = HashSet::new(); for line in lines { @@ -371,6 +372,38 @@ pub(crate) fn validate_http_request_header_block(headers: &[u8]) -> Result<()> { Ok(()) } +fn validate_http_request_line(request_line: &str) -> Result<()> { + let mut parts = request_line.split(' '); + let method = parts + .next() + .ok_or_else(|| miette!("HTTP request line is missing a method"))?; + let target = parts + .next() + .ok_or_else(|| miette!("HTTP request line is missing a target"))?; + let version = parts + .next() + .ok_or_else(|| miette!("HTTP request line is missing a version"))?; + + if parts.next().is_some() || method.is_empty() || target.is_empty() || version.is_empty() { + return Err(miette!( + "HTTP request line must be exactly 'METHOD SP target SP HTTP/1.0|HTTP/1.1'" + )); + } + if !method.bytes().all(is_http_field_name_byte) { + return Err(miette!("HTTP request method is not a valid HTTP token")); + } + if target.bytes().any(|byte| byte <= b' ' || byte == 0x7f) { + return Err(miette!( + "HTTP request target contains whitespace or a control byte" + )); + } + if !matches!(version, "HTTP/1.0" | "HTTP/1.1") { + return Err(miette!("Unsupported HTTP version: {version}")); + } + + Ok(()) +} + fn is_http_field_name_byte(byte: u8) -> bool { byte.is_ascii_alphanumeric() || matches!( @@ -3619,6 +3652,22 @@ mod tests { } } + #[test] + fn buffered_request_parser_rejects_malformed_request_lines() { + for raw in [ + b"GET /v1/items HTTP/1.1 extra\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/1.1\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GET\t/v1/items HTTP/1.1\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GE(T /v1/items HTTP/1.1\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GET /v1/\0items HTTP/1.1\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GET /v1/items HTTP/2\r\nHost: api.example.com\r\n\r\n".as_slice(), + b"GET /v1/items\r\nHost: api.example.com\r\n\r\n".as_slice(), + ] { + request_from_buffered_http("GET", "/v1/items", "/v1/items", raw.to_vec()) + .expect_err("malformed buffered request lines must be rejected"); + } + } + #[test] fn parse_chunked() { let headers = diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index d0d6dd89fa..01449ce0b4 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3493,8 +3493,42 @@ async fn handle_forward_proxy( }; let host_lc = host.to_ascii_lowercase(); + if scheme != "http" { + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Refuse) + .action(ActionId::Denied) + .disposition(DispositionId::Rejected) + .severity(SeverityId::Informational) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(format!( + "FORWARD rejected: unsupported scheme {scheme} for {host_lc}:{port}" + )) + .build(); + ocsf_emit!(event); + if scheme == "https" { + respond( + client, + b"HTTP/1.1 400 Bad Request\r\nContent-Length: 27\r\n\r\nUse CONNECT for HTTPS URLs", + ) + .await?; + } else { + respond( + client, + &build_json_error_response( + 400, + "Bad Request", + "unsupported_proxy_scheme", + "Forward proxy requests must use http", + ), + ) + .await?; + } + return Ok(()); + } + if host_lc == POLICY_LOCAL_HOST { - if scheme != "http" || port != 80 { + if port != 80 { respond( client, &build_json_error_response( @@ -3525,31 +3559,7 @@ async fn handle_forward_proxy( return Ok(()); } - // 2. Reject HTTPS — must use CONNECT for TLS - if scheme == "https" { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Refuse) - .action(ActionId::Denied) - .disposition(DispositionId::Rejected) - .severity(SeverityId::Informational) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!( - "FORWARD rejected: HTTPS requires CONNECT for {host_lc}:{port}" - )) - .build(); - ocsf_emit!(event); - } - respond( - client, - b"HTTP/1.1 400 Bad Request\r\nContent-Length: 27\r\n\r\nUse CONNECT for HTTPS URLs", - ) - .await?; - return Ok(()); - } - - // 3. Evaluate OPA policy (same identity binding as CONNECT) + // 2. Evaluate OPA policy (same identity binding as CONNECT) let peer_addr = client.peer_addr().into_diagnostic()?; let _local_addr = client.local_addr().into_diagnostic()?; @@ -4927,6 +4937,36 @@ network_policies: {} } } + #[tokio::test] + async fn malformed_request_lines_are_rejected_before_connect_or_forward_dispatch() { + for host in ["api.example.com", "unmatched.example.com"] { + for request_line in [ + format!("GET http://{host}/ HTTP/1.1 extra"), + format!("CONNECT {host}:443 HTTP/1.1 extra"), + ] { + let raw = format!("{request_line}\r\nHost: {host}\r\n\r\n").into_bytes(); + let response = Box::pin(drive_raw_request_through_handler(raw)).await; + assert!( + response.starts_with(b"HTTP/1.1 400 Bad Request"), + "malformed request for {host} must fail before dispatch" + ); + } + } + } + + #[tokio::test] + async fn unsupported_forward_scheme_is_rejected_before_policy_or_middleware_dispatch() { + let raw = b"GET ftp://api.example.com/resource HTTP/1.1\r\nHost: api.example.com\r\n\r\n" + .to_vec(); + let response = Box::pin(drive_raw_request_through_handler(raw)).await; + assert!(response.starts_with(b"HTTP/1.1 400 Bad Request")); + assert!( + response + .windows(b"unsupported_proxy_scheme".len()) + .any(|window| window == b"unsupported_proxy_scheme") + ); + } + #[test] fn middleware_denial_response_contains_only_platform_mutation_reason() { const RAW_SECRET: &str = "sk-secret-request-value"; From 7339c239befdfacdd6e40391aaf311ee0b359915 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 15 Jul 2026 10:42:50 -0700 Subject: [PATCH 52/59] fix(proxy): preserve policy local scheme errors Signed-off-by: Piotr Mlocek --- .../openshell-supervisor-network/src/proxy.rs | 81 +++++++++++-------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 01449ce0b4..058bee10f4 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3493,6 +3493,38 @@ async fn handle_forward_proxy( }; let host_lc = host.to_ascii_lowercase(); + if host_lc == POLICY_LOCAL_HOST { + if scheme != "http" || port != 80 { + respond( + client, + &build_json_error_response( + 400, + "Bad Request", + "invalid_policy_local_scheme", + "Use http://policy.local only", + ), + ) + .await?; + return Ok(()); + } + if let Some(ctx) = policy_local_ctx { + return crate::policy_local::handle_forward_request( + &ctx, + method, + &path, + &buf[..used], + client, + ) + .await; + } + respond( + client, + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 31\r\n\r\npolicy.local is not configured", + ) + .await?; + return Ok(()); + } + if scheme != "http" { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) @@ -3527,38 +3559,6 @@ async fn handle_forward_proxy( return Ok(()); } - if host_lc == POLICY_LOCAL_HOST { - if port != 80 { - respond( - client, - &build_json_error_response( - 400, - "Bad Request", - "invalid_policy_local_scheme", - "Use http://policy.local only", - ), - ) - .await?; - return Ok(()); - } - if let Some(ctx) = policy_local_ctx { - return crate::policy_local::handle_forward_request( - &ctx, - method, - &path, - &buf[..used], - client, - ) - .await; - } - respond( - client, - b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 31\r\n\r\npolicy.local is not configured", - ) - .await?; - return Ok(()); - } - // 2. Evaluate OPA policy (same identity binding as CONNECT) let peer_addr = client.peer_addr().into_diagnostic()?; let _local_addr = client.local_addr().into_diagnostic()?; @@ -4967,6 +4967,23 @@ network_policies: {} ); } + #[tokio::test] + async fn policy_local_preserves_specific_non_http_scheme_error() { + for scheme in ["https", "ftp"] { + let raw = format!( + "GET {scheme}://policy.local/resource HTTP/1.1\r\nHost: policy.local\r\n\r\n" + ) + .into_bytes(); + let response = Box::pin(drive_raw_request_through_handler(raw)).await; + assert!(response.starts_with(b"HTTP/1.1 400 Bad Request")); + assert!( + response + .windows(b"invalid_policy_local_scheme".len()) + .any(|window| window == b"invalid_policy_local_scheme") + ); + } + } + #[test] fn middleware_denial_response_contains_only_platform_mutation_reason() { const RAW_SECRET: &str = "sk-secret-request-value"; From e49d20f1f089b85251b4c66d425626eb0802eb5e Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 15 Jul 2026 12:25:11 -0700 Subject: [PATCH 53/59] fix(proxy): canonicalize forward host header Signed-off-by: Piotr Mlocek --- .../openshell-supervisor-network/src/proxy.rs | 306 ++++++++++++++++-- 1 file changed, 279 insertions(+), 27 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 058bee10f4..8f05a9c543 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3229,6 +3229,68 @@ fn parse_proxy_uri(uri: &str) -> Result<(String, String, u16, String)> { Ok((scheme, host, port, path.to_string())) } +/// Build the HTTP/1.1 `Host` value for a plain-HTTP absolute-form target. +/// +/// Forward proxy requests are restricted to `http`, so port 80 is omitted as +/// the default port. IPv6 literals regain the brackets removed by +/// `parse_proxy_uri` before they are written as an authority. +fn canonical_forward_authority(host: &str, port: u16) -> String { + let host = host.to_ascii_lowercase(); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host + }; + if port == 80 { + host + } else { + format!("{host}:{port}") + } +} + +/// Replace every received `Host` field with the authority selected from the +/// absolute-form request-target, preserving any body bytes already read. +/// +/// RFC 9112 section 3.2.2 requires a proxy to ignore the received `Host` field +/// and generate a new value from the absolute request-target. Doing this before +/// L7 and middleware processing also keeps every buffered representation tied +/// to the same authority used for policy selection. +fn canonicalize_forward_host_header(raw: &[u8], authority: &str) -> Result> { + let header_end = raw + .windows(4) + .position(|window| window == b"\r\n\r\n") + .ok_or_else(|| miette::miette!("HTTP request headers are missing the CRLF terminator"))? + + 4; + crate::l7::rest::validate_http_request_header_block(&raw[..header_end])?; + let header_block = std::str::from_utf8(&raw[..header_end]) + .map_err(|_| miette::miette!("HTTP headers contain invalid UTF-8"))? + .strip_suffix("\r\n\r\n") + .expect("validated header block has terminator"); + let mut lines = header_block.split("\r\n"); + let request_line = lines + .next() + .expect("validated header block contains a request line"); + + let mut output = Vec::with_capacity(raw.len() + authority.len() + 8); + output.extend_from_slice(request_line.as_bytes()); + output.extend_from_slice(b"\r\nHost: "); + output.extend_from_slice(authority.as_bytes()); + output.extend_from_slice(b"\r\n"); + for line in lines { + let (field_name, _) = line + .split_once(':') + .expect("validated header field contains colon"); + if field_name.eq_ignore_ascii_case("host") { + continue; + } + output.extend_from_slice(line.as_bytes()); + output.extend_from_slice(b"\r\n"); + } + output.extend_from_slice(b"\r\n"); + output.extend_from_slice(&raw[header_end..]); + Ok(output) +} + /// Rewrite an absolute-form HTTP proxy request to origin-form for upstream. /// /// Transforms `GET http://host:port/path HTTP/1.1` into `GET /path HTTP/1.1`, @@ -3239,6 +3301,7 @@ fn rewrite_forward_request( raw: &[u8], used: usize, path: &str, + canonical_authority: &str, secret_resolver: Option<&SecretResolver>, request_body_credential_rewrite: bool, ) -> Result, secrets::UnresolvedPlaceholderError> { @@ -3294,6 +3357,12 @@ fn rewrite_forward_request( .expect("forward request passed strict ingress header validation"); let field_name = field_name.to_ascii_lowercase(); + // RFC 9112 section 3.2.2 requires proxies to replace every received + // Host field with one generated from the absolute request-target. + if field_name == "host" { + continue; + } + // Strip proxy hop-by-hop headers if matches!( field_name.as_str(), @@ -3324,6 +3393,11 @@ fn rewrite_forward_request( } } + // Generate the only Host field from the absolute request-target authority. + output.extend_from_slice(b"Host: "); + output.extend_from_slice(canonical_authority.as_bytes()); + output.extend_from_slice(b"\r\n"); + // Inject missing headers if websocket_upgrade { output.extend_from_slice(b"Connection: Upgrade\r\n"); @@ -3559,6 +3633,10 @@ async fn handle_forward_proxy( return Ok(()); } + let canonical_authority = canonical_forward_authority(&host_lc, port); + let mut forward_request_bytes = + canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; + // 2. Evaluate OPA policy (same identity binding as CONNECT) let peer_addr = client.peer_addr().into_diagnostic()?; let _local_addr = client.local_addr().into_diagnostic()?; @@ -3697,7 +3775,6 @@ async fn handle_forward_proxy( return Ok(()); } }; - let mut forward_request_bytes = buf[..used].to_vec(); let mut upstream_target = path.clone(); let mut websocket_extensions = crate::l7::rest::WebSocketExtensionMode::Preserve; let mut forward_tunnel_engine: Option = None; @@ -4639,6 +4716,7 @@ async fn handle_forward_proxy( &forward_request_bytes, forward_request_bytes.len(), &upstream_target, + &canonical_authority, secret_resolver.as_deref(), request_body_credential_rewrite, ) { @@ -5498,10 +5576,19 @@ network_policies: request_body_credential_rewrite: bool, ) -> Result { let guard = forward_test_guard(); + let target_uri = std::str::from_utf8(raw) + .expect("forward test request is UTF-8") + .lines() + .next() + .and_then(|line| line.split(' ').nth(1)) + .expect("forward test request has an absolute target"); + let (_, host, port, _) = parse_proxy_uri(target_uri)?; + let authority = canonical_forward_authority(&host, port); let rewritten = rewrite_forward_request( raw, raw.len(), path, + &authority, resolver, request_body_credential_rewrite, ) @@ -5713,8 +5800,15 @@ network_policies: Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\ Sec-WebSocket-Version: 13\r\n\r\n" ); - let rewritten = rewrite_forward_request(raw.as_bytes(), raw.len(), path, None, false) - .expect("forward websocket request should rewrite to origin form"); + let rewritten = rewrite_forward_request( + raw.as_bytes(), + raw.len(), + path, + &canonical_forward_authority(&host, port), + None, + false, + ) + .expect("forward websocket request should rewrite to origin form"); let websocket_extensions = crate::l7::relay::websocket_extension_mode(&config); let target = path.to_string(); let query_params = std::collections::HashMap::new(); @@ -8144,9 +8238,15 @@ network_policies: let with_token = inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &ctx) .await .expect("forward token grant should inject"); - let rewritten = - rewrite_forward_request(&with_token, with_token.len(), "/v1/projects", None, false) - .expect("forward request should rewrite"); + let rewritten = rewrite_forward_request( + &with_token, + with_token.len(), + "/v1/projects", + "api.example.test:8080", + None, + false, + ) + .expect("forward request should rewrite"); let rewritten = String::from_utf8_lossy(&rewritten); assert!(rewritten.starts_with("GET /v1/projects HTTP/1.1\r\n")); @@ -8174,8 +8274,8 @@ network_policies: fn test_rewrite_get_request() { let raw = b"GET http://10.0.0.1:8000/api HTTP/1.1\r\nHost: 10.0.0.1:8000\r\nAccept: */*\r\n\r\n"; - let result = - rewrite_forward_request(raw, raw.len(), "/api", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/api", "10.0.0.1:8000", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.starts_with("GET /api HTTP/1.1\r\n")); assert!(result_str.contains("Host: 10.0.0.1:8000")); @@ -8183,11 +8283,153 @@ network_policies: assert!(result_str.contains("Via: 1.1 openshell-sandbox")); } + #[test] + fn canonical_forward_authority_formats_ports_and_ipv6() { + for (uri, expected) in [ + ("http://API.EXAMPLE.TEST/path", "api.example.test"), + ("http://api.example.test:8080/path", "api.example.test:8080"), + ("http://[2001:DB8::1]/path", "[2001:db8::1]"), + ("http://[2001:DB8::1]:8080/path", "[2001:db8::1]:8080"), + ] { + let (_, host, port, _) = parse_proxy_uri(uri).expect("parse absolute target"); + assert_eq!(canonical_forward_authority(&host, port), expected); + } + } + + #[test] + fn forward_host_header_is_replaced_from_absolute_target() { + let raw = b"POST http://allowed.example.test:8080/api HTTP/1.1\r\n\ + Host: disallowed.example.test\r\n\ + hOsT: second.example.test\r\n\ + Content-Length: 4\r\n\r\nbody"; + let authority = "allowed.example.test:8080"; + + let canonical = canonicalize_forward_host_header(raw, authority) + .expect("canonicalize received Host fields"); + let canonical = String::from_utf8(canonical).expect("canonical request is UTF-8"); + let host_fields: Vec<_> = canonical + .split("\r\n") + .skip(1) + .take_while(|line| !line.is_empty()) + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("host")) + }) + .collect(); + assert_eq!(host_fields, ["Host: allowed.example.test:8080"]); + assert!(!canonical.contains("disallowed.example.test")); + assert!(!canonical.contains("second.example.test")); + assert!(canonical.ends_with("\r\n\r\nbody")); + + let rewritten = rewrite_forward_request(raw, raw.len(), "/api", authority, None, false) + .expect("final rewrite enforces canonical Host"); + let rewritten = String::from_utf8(rewritten).expect("rewritten request is UTF-8"); + assert_eq!( + rewritten + .split("\r\n") + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("host")) + }) + .collect::>(), + ["Host: allowed.example.test:8080"] + ); + assert!(!rewritten.contains("disallowed.example.test")); + assert!(!rewritten.contains("second.example.test")); + } + + #[test] + fn forward_host_header_is_generated_when_missing() { + let raw = b"GET http://allowed.example.test/api HTTP/1.1\r\nAccept: */*\r\n\r\n"; + let canonical = canonicalize_forward_host_header(raw, "allowed.example.test") + .expect("generate missing Host field"); + let canonical = String::from_utf8(canonical).expect("canonical request is UTF-8"); + assert!(canonical.starts_with( + "GET http://allowed.example.test/api HTTP/1.1\r\nHost: allowed.example.test\r\n" + )); + } + + #[tokio::test] + async fn middleware_selected_forward_keeps_canonical_host_on_the_wire() { + let authority = "allowed.example.test"; + let raw = b"GET http://allowed.example.test/api HTTP/1.1\r\n\ + Host: disallowed.example.test\r\n\ + HOST: second.example.test\r\n\r\n"; + let raw = canonicalize_forward_host_header(raw, authority) + .expect("canonicalize Host before middleware"); + let request = crate::l7::rest::request_from_buffered_http("GET", "/api", "/api", raw) + .expect("build middleware request"); + let ctx = crate::l7::relay::L7EvalContext { + host: authority.into(), + port: 80, + policy_name: "test".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let runner = openshell_supervisor_middleware::ChainRunner::new( + openshell_supervisor_middleware_builtins::services() + .into_iter() + .next() + .expect("built-in middleware service"), + ); + let guard = forward_test_guard(); + let pipeline = ForwardMiddlewarePipeline { + ctx: &ctx, + scheme: "http", + runner: &runner, + generation_guard: &guard, + l7_reevaluation: None, + }; + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "redactor".into(), + implementation: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let (_app, mut client) = tokio::io::duplex(8192); + + let allowed = pipeline + .apply(request, &mut client, chain) + .await + .expect("middleware pipeline"); + let crate::l7::middleware::MiddlewareApplyResult::Allowed(request) = allowed else { + panic!("middleware-selected request should be allowed"); + }; + let rewritten = rewrite_forward_request( + &request.raw_header, + request.raw_header.len(), + "/api", + authority, + None, + false, + ) + .expect("rewrite middleware-selected request"); + let rewritten = String::from_utf8(rewritten).expect("rewritten request is UTF-8"); + assert_eq!( + rewritten + .split("\r\n") + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("host")) + }) + .collect::>(), + ["Host: allowed.example.test"] + ); + assert!(!rewritten.contains("disallowed.example.test")); + assert!(!rewritten.contains("second.example.test")); + } + #[test] fn test_rewrite_strips_proxy_headers() { let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nProxy-Authorization: Basic abc\r\nProxy-Connection: keep-alive\r\nAccept: */*\r\n\r\n"; - let result = - rewrite_forward_request(raw, raw.len(), "/p", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/p", "host", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!( !result_str @@ -8201,8 +8443,8 @@ network_policies: #[test] fn test_rewrite_replaces_connection_header() { let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nConnection: keep-alive\r\n\r\n"; - let result = - rewrite_forward_request(raw, raw.len(), "/p", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/p", "host", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.contains("Connection: close")); assert!(!result_str.contains("keep-alive")); @@ -8211,8 +8453,8 @@ network_policies: #[test] fn test_rewrite_strips_connection_nominated_headers() { let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; - let result = - rewrite_forward_request(raw, raw.len(), "/p", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/p", "host", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); let lower = result_str.to_ascii_lowercase(); @@ -8225,8 +8467,8 @@ network_policies: #[test] fn test_rewrite_preserves_body_overflow() { let raw = b"POST http://host/api HTTP/1.1\r\nHost: host\r\nContent-Length: 13\r\n\r\n{\"key\":\"val\"}"; - let result = - rewrite_forward_request(raw, raw.len(), "/api", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/api", "host", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.contains("{\"key\":\"val\"}")); assert!(result_str.contains("POST /api HTTP/1.1")); @@ -8235,8 +8477,8 @@ network_policies: #[test] fn test_rewrite_preserves_existing_via() { let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nVia: 1.0 upstream\r\n\r\n"; - let result = - rewrite_forward_request(raw, raw.len(), "/p", None, false).expect("should succeed"); + let result = rewrite_forward_request(raw, raw.len(), "/p", "host", None, false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.contains("Via: 1.0 upstream")); // Should not add a second Via header @@ -8259,7 +8501,7 @@ network_policies: .expect("canonicalization should succeed for the attack payload"); assert_eq!(canon.path, "/secret"); - let rewritten = rewrite_forward_request(raw, raw.len(), &canon.path, None, false) + let rewritten = rewrite_forward_request(raw, raw.len(), &canon.path, "host", None, false) .expect("rewrite_forward_request should succeed"); let rewritten_str = String::from_utf8_lossy(&rewritten); assert!( @@ -8285,8 +8527,9 @@ network_policies: _ => canon.path, }; - let rewritten = rewrite_forward_request(raw, raw.len(), &upstream_target, None, false) - .expect("rewrite_forward_request should succeed"); + let rewritten = + rewrite_forward_request(raw, raw.len(), &upstream_target, "host", None, false) + .expect("rewrite_forward_request should succeed"); let rewritten_str = String::from_utf8_lossy(&rewritten); assert!( rewritten_str.starts_with( @@ -8304,8 +8547,9 @@ network_policies: .collect(), ); let raw = b"GET http://host/p HTTP/1.1\r\nHost: host\r\nAuthorization: Bearer openshell:resolve:env:ANTHROPIC_API_KEY\r\n\r\n"; - let result = rewrite_forward_request(raw, raw.len(), "/p", resolver.as_ref(), false) - .expect("should succeed"); + let result = + rewrite_forward_request(raw, raw.len(), "/p", "host", resolver.as_ref(), false) + .expect("should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.contains("Authorization: Bearer sk-test")); assert!(!result_str.contains("openshell:resolve:env:ANTHROPIC_API_KEY")); @@ -8411,6 +8655,7 @@ network_policies: raw.as_bytes(), raw.len(), "/api/messages", + "api.example.com", Some(&resolver), true, ) @@ -8459,8 +8704,15 @@ network_policies: Sec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover\r\n\ Sec-WebSocket-Version: 13\r\n\r\n"; - let result = rewrite_forward_request(raw.as_bytes(), raw.len(), "/ws", None, false) - .expect("websocket forward rewrite should succeed"); + let result = rewrite_forward_request( + raw.as_bytes(), + raw.len(), + "/ws", + "gateway.example.test", + None, + false, + ) + .expect("websocket forward rewrite should succeed"); let result_str = String::from_utf8_lossy(&result); assert!(result_str.starts_with("GET /ws HTTP/1.1\r\n")); @@ -8486,7 +8738,7 @@ network_policies: engine.reload(policy, policy_data).unwrap(); let raw = b"GET http://host/api HTTP/1.1\r\nHost: host\r\n\r\n"; - let rewritten = rewrite_forward_request(raw, raw.len(), "/api", None, false) + let rewritten = rewrite_forward_request(raw, raw.len(), "/api", "host", None, false) .expect("rewrite should succeed"); let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); @@ -8529,7 +8781,7 @@ network_policies: .unwrap(); let raw = b"POST http://host/api HTTP/1.1\r\nHost: host\r\nContent-Length: 4\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n"; - let rewritten = rewrite_forward_request(raw, raw.len(), "/api", None, false) + let rewritten = rewrite_forward_request(raw, raw.len(), "/api", "host", None, false) .expect("rewrite should succeed"); let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); From e79cfaa99e366ad9848c270ec3174d1e41acff59 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 15 Jul 2026 17:04:30 -0700 Subject: [PATCH 54/59] refactor(middleware): use registration names for attachments Signed-off-by: Piotr Mlocek --- architecture/gateway.md | 6 +- architecture/sandbox.md | 11 +- crates/openshell-policy/src/lib.rs | 2 +- crates/openshell-policy/src/middleware.rs | 2 +- crates/openshell-server/src/grpc/policy.rs | 2 +- crates/openshell-server/src/middleware.rs | 4 +- .../src/lib.rs | 16 +- .../src/regex.rs | 9 +- .../src/lib.rs | 563 +++++++++--------- .../src/l7/relay.rs | 59 +- docs/extensibility/supervisor-middleware.mdx | 16 +- docs/observability/logging.mdx | 2 +- docs/reference/gateway-config.mdx | 2 +- docs/reference/policy-schema.mdx | 2 +- docs/sandboxes/policies.mdx | 2 +- proto/sandbox.proto | 4 +- proto/supervisor_middleware.proto | 40 +- 17 files changed, 391 insertions(+), 351 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index f810b33c9d..cc649770ea 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -368,8 +368,10 @@ when the policy engine accepts the update. External supervisor middleware registration is operator-owned configuration under `[[openshell.supervisor.middleware]]`. At startup the gateway connects to -each service, validates its described bindings and operator body limit, and -rejects duplicate binding IDs. +each service and validates its described bindings and operator body limit. +Policies attach a complete external middleware by its operator-owned registration +name. Manifest bindings are identified by operation and phase, and each manifest +may declare at most one binding for an operation and phase pair. Before persisting a policy, the gateway asks each selected implementation to validate its config. The effective sandbox config contains only the registered services required by that policy; supervisors invoke those services directly on diff --git a/architecture/sandbox.md b/architecture/sandbox.md index ec6dc8ab9d..205d026475 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,6 +74,10 @@ and phases as enums. Built-ins run in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. +Policy entries resolve built-ins by their platform name and external middleware +by the operator-owned registration name. Each attachment selects the binding +for the active operation and phase. A manifest cannot declare two bindings for +the same operation and phase pair. The platform caps middleware bodies at 4 MiB and derives the gRPC transport limit from bounded request and response components, reserving 292 KiB for the largest valid protobuf envelope. Config, context, target, headers, mutations, @@ -90,7 +94,8 @@ because bindings have not been discovered yet. Registration and manifest body limits above the platform boundary fail before request-body allocation. External per-request reason, finding text, mutation errors, and diagnostic metadata are untrusted: the supervisor maps logged -findings to the validated binding ID and uses platform-owned failure codes. +findings to the validated operator registration name and uses platform-owned +failure codes. At startup, the supervisor installs the in-process registry before connecting to external services, so an external outage cannot remove built-in bindings. For a gateway policy snapshot, the supervisor prepares replacements off to the @@ -109,8 +114,8 @@ implementation is only a best-effort example for simple, self-contained token patterns; it does not infer values from keyword assignments, is not a parser-aware secret scanner, and makes no redaction guarantee. The gateway and supervisor inject -those services explicitly and discover their binding IDs through the same -`Describe` contract used by external services. Reusable compiled DNS host +those services explicitly and discover their operation/phase capabilities +through the same `Describe` contract used by external services. Reusable compiled DNS host patterns and selectors remain in `openshell-core::host_pattern`. `openshell-policy` validates policy-owned middleware structure without knowing diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 5dae917b66..926c3289de 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1917,7 +1917,7 @@ network_policies: ), ( middleware_config("redactor", ""), - "implementation must not be empty", + "middleware must not be empty", ), ( { diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index 7c294d8abe..e1e3723c65 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -209,7 +209,7 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { if middleware.middleware.is_empty() { violations.push(PolicyViolation::InvalidMiddlewareConfig { name: middleware.name.clone(), - reason: "implementation must not be empty".to_string(), + reason: "middleware must not be empty".to_string(), }); } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 89ed9c50c3..f9a888b0be 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1351,7 +1351,7 @@ pub(super) async fn handle_get_sandbox_config( if let Some(policy) = policy.as_ref() { state .middleware_registry - .ensure_policy_bindings_registered(policy) + .ensure_policy_middlewares_registered(policy) .map_err(|error| { Status::failed_precondition(format!( "effective policy middleware registration is invalid: {error}" diff --git a/crates/openshell-server/src/middleware.rs b/crates/openshell-server/src/middleware.rs index 806b8dcc4a..34538ad2c0 100644 --- a/crates/openshell-server/src/middleware.rs +++ b/crates/openshell-server/src/middleware.rs @@ -24,7 +24,7 @@ mod tests { use openshell_core::proto::NetworkMiddlewareConfig; #[tokio::test] - async fn unregistered_external_binding_is_rejected_before_admission() { + async fn unregistered_external_middleware_is_rejected_before_admission() { let policy = SandboxPolicy { network_middlewares: vec![NetworkMiddlewareConfig { name: "guard".into(), @@ -36,7 +36,7 @@ mod tests { let error = validate_policy(&MiddlewareRegistry::default(), &policy) .await - .expect_err("unregistered binding must fail"); + .expect_err("unregistered middleware must fail"); assert_eq!(error.code(), tonic::Code::InvalidArgument); assert!(error.message().contains("not registered")); } diff --git a/crates/openshell-supervisor-middleware-builtins/src/lib.rs b/crates/openshell-supervisor-middleware-builtins/src/lib.rs index 38b2018270..e23a228f05 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/lib.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/lib.rs @@ -15,7 +15,7 @@ use openshell_core::proto::{ }; use tonic::{Request, Response, Status}; -pub use regex::{BINDING_ID as BUILTIN_REGEX, RegexConfig, RegexMode}; +pub use regex::{NAME as BUILTIN_REGEX, RegexConfig, RegexMode}; /// Return the first-party services that the gateway and supervisor install. pub fn services() -> Vec> { @@ -33,7 +33,7 @@ pub fn validate_config(implementation: &str, config: &prost_types::Struct) -> Re } fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result { - match evaluation.binding_id.as_str() { + match evaluation.middleware_name.as_str() { BUILTIN_REGEX => regex::evaluate_http_request(evaluation), other => Err(miette!( "middleware implementation '{other}' is not a registered OpenShell built-in" @@ -41,8 +41,7 @@ fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result, ) -> Result, Status> { Ok(Response::new(MiddlewareManifest { - name: "openshell/builtins".into(), + name: BUILTIN_REGEX.into(), service_version: env!("CARGO_PKG_VERSION").into(), bindings: vec![regex::describe()], })) @@ -66,7 +65,7 @@ impl SupervisorMiddleware for BuiltinMiddlewareService { let request = request.into_inner(); let config = request.config.unwrap_or_default(); Ok(Response::new( - match validate_config(&request.binding_id, &config) { + match validate_config(&request.middleware_name, &config) { Ok(()) => ValidateConfigResponse { valid: true, reason: String::new(), @@ -116,7 +115,6 @@ mod tests { .expect("describe") .into_inner(); assert_eq!(manifest.bindings.len(), 1); - assert_eq!(manifest.bindings[0].id, BUILTIN_REGEX); assert_eq!( manifest.bindings[0].operation, SupervisorMiddlewareOperation::HttpRequest as i32 @@ -162,7 +160,7 @@ mod tests { #[test] fn regex_replacement_evaluates_through_binding() { let result = evaluate_http_request(&HttpRequestEvaluation { - binding_id: BUILTIN_REGEX.into(), + middleware_name: BUILTIN_REGEX.into(), body: br#"{"password":"top-secret","token":"sk-ABCDEFGHIJKLMNOP"}"#.to_vec(), config: Some(prost_types::Struct::default()), ..Default::default() @@ -189,7 +187,7 @@ mod tests { "\npassword=alpha\nnotpassword=omega" ); let result = evaluate_http_request(&HttpRequestEvaluation { - binding_id: BUILTIN_REGEX.into(), + middleware_name: BUILTIN_REGEX.into(), body: body.as_bytes().to_vec(), config: Some(prost_types::Struct::default()), ..Default::default() diff --git a/crates/openshell-supervisor-middleware-builtins/src/regex.rs b/crates/openshell-supervisor-middleware-builtins/src/regex.rs index b8872ce66f..bde6c94442 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/regex.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/regex.rs @@ -19,7 +19,7 @@ use openshell_core::proto::{ use regex::Regex; use serde::Deserialize; -pub const BINDING_ID: &str = "openshell/regex"; +pub const NAME: &str = "openshell/regex"; const MAX_BODY_BYTES: u64 = 256 * 1024; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] @@ -40,9 +40,7 @@ impl RegexConfig { pub fn from_struct(config: &prost_types::Struct) -> Result { serde_json::from_value(openshell_core::proto_struct::struct_to_json_value(config)).map_err( |error| { - miette!( - "invalid {BINDING_ID} config: {error}; this example supports only mode: redact" - ) + miette!("invalid {NAME} config: {error}; this example supports only mode: redact") }, ) } @@ -50,7 +48,6 @@ impl RegexConfig { pub fn describe() -> MiddlewareBinding { MiddlewareBinding { - id: BINDING_ID.into(), operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: MAX_BODY_BYTES, @@ -86,7 +83,7 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result for ChainEntry { } if value.middleware.is_empty() { return Err(miette!( - "middleware config '{}' must name an implementation", + "middleware config '{}' must reference a middleware", value.name )); } @@ -283,6 +283,10 @@ pub struct ChainRunner { } struct MiddlewareServiceState { + /// Policy-facing built-in name or operator-owned registration name. The + /// single-service test constructor leaves this empty and uses the manifest + /// name after Describe. + attachment_name: Option, service: Arc, manifest: OnceCell, diagnostic_policy: MiddlewareDiagnosticPolicy, @@ -297,12 +301,7 @@ impl MiddlewareServiceState { } else { parse_middleware_timeout(&binding.timeout) .map(|binding_timeout| binding_timeout.min(self.operator_timeout)) - .map_err(|reason| { - miette!( - "middleware binding '{}' has invalid timeout: {reason}", - binding.id - ) - }) + .map_err(|reason| miette!("middleware binding has invalid timeout: {reason}")) } } } @@ -333,11 +332,11 @@ impl MiddlewareDiagnosticPolicy { fn process_result( self, - binding: &MiddlewareBinding, + middleware_name: &str, result: &mut openshell_core::proto::HttpRequestResult, ) { if self == Self::Normalize { - normalize_untrusted_diagnostics(binding, result); + normalize_untrusted_diagnostics(middleware_name, result); } } @@ -359,7 +358,7 @@ impl MiddlewareDiagnosticPolicy { pub struct MiddlewareRegistry { services: Arc>>, registered_services: Arc>, - binding_ids: Arc>, + middleware_names: Arc>, } impl std::fmt::Debug for MiddlewareRegistry { @@ -368,7 +367,7 @@ impl std::fmt::Debug for MiddlewareRegistry { .debug_struct("MiddlewareRegistry") .field("service_count", &self.services.len()) .field("registered_service_count", &self.registered_services.len()) - .field("binding_count", &self.binding_ids.len()) + .field("middleware_count", &self.middleware_names.len()) .finish() } } @@ -376,7 +375,6 @@ impl std::fmt::Debug for MiddlewareRegistry { #[derive(Clone)] struct RegisteredMiddlewareService { registration: SupervisorMiddlewareService, - binding_ids: Vec, } impl Default for MiddlewareRegistry { @@ -384,15 +382,21 @@ impl Default for MiddlewareRegistry { Self { services: Arc::new(Vec::new()), registered_services: Arc::new(Vec::new()), - binding_ids: Arc::new(HashSet::new()), + middleware_names: Arc::new(HashSet::new()), } } } fn validate_registration(registration: &SupervisorMiddlewareService) -> Result { - if registration.name.trim().is_empty() { + if !is_stable_identifier(®istration.name) { + return Err(miette!( + "supervisor middleware registration names must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'" + )); + } + if registration.name.starts_with("openshell/") { return Err(miette!( - "supervisor middleware registration name cannot be empty" + "middleware registration '{}' cannot claim the reserved openshell/ namespace", + registration.name )); } if !registration.grpc_endpoint.starts_with("http://") @@ -433,108 +437,76 @@ fn is_stable_identifier(value: &str) -> bool { fn validate_body_limit(source: &str, binding: &MiddlewareBinding) -> Result { if binding.max_body_bytes == 0 { - return Err(miette!( - "middleware binding '{}' must advertise a non-zero body limit", - binding.id - )); + return Err(miette!("{source} must advertise a non-zero body limit")); } if binding.max_body_bytes > MAX_MIDDLEWARE_BODY_BYTES as u64 { return Err(miette!( - "{source} binding '{}' body limit exceeds the platform maximum of {MAX_MIDDLEWARE_BODY_BYTES}", - binding.id + "{source} body limit exceeds the platform maximum of {MAX_MIDDLEWARE_BODY_BYTES}" )); } - usize::try_from(binding.max_body_bytes).map_err(|_| { - miette!( - "middleware binding '{}' reports a body limit too large for this platform", - binding.id - ) - }) + usize::try_from(binding.max_body_bytes) + .map_err(|_| miette!("{source} reports a body limit too large for this platform")) } fn validate_manifest_bindings( source: &str, manifest: &MiddlewareManifest, operator_max_body_bytes: Option, - allow_reserved_bindings: bool, - known_binding_ids: &mut HashSet, -) -> Result> { +) -> Result<()> { if manifest.bindings.is_empty() { return Err(miette!("{source} describes no bindings")); } - let mut described_ids = Vec::with_capacity(manifest.bindings.len()); + let mut described_pairs = HashSet::with_capacity(manifest.bindings.len()); for binding in &manifest.bindings { - if !is_stable_identifier(&binding.id) { - return Err(miette!( - "{source} binding ids must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'" - )); - } - if !allow_reserved_bindings && binding.id.starts_with("openshell/") { - return Err(miette!( - "{source} cannot claim reserved binding '{}'", - binding.id - )); - } if binding.operation != HTTP_REQUEST_OPERATION as i32 || binding.phase != PRE_CREDENTIALS_PHASE as i32 { return Err(miette!( - "middleware binding '{}' must support HTTP_REQUEST/PRE_CREDENTIALS", - binding.id, + "{source} must support HTTP_REQUEST/PRE_CREDENTIALS" + )); + } + if !described_pairs.insert((binding.operation, binding.phase)) { + return Err(miette!( + "{source} describes more than one binding for HTTP_REQUEST/PRE_CREDENTIALS" )); } let advertised = validate_body_limit(source, binding)?; if !binding.timeout.trim().is_empty() { - parse_middleware_timeout(&binding.timeout).map_err(|reason| { - miette!( - "{source} binding '{}' has invalid timeout: {reason}", - binding.id - ) - })?; + parse_middleware_timeout(&binding.timeout) + .map_err(|reason| miette!("{source} has invalid timeout for binding: {reason}"))?; } if operator_max_body_bytes.is_some_and(|limit| limit > advertised) { return Err(miette!( - "{source} max_body_bytes ({}) exceeds binding '{}' capability ({advertised})", - operator_max_body_bytes.expect("operator limit checked above"), - binding.id + "{source} max_body_bytes ({}) exceeds the binding capability ({advertised})", + operator_max_body_bytes.expect("operator limit checked above") )); } - if !known_binding_ids.insert(binding.id.clone()) { - return Err(miette!( - "middleware binding '{}' is described by more than one service", - binding.id - )); - } - described_ids.push(binding.id.clone()); } - Ok(described_ids) + Ok(()) } fn validate_external_manifest( registration: &SupervisorMiddlewareService, manifest: &MiddlewareManifest, operator_max_body_bytes: usize, - known_binding_ids: &mut HashSet, -) -> Result> { +) -> Result<()> { validate_manifest_bindings( &format!("external middleware registration '{}'", registration.name), manifest, Some(operator_max_body_bytes), - false, - known_binding_ids, ) } /// External diagnostic text is untrusted and may contain request data. Keep -/// only values derived from the validated, startup-time binding identifier and -/// numeric finding counts; do not carry per-request free-form text into logs. +/// only values derived from the validated, operator-owned registration name +/// and numeric finding counts; do not carry per-request free-form text into +/// logs. fn normalize_untrusted_diagnostics( - binding: &MiddlewareBinding, + middleware_name: &str, result: &mut openshell_core::proto::HttpRequestResult, ) { - let reason_id: String = binding - .id + let reason_id: String = middleware_name .chars() .map(|character| { if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { @@ -547,7 +519,7 @@ fn normalize_untrusted_diagnostics( result.reason = format!("middleware_denied:{reason_id}"); result.metadata.clear(); for finding in &mut result.findings { - finding.r#type = format!("{}.finding", binding.id); + finding.r#type = format!("{middleware_name}.finding"); finding.label = EXTERNAL_FINDING_LABEL.to_string(); finding.confidence.clear(); finding.severity = match finding.severity.as_str() { @@ -656,8 +628,7 @@ impl MiddlewareRegistry { ) -> Result { let mut services = Vec::with_capacity(in_process_services.len() + registrations.len()); let mut registered_services = Vec::with_capacity(registrations.len()); - let mut registration_names = HashSet::new(); - let mut binding_ids = HashSet::new(); + let mut middleware_names = HashSet::new(); for service in in_process_services { let manifest = call_with_timeout( @@ -678,12 +649,25 @@ impl MiddlewareRegistry { } else { format!("in-process middleware service '{}'", manifest.name) }; - validate_manifest_bindings(&source, &manifest, None, true, &mut binding_ids)?; + if !is_stable_identifier(&manifest.name) { + return Err(miette!( + "in-process middleware names must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'" + )); + } + if !middleware_names.insert(manifest.name.clone()) { + return Err(miette!( + "duplicate supervisor middleware name '{}'", + manifest.name + )); + } + validate_manifest_bindings(&source, &manifest, None)?; + let attachment_name = manifest.name.clone(); let manifest_cell = OnceCell::new(); manifest_cell .set(manifest) .map_err(|_| miette!("middleware manifest cache initialized twice"))?; services.push(Arc::new(MiddlewareServiceState { + attachment_name: Some(attachment_name), service, manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, @@ -694,7 +678,7 @@ impl MiddlewareRegistry { for registration in registrations { let operator_timeout = validate_registration(®istration)?; - if !registration_names.insert(registration.name.clone()) { + if !middleware_names.insert(registration.name.clone()) { return Err(miette!( "duplicate supervisor middleware registration name '{}'", registration.name @@ -729,33 +713,26 @@ impl MiddlewareRegistry { safe_reason(&error.to_string()) ) })?; - let described_ids = validate_external_manifest( - ®istration, - &manifest, - operator_max_body_bytes, - &mut binding_ids, - )?; + validate_external_manifest(®istration, &manifest, operator_max_body_bytes)?; let manifest_cell = OnceCell::new(); manifest_cell .set(manifest) .map_err(|_| miette!("middleware manifest cache initialized twice"))?; services.push(Arc::new(MiddlewareServiceState { + attachment_name: Some(registration.name.clone()), service, manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, operator_max_body_bytes: Some(operator_max_body_bytes), operator_timeout, })); - registered_services.push(RegisteredMiddlewareService { - registration, - binding_ids: described_ids, - }); + registered_services.push(RegisteredMiddlewareService { registration }); } Ok(Self { services: Arc::new(services), registered_services: Arc::new(registered_services), - binding_ids: Arc::new(binding_ids), + middleware_names: Arc::new(middleware_names), }) } @@ -781,13 +758,13 @@ impl MiddlewareRegistry { Ok(()) } - /// Check that every policy binding still belongs to the current static + /// Check that every policy attachment still belongs to the current static /// registry without making a network call. - pub fn ensure_policy_bindings_registered(&self, policy: &SandboxPolicy) -> Result<()> { + pub fn ensure_policy_middlewares_registered(&self, policy: &SandboxPolicy) -> Result<()> { for config in &policy.network_middlewares { - if !self.binding_ids.contains(&config.middleware) { + if !self.middleware_names.contains(&config.middleware) { return Err(miette!( - "middleware binding '{}' used by config '{}' is not registered", + "middleware '{}' used by config '{}' is not registered", config.middleware, config.name )); @@ -811,12 +788,7 @@ impl MiddlewareRegistry { .collect(); self.registered_services .iter() - .filter(|service| { - service - .binding_ids - .iter() - .any(|binding| selected.contains(binding.as_str())) - }) + .filter(|service| selected.contains(service.registration.name.as_str())) .map(|service| service.registration.clone()) .collect() } @@ -833,6 +805,7 @@ impl ChainRunner { Self { registry: Arc::new(MiddlewareRegistry { services: Arc::new(vec![Arc::new(MiddlewareServiceState { + attachment_name: None, service, manifest: OnceCell::new(), diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, @@ -840,7 +813,7 @@ impl ChainRunner { operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT, })]), registered_services: Arc::new(Vec::new()), - binding_ids: Arc::new(HashSet::new()), + middleware_names: Arc::new(HashSet::new()), }), } } @@ -877,6 +850,23 @@ impl ChainRunner { Ok(manifests) } + fn attachment_name<'a>( + state: &'a MiddlewareServiceState, + manifest: &'a MiddlewareManifest, + ) -> &'a str { + state + .attachment_name + .as_deref() + .unwrap_or(manifest.name.as_str()) + } + + fn http_pre_credentials_binding(manifest: &MiddlewareManifest) -> Option<&MiddlewareBinding> { + manifest.bindings.iter().find(|binding| { + binding.operation == HTTP_REQUEST_OPERATION as i32 + && binding.phase == PRE_CREDENTIALS_PHASE as i32 + }) + } + pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result> { ensure_chain_capacity(entries.len())?; let manifests = self.manifests().await?; @@ -885,14 +875,16 @@ impl ChainRunner { entries .iter() .map(|entry| { - let described = manifests.iter().find_map(|(state, manifest)| { - manifest - .bindings - .iter() - .find(|binding| binding.id == entry.implementation) - .cloned() - .map(|binding| (Arc::clone(state), binding)) - }); + let described = manifests + .iter() + .find(|(state, manifest)| { + Self::attachment_name(state, manifest) == entry.implementation + }) + .and_then(|(state, manifest)| { + Self::http_pre_credentials_binding(manifest) + .cloned() + .map(|binding| (Arc::clone(state), binding)) + }); let (service, binding) = described.map_or((None, None), |(service, binding)| { (Some(service), Some(binding)) }); @@ -928,7 +920,7 @@ impl ChainRunner { pub async fn validate_config( &self, - implementation: &str, + middleware_name: &str, config: prost_types::Struct, ) -> Result<()> { if config.encoded_len() > MAX_MIDDLEWARE_CONFIG_BYTES { @@ -938,15 +930,12 @@ impl ChainRunner { } let manifests = self.manifests().await?; let Some((state, binding)) = manifests.iter().find_map(|(state, manifest)| { - manifest - .bindings - .iter() - .find(|binding| binding.id == implementation) + (Self::attachment_name(state, manifest) == middleware_name) + .then(|| Self::http_pre_credentials_binding(manifest)) + .flatten() .map(|binding| (state, binding)) }) else { - return Err(miette!( - "middleware binding '{implementation}' is not registered" - )); + return Err(miette!("middleware '{middleware_name}' is not registered")); }; let response = call_with_timeout( state.timeout_for_binding(binding)?, @@ -954,8 +943,8 @@ impl ChainRunner { state .service .validate_config(Request::new(ValidateConfigRequest { - binding_id: implementation.into(), config: Some(config), + middleware_name: middleware_name.into(), })), ) .await @@ -1121,7 +1110,7 @@ impl ChainRunner { service .diagnostic_policy - .process_result(binding, &mut result); + .process_result(&entry.entry.implementation, &mut result); let decision = match Decision::try_from(result.decision) { Ok(decision @ (Decision::Allow | Decision::Deny)) => decision, @@ -1323,7 +1312,6 @@ fn build_evaluation( body: &[u8], ) -> HttpRequestEvaluation { HttpRequestEvaluation { - binding_id: binding.id.clone(), phase: binding.phase, context: Some(RequestContext { request_id: input.request_id.clone(), @@ -1347,6 +1335,7 @@ fn build_evaluation( }) .collect(), body: body.to_vec(), + middleware_name: entry.entry.implementation.clone(), } } @@ -1568,7 +1557,7 @@ mod tests { } #[tokio::test] - async fn injected_service_bindings_drive_registration_checks() { + async fn injected_service_names_drive_registration_checks() { let registry = MiddlewareRegistry::connect_services(services(), Vec::new()) .await .expect("connect built-in service"); @@ -1581,48 +1570,38 @@ mod tests { ..Default::default() }; registry - .ensure_policy_bindings_registered(&policy) - .expect("described binding is registered"); - - let unknown = SandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "unknown".into(), - middleware: "openshell/unknown".into(), - ..Default::default() - }], - ..Default::default() - }; - assert!( - registry - .ensure_policy_bindings_registered(&unknown) - .is_err() - ); + .ensure_policy_middlewares_registered(&policy) + .expect("described middleware is registered"); } #[tokio::test] - async fn injected_services_cannot_duplicate_binding_ids() { + async fn injected_services_cannot_duplicate_middleware_names() { let first: Arc = Arc::new(ScriptedService { - binding_id: "openshell/test".into(), + manifest_name: "openshell/test".into(), max_body_bytes: 1024, result: allow_result(), }); let second: Arc = Arc::new(ScriptedService { - binding_id: "openshell/test".into(), + manifest_name: "openshell/test".into(), max_body_bytes: 1024, result: allow_result(), }); let error = MiddlewareRegistry::connect_services(vec![first, second], Vec::new()) .await - .expect_err("duplicate injected binding must fail registry construction"); - assert!(error.to_string().contains("more than one service")); + .expect_err("duplicate injected middleware name must fail registry construction"); + assert!( + error + .to_string() + .contains("duplicate supervisor middleware name") + ); } /// A mock middleware that returns a fixed, caller-supplied result for every /// evaluation. Used to exercise chain behavior the built-in cannot produce /// (explicit deny, metadata, findings, unsafe header mutations). struct ScriptedService { - binding_id: String, + manifest_name: String, max_body_bytes: u64, result: openshell_core::proto::HttpRequestResult, } @@ -1634,10 +1613,9 @@ mod tests { _request: Request<()>, ) -> std::result::Result, tonic::Status> { Ok(tonic::Response::new(MiddlewareManifest { - name: "test/middleware".into(), + name: self.manifest_name.clone(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: self.binding_id.clone(), operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: self.max_body_bytes, @@ -1687,7 +1665,6 @@ mod tests { name: "test/slow".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/slow".into(), operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 4096, @@ -1724,8 +1701,9 @@ mod tests { } } - /// A two-binding service for exercising per-stage validation: `test/transform` - /// replaces the body, `test/second` records that it ran and allows. + /// A middleware attached twice for exercising per-stage validation. The + /// first policy config requests a body transformation; the second records + /// that it ran and allows. struct TwoStageService { second_ran: Arc, } @@ -1736,17 +1714,15 @@ mod tests { &self, _request: Request<()>, ) -> std::result::Result, tonic::Status> { - let binding = |id: &str| MiddlewareBinding { - id: id.into(), - operation: SupervisorMiddlewareOperation::HttpRequest as i32, - phase: SupervisorMiddlewarePhase::PreCredentials as i32, - max_body_bytes: 256 * 1024, - timeout: String::new(), - }; Ok(tonic::Response::new(MiddlewareManifest { name: "test/two-stage".into(), service_version: "test".into(), - bindings: vec![binding("test/transform"), binding("test/second")], + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: 256 * 1024, + timeout: String::new(), + }], })) } @@ -1774,7 +1750,14 @@ mod tests { > { let evaluation = request.into_inner(); let mut result = allow_result(); - if evaluation.binding_id == "test/transform" { + if evaluation.config.as_ref().is_some_and(|config| { + config.fields.get("transform").is_some_and(|value| { + matches!( + value.kind.as_ref(), + Some(prost_types::value::Kind::BoolValue(true)) + ) + }) + }) { result.body = b"TRANSFORMED".to_vec(); result.has_body = true; } else { @@ -1797,14 +1780,22 @@ mod tests { let runner = ChainRunner::new(service); let transform = ChainEntry { name: "transform".into(), - implementation: "test/transform".into(), + implementation: "test/two-stage".into(), order: 0, - config: prost_types::Struct::default(), + config: prost_types::Struct { + fields: std::iter::once(( + "transform".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::BoolValue(true)), + }, + )) + .collect(), + }, on_error: OnError::FailClosed, }; let second = ChainEntry { name: "second".into(), - implementation: "test/second".into(), + implementation: "test/two-stage".into(), order: 10, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -1851,14 +1842,22 @@ mod tests { let runner = ChainRunner::new(service); let transform = ChainEntry { name: "transform".into(), - implementation: "test/transform".into(), + implementation: "test/two-stage".into(), order: 0, - config: prost_types::Struct::default(), + config: prost_types::Struct { + fields: std::iter::once(( + "transform".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::BoolValue(true)), + }, + )) + .collect(), + }, on_error: OnError::FailClosed, }; let second = ChainEntry { name: "second".into(), - implementation: "test/second".into(), + implementation: "test/two-stage".into(), order: 10, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -1896,14 +1895,22 @@ mod tests { let entries = [ ChainEntry { name: "transform".into(), - implementation: "test/transform".into(), + implementation: "test/two-stage".into(), order: 0, - config: prost_types::Struct::default(), + config: prost_types::Struct { + fields: std::iter::once(( + "transform".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::BoolValue(true)), + }, + )) + .collect(), + }, on_error: OnError::FailClosed, }, ChainEntry { name: "second".into(), - implementation: "test/second".into(), + implementation: "test/two-stage".into(), order: 10, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -1939,7 +1946,7 @@ mod tests { fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService { ScriptedService { - binding_id: BUILTIN_REGEX.into(), + manifest_name: BUILTIN_REGEX.into(), max_body_bytes: 256 * 1024, result, } @@ -1960,6 +1967,7 @@ mod tests { /// A middleware that records every evaluation it receives and allows the /// request, for asserting what the supervisor actually sends to services. struct RecordingService { + validated: std::sync::Mutex>, received: std::sync::Mutex>, } @@ -1973,7 +1981,6 @@ mod tests { name: "test/recorder".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/recorder".into(), operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 4096, @@ -1984,11 +1991,15 @@ mod tests { async fn validate_config( &self, - _request: Request, + request: Request, ) -> std::result::Result< tonic::Response, tonic::Status, > { + self.validated + .lock() + .expect("validated config lock") + .push(request.into_inner()); Ok(tonic::Response::new( openshell_core::proto::ValidateConfigResponse { valid: true, @@ -2029,7 +2040,6 @@ mod tests { name: "test/header-chain".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/header-chain".into(), operation: SupervisorMiddlewareOperation::HttpRequest as i32, phase: SupervisorMiddlewarePhase::PreCredentials as i32, max_body_bytes: 4096, @@ -2100,21 +2110,21 @@ mod tests { let entries = [ ChainEntry { name: "first".into(), - implementation: "example/header-chain".into(), + implementation: "test/header-chain".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }, ChainEntry { name: "second".into(), - implementation: "example/header-chain".into(), + implementation: "test/header-chain".into(), order: 10, config: prost_types::Struct::default(), on_error: OnError::FailClosed, }, ChainEntry { name: "observer".into(), - implementation: "example/header-chain".into(), + implementation: "test/header-chain".into(), order: 20, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2144,13 +2154,18 @@ mod tests { // inspection differential. The service must see each entry in wire // order. let service = Arc::new(RecordingService { + validated: std::sync::Mutex::new(Vec::new()), received: std::sync::Mutex::new(Vec::new()), }); let recorder: Arc = service.clone(); let runner = ChainRunner::new(recorder); + runner + .validate_config("test/recorder", prost_types::Struct::default()) + .await + .expect("validate recorder config"); let recorder_entry = ChainEntry { name: "recorder".into(), - implementation: "example/recorder".into(), + implementation: "test/recorder".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2168,8 +2183,14 @@ mod tests { .expect("evaluate recording chain"); assert!(outcome.allowed); + let validated = service.validated.lock().expect("validated configs"); + assert_eq!(validated.len(), 1); + assert_eq!(validated[0].middleware_name, "test/recorder"); + drop(validated); + let received = service.received.lock().expect("recorded evaluations"); assert_eq!(received.len(), 1); + assert_eq!(received[0].middleware_name, "test/recorder"); let headers: Vec<(&str, &str)> = received[0] .headers .iter() @@ -2207,15 +2228,9 @@ mod tests { .await .expect("describe built-in service") .into_inner(); - let mut known = HashSet::new(); - validate_manifest_bindings( - "test built-in service", - &builtin_manifest, - None, - true, - &mut known, - ) - .expect("valid built-in manifest"); + validate_manifest_bindings("test built-in service", &builtin_manifest, None) + .expect("valid built-in manifest"); + let builtin_name = builtin_manifest.name.clone(); let builtin_manifest_cell = OnceCell::new(); builtin_manifest_cell .set(builtin_manifest) @@ -2228,18 +2243,15 @@ mod tests { .into_inner(); let operator_max_body_bytes = usize::try_from(registration.max_body_bytes).unwrap(); let operator_timeout = validate_registration(®istration).expect("valid registration"); - let binding_ids = validate_external_manifest( - ®istration, - &manifest, - operator_max_body_bytes, - &mut known, - ) - .expect("valid external manifest"); + validate_external_manifest(®istration, &manifest, operator_max_body_bytes) + .expect("valid external manifest"); let manifest_cell = OnceCell::new(); manifest_cell.set(manifest).expect("manifest cache"); + let registration_name = registration.name.clone(); MiddlewareRegistry { services: Arc::new(vec![ Arc::new(MiddlewareServiceState { + attachment_name: Some(builtin_name.clone()), service: builtin_service, manifest: builtin_manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve, @@ -2247,6 +2259,7 @@ mod tests { operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT, }), Arc::new(MiddlewareServiceState { + attachment_name: Some(registration_name.clone()), service, manifest: manifest_cell, diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize, @@ -2254,11 +2267,8 @@ mod tests { operator_timeout, }), ]), - registered_services: Arc::new(vec![RegisteredMiddlewareService { - registration, - binding_ids, - }]), - binding_ids: Arc::new(known), + registered_services: Arc::new(vec![RegisteredMiddlewareService { registration }]), + middleware_names: Arc::new(HashSet::from([builtin_name, registration_name])), } } @@ -2285,13 +2295,13 @@ mod tests { #[tokio::test] async fn descriptors_are_resolved_from_any_middleware_service() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: "example/redactor".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: allow_result(), })); let entry = ChainEntry { name: "external".into(), - implementation: "example/redactor".into(), + implementation: "test/middleware".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2321,7 +2331,7 @@ mod tests { #[tokio::test] async fn mixed_builtin_and_external_chain_uses_operator_limit() { let external = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: allow_result(), }); @@ -2329,7 +2339,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let external_entry = ChainEntry { name: "external".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2361,7 +2371,7 @@ mod tests { // own `on_error`, not the whole chain: the 1 KiB fail-open guard is // skipped while the 256 KiB fail-closed redactor still runs. let external = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: allow_result(), }); @@ -2369,7 +2379,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let guard_entry = ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailOpen, @@ -2404,7 +2414,7 @@ mod tests { // replacement is still over the 1 KiB guard limit, so the fail-closed // guard denies through its own `on_error` after the redactor ran. let external = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: allow_result(), }); @@ -2412,7 +2422,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let guard_entry = ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 10, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2445,14 +2455,13 @@ mod tests { name: "example/service".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/content-guard".into(), operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, timeout: String::new(), }], }; - let error = validate_external_manifest(®istration, &manifest, 4097, &mut HashSet::new()) + let error = validate_external_manifest(®istration, &manifest, 4097) .expect_err("operator limit must fit capability"); assert!(error.to_string().contains("exceeds")); } @@ -2472,54 +2481,39 @@ mod tests { name: "example/service".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/content-guard".into(), operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: u64::MAX, timeout: String::new(), }], }; - let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) + let error = validate_external_manifest(®istration, &manifest, 4096) .expect_err("extreme advertised body limit must be rejected"); assert!(error.to_string().contains("platform maximum")); } #[test] - fn manifest_rejects_unstable_binding_identifier() { + fn manifest_rejects_duplicate_operation_phase_pairs() { let registration = external_registration(4096); - let manifest = MiddlewareManifest { - name: "example/service".into(), - service_version: "test".into(), - bindings: vec![MiddlewareBinding { - id: "example/content-guard\nforged".into(), - operation: HTTP_REQUEST_OPERATION as i32, - phase: PRE_CREDENTIALS_PHASE as i32, - max_body_bytes: 4096, - timeout: String::new(), - }], + let binding = || MiddlewareBinding { + operation: HTTP_REQUEST_OPERATION as i32, + phase: PRE_CREDENTIALS_PHASE as i32, + max_body_bytes: 4096, + timeout: String::new(), }; - let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) - .expect_err("control characters must be rejected in binding ids"); - assert!(error.to_string().contains("binding ids must be")); - } - - #[test] - fn external_manifest_cannot_claim_reserved_binding() { - let registration = external_registration(4096); let manifest = MiddlewareManifest { name: "example/service".into(), service_version: "test".into(), - bindings: vec![MiddlewareBinding { - id: "openshell/regex".into(), - operation: HTTP_REQUEST_OPERATION as i32, - phase: PRE_CREDENTIALS_PHASE as i32, - max_body_bytes: 4096, - timeout: String::new(), - }], + bindings: vec![binding(), binding()], }; - let error = validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) - .expect_err("external service cannot claim reserved namespace"); - assert!(error.to_string().contains("reserved binding")); + + let error = validate_external_manifest(®istration, &manifest, 4096) + .expect_err("one service cannot advertise two bindings for the same pair"); + assert!( + error + .to_string() + .contains("more than one binding for HTTP_REQUEST/PRE_CREDENTIALS") + ); } #[test] @@ -2542,6 +2536,18 @@ mod tests { assert!(error.to_string().contains("http:// or https://")); } + #[test] + fn external_registration_name_is_stable_and_cannot_shadow_builtins() { + for name in ["", "guard\nforged", "openshell/regex"] { + let mut registration = external_registration(4096); + registration.name = name.into(); + assert!( + validate_registration(®istration).is_err(), + "registration name {name:?} must be rejected" + ); + } + } + #[test] fn registration_timeout_uses_default_and_operator_override() { let registration = external_registration(4096); @@ -2571,16 +2577,14 @@ mod tests { name: "example/service".into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { - id: "example/content-guard".into(), operation: HTTP_REQUEST_OPERATION as i32, phase: PRE_CREDENTIALS_PHASE as i32, max_body_bytes: 4096, timeout: timeout.into(), }], }; - let error = - validate_external_manifest(®istration, &manifest, 4096, &mut HashSet::new()) - .expect_err("out-of-bounds binding timeout must be rejected"); + let error = validate_external_manifest(®istration, &manifest, 4096) + .expect_err("out-of-bounds binding timeout must be rejected"); assert!(error.to_string().contains("invalid timeout")); } } @@ -2600,7 +2604,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let slow_entry = |on_error| ChainEntry { name: "slow".into(), - implementation: "example/slow".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error, @@ -2642,7 +2646,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let slow_entry = ChainEntry { name: "slow".into(), - implementation: "example/slow".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2677,7 +2681,7 @@ mod tests { let runner = ChainRunner::from_registry(registry); let slow_entry = ChainEntry { name: "slow".into(), - implementation: "example/slow".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2690,7 +2694,7 @@ mod tests { assert_eq!(described[0].timeout(), Duration::from_millis(10)); let validation_error = runner - .validate_config("example/slow", prost_types::Struct::default()) + .validate_config("local-guard-service", prost_types::Struct::default()) .await .expect_err("operator timeout must cap ValidateConfig"); assert!( @@ -2709,7 +2713,7 @@ mod tests { } #[tokio::test] - async fn external_registry_invokes_remote_service_over_grpc() { + async fn external_registry_attaches_same_service_under_multiple_names() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind test middleware"); @@ -2717,7 +2721,7 @@ mod tests { let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); let server = tonic::transport::Server::builder() .add_service(SupervisorMiddlewareServer::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: allow_result(), })) @@ -2728,13 +2732,18 @@ mod tests { let mut registration = external_registration(1024); registration.grpc_endpoint = format!("http://{address}"); - let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration.clone()]) - .await - .expect("connect external middleware"); + let mut second_registration = registration.clone(); + second_registration.name = "secondary-guard-service".into(); + let registry = MiddlewareRegistry::connect_services( + Vec::new(), + vec![registration.clone(), second_registration.clone()], + ) + .await + .expect("connect the same external middleware binding under two names"); let policy = SandboxPolicy { network_middlewares: vec![NetworkMiddlewareConfig { name: "guard".into(), - middleware: "example/content-guard".into(), + middleware: "local-guard-service".into(), order: 0, config: Some(prost_types::Struct::default()), on_error: "fail_closed".into(), @@ -2749,23 +2758,35 @@ mod tests { .expect("remote config validates"); assert_eq!( registry.required_services(Some(&policy)), - vec![registration] + vec![registration.clone()] ); let outcome = ChainRunner::from_registry(registry) .evaluate( - &[ChainEntry { - name: "guard".into(), - implementation: "example/content-guard".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: OnError::FailClosed, - }], + &[ + ChainEntry { + name: "primary".into(), + implementation: "local-guard-service".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ChainEntry { + name: "secondary".into(), + implementation: "secondary-guard-service".into(), + order: 10, + config: prost_types::Struct::default(), + on_error: OnError::FailClosed, + }, + ], input("hello"), ) .await .expect("remote evaluation"); assert!(outcome.allowed); + assert_eq!(outcome.applied.len(), 2); + assert_eq!(outcome.applied[0].implementation, registration.name); + assert_eq!(outcome.applied[1].implementation, second_registration.name); let _ = shutdown_tx.send(()); server_task @@ -2793,7 +2814,7 @@ mod tests { let server = tonic::transport::Server::builder() .add_service( SupervisorMiddlewareServer::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: MAX_MIDDLEWARE_BODY_BYTES as u64, result: openshell_core::proto::HttpRequestResult { reason: "r".repeat(MAX_MIDDLEWARE_REASON_BYTES - 128), @@ -2850,7 +2871,7 @@ mod tests { .evaluate( &[ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config, on_error: OnError::FailClosed, @@ -2885,7 +2906,7 @@ mod tests { let secret = "sk-secret-request-value"; let registration = external_registration(4096); let service = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, @@ -2906,7 +2927,7 @@ mod tests { .evaluate( &[ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2916,10 +2937,10 @@ mod tests { .await .expect("evaluate external middleware"); - assert_eq!(outcome.reason, "middleware_denied:example_content-guard"); + assert_eq!(outcome.reason, "middleware_denied:local-guard-service"); assert_eq!( outcome.findings[0].finding.r#type, - "example/content-guard.finding" + "local-guard-service.finding" ); assert_eq!(outcome.findings[0].finding.label, EXTERNAL_FINDING_LABEL); assert_eq!(outcome.findings[0].finding.severity, "medium"); @@ -2933,7 +2954,7 @@ mod tests { let secret = "sk-secret-request-value"; let registration = external_registration(4096); let service = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { header_mutations: vec![write_header( @@ -2949,7 +2970,7 @@ mod tests { .evaluate( &[ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -2987,7 +3008,7 @@ mod tests { for mutation in mutations { let service = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { header_mutations: vec![mutation], @@ -3002,7 +3023,7 @@ mod tests { .evaluate( &[ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -3024,7 +3045,7 @@ mod tests { async fn finding_overflow_is_an_invalid_response_governed_by_on_error() { let registration = external_registration(4096); let service = Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { findings: vec![Finding::default(); MAX_MIDDLEWARE_FINDINGS_PER_STAGE + 1], @@ -3039,7 +3060,7 @@ mod tests { .evaluate( &[ChainEntry { name: "guard".into(), - implementation: "example/content-guard".into(), + implementation: "local-guard-service".into(), order: 0, config: prost_types::Struct::default(), on_error, @@ -3065,7 +3086,7 @@ mod tests { #[tokio::test] async fn maximum_chain_retains_findings_from_every_stage() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: "example/content-guard".into(), + manifest_name: "test/middleware".into(), max_body_bytes: 4096, result: openshell_core::proto::HttpRequestResult { findings: vec![ @@ -3084,7 +3105,7 @@ mod tests { let entries: Vec<_> = (0..MAX_MIDDLEWARE_CHAIN_STAGES) .map(|index| ChainEntry { name: format!("guard-{index}"), - implementation: "example/content-guard".into(), + implementation: "test/middleware".into(), order: i32::try_from(index).expect("bounded stage index"), config: prost_types::Struct::default(), on_error: OnError::FailClosed, @@ -3170,7 +3191,7 @@ mod tests { #[tokio::test] async fn deny_decision_ignores_oversized_replacement_under_fail_open() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_REGEX.into(), + manifest_name: BUILTIN_REGEX.into(), max_body_bytes: 4, result: openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, @@ -3291,7 +3312,7 @@ mod tests { #[tokio::test] async fn oversized_replacement_body_honors_on_error() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_REGEX.into(), + manifest_name: BUILTIN_REGEX.into(), max_body_bytes: 4, result: openshell_core::proto::HttpRequestResult { body: b"too large".to_vec(), @@ -3326,7 +3347,7 @@ mod tests { #[tokio::test] async fn oversized_request_body_honors_on_error() { let runner = ChainRunner::new(Arc::new(ScriptedService { - binding_id: BUILTIN_REGEX.into(), + manifest_name: BUILTIN_REGEX.into(), max_body_bytes: 4, result: allow_result(), })); diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 9bc2e4a0e9..abe046416a 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -3244,7 +3244,6 @@ network_policies: name: "test/rewriter".into(), service_version: "test".into(), bindings: vec![openshell_core::proto::MiddlewareBinding { - id: "example/rewriter".into(), operation: openshell_core::proto::SupervisorMiddlewareOperation::HttpRequest as i32, phase: openshell_core::proto::SupervisorMiddlewarePhase::PreCredentials @@ -3297,7 +3296,7 @@ network_policies: r#" network_middlewares: - name: rewriter - middleware: example/rewriter + middleware: test/rewriter on_error: fail_closed endpoints: include: ["api.example.test"] @@ -3467,7 +3466,7 @@ network_policies: let data = r#" network_middlewares: - name: rewriter - middleware: example/rewriter + middleware: test/rewriter on_error: fail_closed endpoints: include: ["api.example.test"] @@ -3726,14 +3725,17 @@ network_policies: ); } - /// A middleware service advertising two bindings with different body - /// limits, for exercising mixed-limit chain buffering at the relay level. - /// The redactor binding replaces the body; the guard binding allows as-is. - struct TwoLimitService; + /// One named middleware with one HTTP/pre-credentials binding. Two + /// instances exercise mixed-limit chain buffering at the relay level. + struct LimitService { + name: &'static str, + max_body_bytes: u64, + replacement: Option<&'static [u8]>, + } #[tonic::async_trait] impl openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware - for TwoLimitService + for LimitService { async fn describe( &self, @@ -3746,17 +3748,15 @@ network_policies: MiddlewareBinding, MiddlewareManifest, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, }; - let binding = |id: &str, max_body_bytes: u64| MiddlewareBinding { - id: id.into(), - operation: SupervisorMiddlewareOperation::HttpRequest as i32, - phase: SupervisorMiddlewarePhase::PreCredentials as i32, - max_body_bytes, - timeout: String::new(), - }; Ok(tonic::Response::new(MiddlewareManifest { - name: "test/two-limits".into(), + name: self.name.into(), service_version: "test".into(), - bindings: vec![binding("test/redactor", 8192), binding("test/guard", 16)], + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_body_bytes: self.max_body_bytes, + timeout: String::new(), + }], })) } @@ -3782,13 +3782,13 @@ network_policies: tonic::Response, tonic::Status, > { - let evaluation = request.into_inner(); + let _evaluation = request.into_inner(); let mut result = openshell_core::proto::HttpRequestResult { decision: openshell_core::proto::Decision::Allow as i32, ..Default::default() }; - if evaluation.binding_id == "test/redactor" { - result.body = b"[SCRUBBED BY TEST REDACTOR]".to_vec(); + if let Some(replacement) = self.replacement { + result.body = replacement.to_vec(); result.has_body = true; } Ok(tonic::Response::new(result)) @@ -3806,7 +3806,24 @@ network_policies: // chain taking the unbuffered over-capacity path. let (_config, tunnel_engine, ctx) = middleware_relay_context("openshell/regex", "fail_closed"); - let runner = ChainRunner::new(Arc::new(TwoLimitService)); + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + vec![ + Arc::new(LimitService { + name: "test/redactor", + max_body_bytes: 8192, + replacement: Some(b"[SCRUBBED BY TEST REDACTOR]"), + }), + Arc::new(LimitService { + name: "test/guard", + max_body_bytes: 16, + replacement: None, + }), + ], + Vec::new(), + ) + .await + .expect("connect named middleware services"); + let runner = ChainRunner::from_registry(registry); let chain = vec![ ChainEntry { name: "redact".into(), diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 0398bf8e75..830d091431 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -37,7 +37,7 @@ Middleware receives the request before credential injection. Operator-run servic `openshell/regex` is an example built-in middleware. It replaces only simple, self-contained token patterns in UTF-8 request bodies; the initial pattern recognizes `sk-` tokens. It does not infer values from keyword assignments such as JSON `password` fields. This best-effort text transformation is not parser-aware and does not guarantee that it will detect or fully remove sensitive values. Its `config` accepts one field, `mode: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Custom expressions are not configurable yet. -Operator-run services expose one or more binding IDs. Policies reference a binding ID, such as `example/content-guard`, rather than the gateway registration name. +Operator-run services expose bindings for supported operation and phase pairs. A binding is identified by its operation and phase; V1 supports only `HttpRequest/pre_credentials`. Policies attach the complete middleware by its operator-owned gateway registration name. ## Register a Middleware Service @@ -53,14 +53,14 @@ timeout = "500ms" | Field | Description | | --- | --- | -| `name` | Operator-facing registration name used in diagnostics. Policies do not reference this value. | +| `name` | Operator-owned registration name used by policy attachments and diagnostics. Names must be unique, and `openshell/` is reserved for built-ins. | | `grpc_endpoint` | Service address reachable from both the gateway and sandbox supervisors. Supports plaintext `http://` and TLS `https://` with platform trust roots. | | `max_body_bytes` | Operator limit applied to every binding exposed by the service, up to the 4 MiB platform maximum. | | `timeout` | Optional service-wide RPC timeout using an integer with an `ms` or `s` suffix. Defaults to `500ms`; valid values range from `10ms` through `30s`. | Each binding returned by `Describe` may advertise a shorter `timeout` using the same syntax and bounds. The operator-configured service timeout is a ceiling: OpenShell uses the smaller of the binding and service values. An omitted binding timeout inherits the service setting, and an omitted service setting uses the 500 ms platform default. OpenShell rejects an invalid timeout before accepting the manifest. The service timeout applies to `Describe`, while the effective binding timeout applies to `ValidateConfig` and `EvaluateHttpRequest`. -The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes a binding ID already owned by another service. Operator-run services cannot claim the reserved `openshell/` namespace. +The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes more than one binding for the same operation and phase. The manifest `name` is diagnostic metadata and does not need to match the operator registration name. Operator-run registration names cannot claim the reserved `openshell/` namespace. Registration is static. Restart the gateway after adding, removing, or changing a service. See [Gateway Configuration](/reference/gateway-config#supervisor-middleware-services) for the complete gateway TOML context. @@ -81,11 +81,11 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Each config has a policy-local `name`, a built-in or operator-provided binding ID in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. A policy accepts at most 10 middleware configs. +Each config has a policy-local `name`, a built-in or operator-owned registration name in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. A policy accepts at most 10 middleware configs. `include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Each config accepts at most 32 combined include and exclude patterns. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. -Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may reference the same binding and run as separate stages. Config names must be unique. Runtime selection defensively rejects chains with more than 10 stages. +Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may attach the same middleware and run as separate stages. Config names must be unique. Runtime selection defensively rejects chains with more than 10 stages. See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. @@ -147,11 +147,11 @@ When the effective sandbox configuration changes, a running supervisor validates Middleware activity is emitted through OpenShell's OCSF logging: -- Each invocation records its policy-local middleware name, binding, decision, transformation state, and failure state. -- A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the validated binding ID instead of service-provided per-request text. +- Each invocation records its policy-local config name, attached middleware name, decision, transformation state, and failure state. +- A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the operator-owned registration name instead of service-provided per-request text. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. -- Built-in findings include their type, label, and aggregate count. Operator-run findings use the validated binding ID and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 32 findings. Exceeding the per-stage cap is an invalid response handled through `on_error`. A maximum 10-stage chain retains and emits up to 320 findings without silently dropping findings from later stages. +- Built-in findings include their type, label, and aggregate count. Operator-run findings use the operator-owned registration name and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 32 findings. Exceeding the per-stage cap is an invalid response handled through `on_error`. A maximum 10-stage chain retains and emits up to 320 findings without silently dropping findings from later stages. - Registry reload success and failure are emitted as configuration state changes. See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 5941d34bec..bac936ee21 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -167,7 +167,7 @@ OCSF CONFIG:LOADED [INFO] Policy reloaded successfully [policy_hash:0cc0c2b52557 Denied `NET:` and `HTTP:` events carry a `[reason:...]` suffix that surfaces the decision detail from the event's `status_detail` field. The reason helps distinguish between policy misses, SSRF hardening, and L7 enforcement without inspecting the full OCSF JSONL record. -For supervisor middleware denials, `status_detail` contains an audit-safe reason. Built-in middleware can provide that reason directly. Operator-run denial reasons are derived from the validated binding ID, and operator-run failure details use platform-owned error codes; OpenShell does not copy per-request service text into logs. +For supervisor middleware denials, `status_detail` contains an audit-safe reason. Built-in middleware can provide that reason directly. Operator-run denial reasons are derived from the operator-owned registration name, and operator-run failure details use platform-owned error codes; OpenShell does not copy per-request service text into logs. Common reason phrases emitted by the sandbox include: diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a7a10848ba..6e2531b807 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -187,7 +187,7 @@ max_body_bytes = 262144 timeout = "500ms" ``` -Each service implements the supervisor middleware gRPC contract and may expose multiple binding IDs through `Describe`. Policies reference those binding IDs, not the registration `name`. The gateway rejects duplicate binding IDs across services and prevents operator-run services from claiming the reserved `openshell/` namespace. +Each service implements the supervisor middleware gRPC contract and exposes bindings through `Describe`. Policies reference the operator-owned registration `name`, attaching the complete middleware and all of its bindings. Bindings are identified by operation and phase. A manifest may expose at most one binding for each operation and phase pair; V1 supports only `HttpRequest/pre_credentials`, so a service currently exposes one binding. Registration names must be unique, and operator-run registrations cannot claim the reserved `openshell/` namespace. The service-reported manifest name is diagnostic metadata and does not need to match the registration name. The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index e7f065e84b..44a3dab0c9 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -496,7 +496,7 @@ network_middlewares: | Field | Type | Required | Description | |---|---|---|---| | `name` | string | Yes | Policy-local config name. Names must be unique within the list. | -| `middleware` | string | Yes | Built-in or operator-registered binding ID. `openshell/` is reserved for built-ins. | +| `middleware` | string | Yes | Built-in middleware name or operator-owned registration name. `openshell/` is reserved for built-ins. | | `order` | integer | No | Execution priority. Lower values run first; names break ties. Defaults to `0`. | | `config` | object | No | Implementation-owned configuration validated by the selected middleware. | | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 09075889c2..72a0ed1729 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -90,7 +90,7 @@ network_middlewares: Matching entries run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. -`openshell/regex` is an example built into the supervisor. It applies fixed regular expressions as a best-effort UTF-8 text transformation, without guarantees that sensitive values will be detected or fully removed. Custom expressions are not configurable yet. Operator-provided binding IDs must be registered before a policy can reference them. The gateway validates implementation-owned config before accepting the policy. +`openshell/regex` is an example built into the supervisor. It applies fixed regular expressions as a best-effort UTF-8 text transformation, without guarantees that sensitive values will be detected or fully removed. Custom expressions are not configurable yet. Operator-run middleware must be registered by name before a policy can reference it. The gateway validates implementation-owned config before accepting the policy. `on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a failed middleware is acceptable. Middleware applies only to HTTP traffic the supervisor can parse and inspect. Policy validation rejects a fail-closed selector that can cover a `tls: skip` endpoint. An all-`fail_open` match may cover the endpoint; the supervisor bypasses the middleware and emits a detection finding. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 8236556b63..a77b683dc2 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -70,7 +70,7 @@ message NetworkPolicyRule { message NetworkMiddlewareConfig { // Policy-local config name. string name = 1; - // Built-in or registered middleware implementation name. + // Built-in middleware name or operator-owned registration name. string middleware = 2; // Service-specific configuration. google.protobuf.Struct config = 3; @@ -368,7 +368,7 @@ message GetSandboxConfigResponse { // Connection details for one operator-registered supervisor middleware service. // V1 supports plaintext and server-authenticated TLS gRPC. message SupervisorMiddlewareService { - // Operator-facing registration name used for diagnostics. + // Operator-owned registration name used by policy attachments and diagnostics. string name = 1; // gRPC endpoint reachable from the sandbox supervisor. string grpc_endpoint = 2; diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index ca63b2dfe5..85d5518445 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -26,7 +26,8 @@ service SupervisorMiddleware { // exposes. The service is the operator-run gRPC server implementing // SupervisorMiddleware. message MiddlewareManifest { - // Human-readable middleware service name used for diagnostics. + // Human-readable middleware service name used only for diagnostics. This is + // not required to match an operator-owned registration name. string name = 1; // Release version of the middleware service implementation, used for // diagnostics. @@ -37,28 +38,26 @@ message MiddlewareManifest { // MiddlewareBinding declares one operation and phase supported by a service. message MiddlewareBinding { - // Stable binding id used by policy configuration and audit logs. - string id = 1; // Supported operation. V1 supports HTTP_REQUEST. - SupervisorMiddlewareOperation operation = 2; + SupervisorMiddlewareOperation operation = 1; // Supported evaluation phase. V1 supports PRE_CREDENTIALS. - SupervisorMiddlewarePhase phase = 3; + SupervisorMiddlewarePhase phase = 2; // Maximum request or replacement body this binding can process. - uint64 max_body_bytes = 4; + uint64 max_body_bytes = 3; // Optional binding-specific RPC timeout. Empty uses the operator-configured // service timeout, or the 500ms platform default when that is also omitted. // A non-empty value may shorten but cannot extend the operator timeout. // Values use an integer with an `ms` or `s` suffix and must be between // 10ms and 30s. - string timeout = 5; + string timeout = 4; } // ValidateConfigRequest contains one policy configuration to validate. message ValidateConfigRequest { - // Manifest binding id associated with this configuration. - string binding_id = 1; // Service-specific policy configuration. - google.protobuf.Struct config = 2; + google.protobuf.Struct config = 1; + // Built-in middleware name or operator-owned registration name. + string middleware_name = 2; } // ValidateConfigResponse reports whether a policy configuration is accepted. @@ -71,26 +70,26 @@ message ValidateConfigResponse { // HttpRequestEvaluation contains one buffered HTTP request to evaluate. message HttpRequestEvaluation { - // Manifest binding id selected for this evaluation. - string binding_id = 1; // Evaluation phase selected for this request. - SupervisorMiddlewarePhase phase = 2; + SupervisorMiddlewarePhase phase = 1; // Sandbox and request identity available to the supervisor. // The encoded context is limited to 4 KiB. - RequestContext context = 3; + RequestContext context = 2; // Validated service-specific policy configuration. // The encoded configuration is limited to 64 KiB. - google.protobuf.Struct config = 4; + google.protobuf.Struct config = 3; // Destination and HTTP request target. // The encoded target is limited to 32 KiB. - HttpRequestTarget target = 5; + HttpRequestTarget target = 4; // HTTP request headers before OpenShell injects credentials, in wire // order. Repeated header names are preserved as separate entries. Protected // credential, routing, framing, and hop-by-hop headers are omitted. // At most 128 lines and 64 KiB of encoded headers are included. - repeated HttpHeader headers = 6; + repeated HttpHeader headers = 5; // Buffered request body, limited to 4 MiB. Empty for a bodyless request. - bytes body = 7; + bytes body = 6; + // Built-in middleware name or operator-owned registration name. + string middleware_name = 7; } // HttpHeader is one request header line. @@ -227,8 +226,9 @@ message HttpRequestResult { // validated name/value data, and 64 KiB encoded are accepted. repeated HeaderMutation header_mutations = 5; // Audit-safe findings produced during evaluation. For operator-run services, - // OpenShell logs platform-owned fields derived from the validated binding id - // rather than service-provided type, label, confidence, or metadata text. + // OpenShell logs platform-owned fields derived from the operator-owned + // registration name rather than service-provided type, label, confidence, + // or metadata text. // At most 32 findings of at most 4 KiB encoded each are accepted per stage. // A policy selects at most 10 stages, so one chain retains at most 320. repeated Finding findings = 6; From b4039ecf187f033fe1553e7128a615bd2f2d29ae Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 16 Jul 2026 09:58:30 -0700 Subject: [PATCH 55/59] refactor(middleware): key policy configs by name Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 11 +- crates/openshell-policy/src/lib.rs | 201 ++++++++++-------- crates/openshell-policy/src/middleware.rs | 132 +++++++----- .../openshell-sandbox/src/sidecar_control.rs | 4 +- crates/openshell-server/src/grpc/policy.rs | 41 ++-- .../openshell-server/src/grpc/validation.rs | 22 +- crates/openshell-server/src/middleware.rs | 42 ++-- .../src/lib.rs | 54 ++--- .../data/sandbox-policy.rego | 2 +- .../src/l7/relay.rs | 14 +- .../openshell-supervisor-network/src/opa.rs | 148 ++++++++----- .../openshell-supervisor-network/src/proxy.rs | 2 +- docs/extensibility/supervisor-middleware.mdx | 11 +- docs/reference/policy-schema.mdx | 13 +- docs/sandboxes/policies.mdx | 12 +- proto/sandbox.proto | 13 +- 16 files changed, 414 insertions(+), 308 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 205d026475..050dc7d123 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -68,9 +68,10 @@ are relayed but are not currently parsed for policy enforcement. For admitted HTTP requests, the proxy can run an ordered supervisor middleware chain before credential injection. Host selectors choose the chain independently -of the network rule that admitted the request. Policy entries use integer order -values with stable name tie-breaking, and the gRPC contract represents operations -and phases as enums. Built-ins run in-process; +of the network rule that admitted the request. Policies key middleware configs by +stable policy-local identities, allow optional human-readable names, and require +unique integer order values. The gRPC contract represents operations and phases as enums. Built-ins run +in-process; operator-registered services are called directly from the supervisor over the common middleware gRPC contract. The gateway validates external service capabilities and implementation-owned config before delivery. @@ -122,8 +123,8 @@ patterns and selectors remain in `openshell-core::host_pattern`. which implementations are installed. The active middleware registry validates implementation-owned config before gateway admission. The supervisor's local-file path supplies its built-in catalog to the same JSON projection so it -retains early config validation. Rego exposes the middleware list as policy -data, but Rust performs selector validation, overlap detection, matching, chain +retains early config validation. Rego exposes the keyed middleware configs as +policy data, but Rust performs selector validation, overlap detection, matching, chain ordering, implementation discovery, and config validation. When provider layers are composed just in time, the gateway reruns structural validation on the complete effective policy before delivering it to a supervisor. diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 926c3289de..244518e5bc 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -53,8 +53,8 @@ struct PolicyFile { process: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] network_policies: BTreeMap, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - network_middlewares: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + network_middlewares: BTreeMap, } #[derive(Debug, Serialize, Deserialize)] @@ -1078,7 +1078,7 @@ pub fn restrictive_default_policy() -> SandboxPolicy { run_as_group: "sandbox".into(), }), network_policies: HashMap::new(), - network_middlewares: vec![], + network_middlewares: HashMap::default(), } } @@ -1138,10 +1138,14 @@ pub enum PolicyViolation { InvalidMiddlewareConfig { name: String, reason: String }, /// Too many middleware configurations are attached to one policy. TooManyMiddlewareConfigs { count: usize }, + /// Two middleware configurations use the same execution order. + DuplicateMiddlewareOrder { + order: i32, + first_name: String, + second_name: String, + }, /// Too many include and exclude patterns are attached to one middleware. TooManyMiddlewareSelectorPatterns { name: String, count: usize }, - /// Middleware configuration names must be unique. - DuplicateMiddlewareConfigName { name: String }, /// A middleware selector conflicts with an endpoint that skips TLS inspection. MiddlewareTlsSkipConflict { middleware_name: String, @@ -1222,6 +1226,16 @@ impl fmt::Display for PolicyViolation { openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS ) } + Self::DuplicateMiddlewareOrder { + order, + first_name, + second_name, + } => { + write!( + f, + "middleware configs '{first_name}' and '{second_name}' use duplicate order {order}" + ) + } Self::TooManyMiddlewareSelectorPatterns { name, count } => { write!( f, @@ -1229,9 +1243,6 @@ impl fmt::Display for PolicyViolation { openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS ) } - Self::DuplicateMiddlewareConfigName { name } => { - write!(f, "duplicate middleware config '{name}'") - } Self::MiddlewareTlsSkipConflict { middleware_name, policy_name, @@ -1508,7 +1519,8 @@ network_policies: let yaml = r#" version: 1 network_middlewares: - - name: global-redactor + global-redactor: + name: Global redactor middleware: openshell/regex order: 20 on_error: fail_open @@ -1517,7 +1529,7 @@ network_middlewares: exclude: ["internal.example.com"] config: mode: redact - - name: secondary-redactor + secondary-redactor: middleware: openshell/regex endpoints: include: ["api.example.com"] @@ -1533,28 +1545,21 @@ network_policies: "#; let proto = parse_sandbox_policy(yaml).expect("parse failed"); assert_eq!(proto.network_middlewares.len(), 2); - assert_eq!(proto.network_middlewares[0].name, "global-redactor"); - assert_eq!(proto.network_middlewares[0].middleware, "openshell/regex"); - assert_eq!(proto.network_middlewares[0].order, 20); - assert_eq!(proto.network_middlewares[0].on_error, "fail_open"); + let redactor = &proto.network_middlewares["global-redactor"]; + assert_eq!(redactor.name, "Global redactor"); + assert_eq!(redactor.middleware, "openshell/regex"); + assert_eq!(redactor.order, 20); + assert_eq!(redactor.on_error, "fail_open"); assert_eq!( - proto.network_middlewares[0] - .endpoints - .as_ref() - .expect("selector") - .include, + redactor.endpoints.as_ref().expect("selector").include, vec!["api.example.com", "*.service.test"] ); assert_eq!( - proto.network_middlewares[0] - .endpoints - .as_ref() - .expect("selector") - .exclude, + redactor.endpoints.as_ref().expect("selector").exclude, vec!["internal.example.com"] ); assert_eq!( - proto.network_middlewares[0] + redactor .config .as_ref() .expect("config") @@ -1563,6 +1568,10 @@ network_policies: .and_then(|value| value.kind.as_ref()), Some(&prost_types::value::Kind::StringValue("redact".into())) ); + assert_eq!( + proto.network_middlewares["secondary-redactor"].name, + "secondary-redactor" + ); let yaml_out = serialize_sandbox_policy(&proto).expect("serialize failed"); let reparsed = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); assert_eq!(reparsed.network_middlewares, proto.network_middlewares); @@ -1796,12 +1805,9 @@ network_policies: // ---- Policy validation tests ---- - fn middleware_config( - name: &str, - implementation: &str, - ) -> openshell_core::proto::NetworkMiddlewareConfig { + fn middleware_config(implementation: &str) -> openshell_core::proto::NetworkMiddlewareConfig { openshell_core::proto::NetworkMiddlewareConfig { - name: name.into(), + name: String::new(), middleware: implementation.into(), order: 0, config: None, @@ -1813,17 +1819,27 @@ network_policies: } } + fn add_middleware( + policy: &mut SandboxPolicy, + name: &str, + config: openshell_core::proto::NetworkMiddlewareConfig, + ) { + policy.network_middlewares.insert(name.into(), config); + } + #[test] fn structural_validation_defers_implementation_owned_config() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("future", "openshell/future"); + let mut middleware = middleware_config("openshell/future"); middleware.config = Some( openshell_core::proto_struct::json_object_to_struct( std::iter::once(("implementation_field".into(), serde_json::json!(42))).collect(), ) .unwrap(), ); - policy.network_middlewares.push(middleware); + policy + .network_middlewares + .insert("future".into(), middleware); validate_sandbox_policy(&policy) .expect("generic policy validation must not select installed implementations"); @@ -1832,12 +1848,13 @@ network_policies: #[test] fn json_validation_delegates_implementation_owned_config() { let data = serde_json::json!({ - "network_middlewares": [{ - "name": "future", + "network_middlewares": { + "future": { "middleware": "openshell/future", "config": {"implementation_field": 42}, "endpoints": {"include": ["api.example.com"]} - }] + } + } }); let violations = @@ -1854,13 +1871,16 @@ network_policies: #[test] fn json_validation_skips_config_callbacks_when_middleware_count_is_invalid() { - let configs: Vec<_> = (0..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS) + let configs: serde_json::Map = (0 + ..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS) .map(|index| { - serde_json::json!({ - "name": format!("middleware-{index}"), - "middleware": "openshell/regex", - "endpoints": {"include": ["api.example.com"]} - }) + ( + format!("middleware-{index}"), + serde_json::json!({ + "middleware": "openshell/regex", + "endpoints": {"include": ["api.example.com"]} + }), + ) }) .collect(); let data = serde_json::json!({"network_middlewares": configs}); @@ -1912,32 +1932,37 @@ network_policies: fn validate_rejects_invalid_middleware_control_fields() { let cases = [ ( - middleware_config("", "openshell/regex"), + "", + middleware_config("openshell/regex"), "name must not be empty", ), ( - middleware_config("redactor", ""), + "redactor", + middleware_config(""), "middleware must not be empty", ), ( + "redactor", { - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.on_error = "maybe".into(); middleware }, "invalid on_error", ), ( + "redactor", { - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.endpoints = None; middleware }, "endpoint selector is required", ), ( + "redactor", { - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.endpoints.as_mut().unwrap().include.clear(); middleware }, @@ -1945,9 +1970,9 @@ network_policies: ), ]; - for (middleware, expected) in cases { + for (name, middleware, expected) in cases { let mut policy = restrictive_default_policy(); - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, name, middleware); let errors = validate_sandbox_policy(&policy) .expect_err("invalid middleware must be rejected") .into_iter() @@ -1962,19 +1987,23 @@ network_policies: } #[test] - fn validate_rejects_duplicate_middleware_config_names() { + fn validate_rejects_duplicate_middleware_orders() { let mut policy = restrictive_default_policy(); - policy - .network_middlewares - .push(middleware_config("redactor", "openshell/regex")); - policy - .network_middlewares - .push(middleware_config("redactor", "openshell/regex")); - - let violations = validate_sandbox_policy(&policy).expect_err("duplicate name"); + let mut alpha = middleware_config("openshell/regex"); + alpha.order = 10; + add_middleware(&mut policy, "alpha", alpha); + let mut beta = middleware_config("openshell/regex"); + beta.order = 10; + add_middleware(&mut policy, "beta", beta); + + let violations = validate_sandbox_policy(&policy).expect_err("duplicate order"); assert!(violations.iter().any(|violation| matches!( violation, - PolicyViolation::DuplicateMiddlewareConfigName { name } if name == "redactor" + PolicyViolation::DuplicateMiddlewareOrder { + order: 10, + first_name, + second_name, + } if first_name == "alpha" && second_name == "beta" ))); } @@ -1982,10 +2011,10 @@ network_policies: fn validate_accepts_maximum_middleware_configs() { let mut policy = restrictive_default_policy(); for index in 0..openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { - policy.network_middlewares.push(middleware_config( - &format!("middleware-{index}"), - "openshell/regex", - )); + let name = format!("middleware-{index}"); + let mut config = middleware_config("openshell/regex"); + config.order = i32::try_from(index).unwrap(); + add_middleware(&mut policy, &name, config); } validate_sandbox_policy(&policy).expect("maximum middleware config count"); @@ -1995,10 +2024,10 @@ network_policies: fn validate_rejects_middleware_config_over_capacity() { let mut policy = restrictive_default_policy(); for index in 0..=openshell_core::middleware::MAX_MIDDLEWARE_CONFIGS { - policy.network_middlewares.push(middleware_config( - &format!("middleware-{index}"), - "openshell/regex", - )); + let name = format!("middleware-{index}"); + let mut config = middleware_config("openshell/regex"); + config.order = i32::try_from(index).unwrap(); + add_middleware(&mut policy, &name, config); } let violations = validate_sandbox_policy(&policy).expect_err("config count over capacity"); @@ -2012,13 +2041,13 @@ network_policies: #[test] fn validate_accepts_maximum_middleware_selector_patterns() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); let selector = middleware.endpoints.as_mut().expect("selector"); selector.exclude = vec![ "excluded.example.com".into(); openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS - 1 ]; - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); validate_sandbox_policy(&policy).expect("maximum selector pattern count"); } @@ -2026,13 +2055,13 @@ network_policies: #[test] fn validate_rejects_middleware_selector_patterns_over_capacity() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); let selector = middleware.endpoints.as_mut().expect("selector"); selector.exclude = vec![ "excluded.example.com".into(); openshell_core::middleware::MAX_MIDDLEWARE_SELECTOR_PATTERNS ]; - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); let violations = validate_sandbox_policy(&policy).expect_err("selector patterns over capacity"); @@ -2048,9 +2077,9 @@ network_policies: #[test] fn validate_rejects_malformed_middleware_selector_patterns() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.endpoints.as_mut().unwrap().include = vec!["api[.example.com".into()]; - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); let errors = validate_sandbox_policy(&policy) .expect_err("malformed selector") @@ -2073,9 +2102,11 @@ network_policies: #[test] fn validate_rejects_middleware_selector_matching_tls_skip_endpoint() { let mut policy = restrictive_default_policy(); - policy - .network_middlewares - .push(middleware_config("redactor", "openshell/regex")); + add_middleware( + &mut policy, + "redactor", + middleware_config("openshell/regex"), + ); policy.network_policies.insert( "api".into(), NetworkPolicyRule { @@ -2104,9 +2135,9 @@ network_policies: #[test] fn validate_accepts_fail_open_middleware_selector_matching_tls_skip_endpoint() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.on_error = "fail_open".into(); - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); policy.network_policies.insert( "api".into(), NetworkPolicyRule { @@ -2128,9 +2159,9 @@ network_policies: #[test] fn validate_rejects_explicit_fail_closed_middleware_on_tls_skip_endpoint() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.on_error = "fail_closed".into(); - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); policy.network_policies.insert( "api".into(), NetworkPolicyRule { @@ -2156,9 +2187,9 @@ network_policies: #[test] fn validate_rejects_concrete_selector_overlapping_tls_skip_wildcard() { let mut policy = restrictive_default_policy(); - let mut middleware = middleware_config("redactor", "openshell/regex"); + let mut middleware = middleware_config("openshell/regex"); middleware.endpoints.as_mut().unwrap().include = vec!["api.example.com".into()]; - policy.network_middlewares.push(middleware); + add_middleware(&mut policy, "redactor", middleware); policy.network_policies.insert( "api".into(), NetworkPolicyRule { @@ -2268,7 +2299,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), - network_middlewares: Vec::new(), + network_middlewares: HashMap::default(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } @@ -2650,7 +2681,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), - network_middlewares: Vec::new(), + network_middlewares: HashMap::default(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } @@ -2666,7 +2697,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), - network_middlewares: Vec::new(), + network_middlewares: HashMap::default(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } @@ -2745,7 +2776,7 @@ network_policies: filesystem: None, landlock: None, network_policies: HashMap::new(), - network_middlewares: Vec::new(), + network_middlewares: HashMap::default(), }; assert!(validate_sandbox_policy(&policy).is_ok()); } diff --git a/crates/openshell-policy/src/middleware.rs b/crates/openshell-policy/src/middleware.rs index e1e3723c65..9e323e6de0 100644 --- a/crates/openshell-policy/src/middleware.rs +++ b/crates/openshell-policy/src/middleware.rs @@ -3,7 +3,7 @@ //! YAML schema and protobuf conversion for supervisor middleware policies. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use openshell_core::middleware::{MAX_MIDDLEWARE_CONFIGS, MAX_MIDDLEWARE_SELECTOR_PATTERNS}; use openshell_core::proto::{ @@ -23,6 +23,7 @@ use openshell_core::host_pattern::{HostPattern, HostSelector}; #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct NetworkMiddlewareConfigDef { + #[serde(default, skip_serializing_if = "String::is_empty")] name: String, middleware: String, #[serde(default, skip_serializing_if = "is_default")] @@ -54,7 +55,7 @@ struct MiddlewareEndpointSelectorDef { #[derive(Debug, Default, Deserialize)] struct MiddlewareValidationPolicyDef { #[serde(default)] - network_middlewares: Vec, + network_middlewares: BTreeMap, #[serde(default)] network_policies: BTreeMap, } @@ -76,51 +77,65 @@ struct MiddlewareValidationEndpointDef { } pub fn into_proto( - definitions: Vec, -) -> Result, ProtoStructError> { + definitions: BTreeMap, +) -> Result, ProtoStructError> { definitions .into_iter() - .map(|definition| { - Ok(NetworkMiddlewareConfig { - name: definition.name, - middleware: definition.middleware, - order: definition.order, - config: Some(json_object_to_struct( - definition.config.into_iter().collect(), - )?), - on_error: definition.on_error, - endpoints: definition - .endpoints - .map(|selector| MiddlewareEndpointSelector { - include: selector.include, - exclude: selector.exclude, - }), - }) + .map(|(key, definition)| { + Ok(( + key.clone(), + NetworkMiddlewareConfig { + name: if definition.name.is_empty() { + key + } else { + definition.name + }, + middleware: definition.middleware, + order: definition.order, + config: Some(json_object_to_struct( + definition.config.into_iter().collect(), + )?), + on_error: definition.on_error, + endpoints: definition + .endpoints + .map(|selector| MiddlewareEndpointSelector { + include: selector.include, + exclude: selector.exclude, + }), + }, + )) }) .collect() } -pub fn from_proto(middlewares: &[NetworkMiddlewareConfig]) -> Vec { +pub fn from_proto( + middlewares: &HashMap, +) -> BTreeMap { middlewares .iter() - .map(|middleware| NetworkMiddlewareConfigDef { - name: middleware.name.clone(), - middleware: middleware.middleware.clone(), - order: middleware.order, - config: middleware - .config - .as_ref() - .map(struct_to_json_object) - .unwrap_or_default() - .into_iter() - .collect(), - on_error: middleware.on_error.clone(), - endpoints: middleware.endpoints.as_ref().map(|selector| { - MiddlewareEndpointSelectorDef { - include: selector.include.clone(), - exclude: selector.exclude.clone(), - } - }), + .map(|(name, middleware)| { + ( + name.clone(), + NetworkMiddlewareConfigDef { + name: middleware.name.clone(), + middleware: middleware.middleware.clone(), + order: middleware.order, + config: middleware + .config + .as_ref() + .map(struct_to_json_object) + .unwrap_or_default() + .into_iter() + .collect(), + on_error: middleware.on_error.clone(), + endpoints: middleware.endpoints.as_ref().map(|selector| { + MiddlewareEndpointSelectorDef { + include: selector.include.clone(), + exclude: selector.exclude.clone(), + } + }), + }, + ) }) .collect() } @@ -171,11 +186,11 @@ where }; let mut violations = validate(&policy); if policy.network_middlewares.len() <= MAX_MIDDLEWARE_CONFIGS { - for middleware in &policy.network_middlewares { + for (name, middleware) in &policy.network_middlewares { let config = middleware.config.clone().unwrap_or_default(); if let Err(reason) = validate_config(&middleware.middleware, &config) { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason, }); } @@ -186,29 +201,34 @@ where pub fn validate(policy: &SandboxPolicy) -> Vec { let mut violations = Vec::new(); - let mut names = HashSet::new(); - + let mut orders = BTreeMap::new(); if policy.network_middlewares.len() > MAX_MIDDLEWARE_CONFIGS { violations.push(PolicyViolation::TooManyMiddlewareConfigs { count: policy.network_middlewares.len(), }); } - for middleware in &policy.network_middlewares { - if middleware.name.is_empty() { + let mut middlewares: Vec<_> = policy.network_middlewares.iter().collect(); + middlewares.sort_by_key(|(name, _)| name.as_str()); + for (name, middleware) in middlewares { + if name.is_empty() { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: "name must not be empty".to_string(), }); - } else if !names.insert(middleware.name.clone()) { - violations.push(PolicyViolation::DuplicateMiddlewareConfigName { - name: middleware.name.clone(), + } + + if let Some(first_name) = orders.insert(middleware.order, name.clone()) { + violations.push(PolicyViolation::DuplicateMiddlewareOrder { + order: middleware.order, + first_name, + second_name: name.clone(), }); } if middleware.middleware.is_empty() { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: "middleware must not be empty".to_string(), }); } @@ -218,21 +238,21 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { "" | "fail_closed" | "fail_open" ) { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: format!("invalid on_error '{}'", middleware.on_error), }); } let Some(selector) = &middleware.endpoints else { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: "endpoint selector is required".to_string(), }); continue; }; if selector.include.is_empty() { violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: "endpoint selector must include at least one host pattern".to_string(), }); } @@ -242,7 +262,7 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { .saturating_add(selector.exclude.len()); if selector_patterns > MAX_MIDDLEWARE_SELECTOR_PATTERNS { violations.push(PolicyViolation::TooManyMiddlewareSelectorPatterns { - name: middleware.name.clone(), + name: name.clone(), count: selector_patterns, }); continue; @@ -252,7 +272,7 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { if let Err(reason) = HostPattern::new(pattern) { selector_valid = false; violations.push(PolicyViolation::InvalidMiddlewareConfig { - name: middleware.name.clone(), + name: name.clone(), reason: format!("endpoint selector pattern '{pattern}' is invalid: {reason}"), }); } @@ -279,7 +299,7 @@ pub fn validate(policy: &SandboxPolicy) -> Vec { }); if overlaps_tls_skip { violations.push(PolicyViolation::MiddlewareTlsSkipConflict { - middleware_name: middleware.name.clone(), + middleware_name: name.clone(), policy_name: policy_name.clone(), host: endpoint.host.clone(), }); diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs index 03cfeff886..c98dbc33bf 100644 --- a/crates/openshell-sandbox/src/sidecar_control.rs +++ b/crates/openshell-sandbox/src/sidecar_control.rs @@ -49,7 +49,7 @@ pub enum ControlUpdate { provider_child_env: HashMap, }, PolicyUpdated { - policy_proto: openshell_core::proto::SandboxPolicy, + policy_proto: Box, policy_hash: String, config_revision: u64, }, @@ -231,7 +231,7 @@ impl TryFrom for ControlUpdate { .into_diagnostic() .wrap_err("failed to decode sidecar policy update")?; Ok(Self::PolicyUpdated { - policy_proto, + policy_proto: Box::new(policy_proto), policy_hash, config_revision, }) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index f9a888b0be..fd74215a38 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3374,7 +3374,10 @@ fn deterministic_policy_hash(policy: &ProtoSandboxPolicy) -> String { } if !policy.network_middlewares.is_empty() { hasher.update(b"network_middlewares"); - for middleware in &policy.network_middlewares { + let mut entries: Vec<_> = policy.network_middlewares.iter().collect(); + entries.sort_by_key(|(name, _)| name.as_str()); + for (name, middleware) in entries { + hasher.update(name.as_bytes()); let encoded = middleware.encode_to_vec(); hasher.update( u64::try_from(encoded.len()) @@ -5311,16 +5314,18 @@ mod tests { .unwrap(); let policy = ProtoSandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "redactor".to_string(), - middleware: "openshell/regex".to_string(), - on_error: "fail_closed".to_string(), - endpoints: Some(MiddlewareEndpointSelector { - include: vec!["api.example.com".to_string()], - exclude: Vec::new(), - }), - ..Default::default() - }], + network_middlewares: HashMap::from([( + "redactor".to_string(), + NetworkMiddlewareConfig { + middleware: "openshell/regex".to_string(), + on_error: "fail_closed".to_string(), + endpoints: Some(MiddlewareEndpointSelector { + include: vec!["api.example.com".to_string()], + exclude: Vec::new(), + }), + ..Default::default() + }, + )]), ..Default::default() }; state @@ -10186,12 +10191,14 @@ mod tests { fn policy_hash_changes_when_network_middlewares_change() { let policy_a = ProtoSandboxPolicy::default(); let policy_b = ProtoSandboxPolicy { - network_middlewares: vec![openshell_core::proto::NetworkMiddlewareConfig { - name: "regex-redactor".into(), - middleware: "openshell/regex".into(), - on_error: "fail_closed".into(), - ..Default::default() - }], + network_middlewares: HashMap::from([( + "regex-redactor".into(), + openshell_core::proto::NetworkMiddlewareConfig { + middleware: "openshell/regex".into(), + on_error: "fail_closed".into(), + ..Default::default() + }, + )]), ..Default::default() }; diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 263f92254c..9e47a01524 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -1660,16 +1660,18 @@ mod tests { use openshell_core::proto::{MiddlewareEndpointSelector, NetworkMiddlewareConfig}; let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_middlewares.push(NetworkMiddlewareConfig { - name: "redactor".into(), - middleware: "openshell/regex".into(), - on_error: "maybe".into(), - endpoints: Some(MiddlewareEndpointSelector { - include: vec!["api[.example.com".into()], - exclude: Vec::new(), - }), - ..Default::default() - }); + policy.network_middlewares.insert( + "redactor".into(), + NetworkMiddlewareConfig { + middleware: "openshell/regex".into(), + on_error: "maybe".into(), + endpoints: Some(MiddlewareEndpointSelector { + include: vec!["api[.example.com".into()], + exclude: Vec::new(), + }), + ..Default::default() + }, + ); let err = validate_policy_safety(&policy).unwrap_err(); assert_eq!(err.code(), Code::InvalidArgument); diff --git a/crates/openshell-server/src/middleware.rs b/crates/openshell-server/src/middleware.rs index 34538ad2c0..31865b02a4 100644 --- a/crates/openshell-server/src/middleware.rs +++ b/crates/openshell-server/src/middleware.rs @@ -26,11 +26,13 @@ mod tests { #[tokio::test] async fn unregistered_external_middleware_is_rejected_before_admission() { let policy = SandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "guard".into(), - middleware: "example/content-guard".into(), - ..Default::default() - }], + network_middlewares: std::collections::HashMap::from([( + "guard".into(), + NetworkMiddlewareConfig { + middleware: "example/content-guard".into(), + ..Default::default() + }, + )]), ..Default::default() }; @@ -50,20 +52,22 @@ mod tests { .await .expect("built-in registry"); let policy = SandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "redactor".into(), - middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), - config: Some(prost_types::Struct { - fields: std::iter::once(( - "mode".into(), - prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("allow".into())), - }, - )) - .collect(), - }), - ..Default::default() - }], + network_middlewares: std::collections::HashMap::from([( + "redactor".into(), + NetworkMiddlewareConfig { + middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), + config: Some(prost_types::Struct { + fields: std::iter::once(( + "mode".into(), + prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("allow".into())), + }, + )) + .collect(), + }), + ..Default::default() + }, + )]), ..Default::default() }; diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 4e12540e9c..c6e6c33117 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -109,21 +109,21 @@ pub struct ChainEntry { pub on_error: OnError, } -impl TryFrom<&NetworkMiddlewareConfig> for ChainEntry { +impl TryFrom<(&str, &NetworkMiddlewareConfig)> for ChainEntry { type Error = miette::Report; - fn try_from(value: &NetworkMiddlewareConfig) -> Result { - if value.name.is_empty() { + fn try_from((name, value): (&str, &NetworkMiddlewareConfig)) -> Result { + if name.is_empty() { return Err(miette!("middleware config name cannot be empty")); } if value.middleware.is_empty() { return Err(miette!( "middleware config '{}' must reference a middleware", - value.name + name )); } Ok(Self { - name: value.name.clone(), + name: name.to_string(), implementation: value.middleware.clone(), order: value.order, config: value.config.clone().unwrap_or_default(), @@ -740,7 +740,7 @@ impl MiddlewareRegistry { pub async fn validate_policy_configs(&self, policy: &SandboxPolicy) -> Result<()> { ensure_config_capacity(policy.network_middlewares.len())?; let runner = ChainRunner::from_registry(self.clone()); - for config in &policy.network_middlewares { + for (name, config) in &policy.network_middlewares { runner .validate_config( &config.middleware, @@ -750,7 +750,7 @@ impl MiddlewareRegistry { .map_err(|error| { miette!( "middleware config '{}' is invalid: {}", - config.name, + name, safe_reason(&error.to_string()) ) })?; @@ -761,12 +761,12 @@ impl MiddlewareRegistry { /// Check that every policy attachment still belongs to the current static /// registry without making a network call. pub fn ensure_policy_middlewares_registered(&self, policy: &SandboxPolicy) -> Result<()> { - for config in &policy.network_middlewares { + for (name, config) in &policy.network_middlewares { if !self.middleware_names.contains(&config.middleware) { return Err(miette!( "middleware '{}' used by config '{}' is not registered", config.middleware, - config.name + name )); } } @@ -783,7 +783,7 @@ impl MiddlewareRegistry { }; let selected: HashSet<&str> = policy .network_middlewares - .iter() + .values() .map(|config| config.middleware.as_str()) .collect(); self.registered_services @@ -1277,7 +1277,8 @@ impl ChainRunner { } } -/// Sort middleware using the policy-defined priority and a stable name tie-breaker. +/// Sort middleware by policy-defined priority. Valid policies have unique order +/// values; the name comparison only keeps direct internal callers deterministic. pub fn sort_chain_entries(entries: &mut [ChainEntry]) { entries.sort_by(|left, right| { left.order @@ -1562,11 +1563,13 @@ mod tests { .await .expect("connect built-in service"); let policy = SandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "redactor".into(), - middleware: BUILTIN_REGEX.into(), - ..Default::default() - }], + network_middlewares: HashMap::from([( + "redactor".into(), + NetworkMiddlewareConfig { + middleware: BUILTIN_REGEX.into(), + ..Default::default() + }, + )]), ..Default::default() }; registry @@ -2741,14 +2744,17 @@ mod tests { .await .expect("connect the same external middleware binding under two names"); let policy = SandboxPolicy { - network_middlewares: vec![NetworkMiddlewareConfig { - name: "guard".into(), - middleware: "local-guard-service".into(), - order: 0, - config: Some(prost_types::Struct::default()), - on_error: "fail_closed".into(), - endpoints: None, - }], + network_middlewares: HashMap::from([( + "guard".into(), + NetworkMiddlewareConfig { + name: String::new(), + middleware: "local-guard-service".into(), + order: 0, + config: Some(prost_types::Struct::default()), + on_error: "fail_closed".into(), + endpoints: None, + }, + )]), ..Default::default() }; diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index 2684b018a8..780080c54a 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -870,7 +870,7 @@ matched_endpoint_config := _matching_endpoint_configs[0] if { # Expose middleware policy data to Rust. Selection and validation stay in Rust; # Rego does not evaluate middleware selectors. -network_middlewares := object.get(data, "network_middlewares", []) +network_middlewares := object.get(data, "network_middlewares", {}) _policy_has_exact_declared_endpoint(policy) if { some ep diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index abe046416a..599cf70491 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -2326,7 +2326,7 @@ network_policies: let data = format!( r#" network_middlewares: - - name: request-middleware + request-middleware: middleware: {middleware_impl} on_error: {on_error} endpoints: @@ -2968,7 +2968,7 @@ network_policies: async fn jsonrpc_middleware_fail_closed_does_not_reach_upstream() { let data = r#" network_middlewares: - - name: request-middleware + request-middleware: middleware: example/unavailable on_error: fail_closed endpoints: @@ -3295,7 +3295,7 @@ network_policies: let data = format!( r#" network_middlewares: - - name: rewriter + rewriter: middleware: test/rewriter on_error: fail_closed endpoints: @@ -3465,7 +3465,7 @@ network_policies: // rewrites the body into a denied mutation. let data = r#" network_middlewares: - - name: rewriter + rewriter: middleware: test/rewriter on_error: fail_closed endpoints: @@ -3575,7 +3575,7 @@ network_policies: let data = format!( r#" network_middlewares: - - name: guard + guard: middleware: example/unavailable on_error: {on_error} endpoints: @@ -4164,7 +4164,7 @@ network_policies: // host-selected middleware must still inspect and redact its body. let data = r#" network_middlewares: - - name: request-middleware + request-middleware: middleware: openshell/regex on_error: fail_closed endpoints: @@ -4261,7 +4261,7 @@ network_policies: // forwarded. let data = r#" network_middlewares: - - name: request-middleware + request-middleware: middleware: example/unavailable on_error: fail_closed endpoints: diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index c47b655483..610d372ff1 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -872,9 +872,30 @@ fn query_middleware_chain_locked( fn parse_middleware_configs(value: ®orus::Value) -> Result> { match value { regorus::Value::Undefined => Ok(Vec::new()), - regorus::Value::Array(values) => Ok(values.to_vec()), + regorus::Value::Object(configs) => configs + .iter() + .map(|(name, config)| { + let regorus::Value::String(_) = name else { + return Err(miette::miette!("network_middlewares keys must be strings")); + }; + let regorus::Value::Object(fields) = config else { + return Err(miette::miette!( + "network middleware config {name:?} must be an object" + )); + }; + let fields = fields + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .chain(std::iter::once(( + regorus::Value::String("__openshell_policy_key".into()), + name.clone(), + ))) + .collect::>(); + Ok(fields.into()) + }) + .collect(), other => Err(miette::miette!( - "network_middlewares must be an array, got {other:?}" + "network_middlewares must be an object, got {other:?}" )), } } @@ -907,7 +928,7 @@ fn middleware_selector_matches(config: ®orus::Value, host: &str) -> Result Result { - let name = get_str(value, "name").unwrap_or_default(); + let name = get_str(value, "__openshell_policy_key").unwrap_or_default(); let implementation = get_str(value, "middleware").unwrap_or_default(); Ok(ChainEntry { name, @@ -1666,15 +1687,18 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St }) .collect(); - let network_middlewares: Vec = proto - .network_middlewares - .iter() - .map(|mw| { + let mut middleware_entries: Vec<_> = proto.network_middlewares.iter().collect(); + middleware_entries.sort_by_key(|(name, _)| name.as_str()); + let network_middlewares: serde_json::Map = middleware_entries + .into_iter() + .map(|(name, mw)| { let mut value = serde_json::json!({ - "name": mw.name, "middleware": mw.middleware, "order": mw.order, }); + if !mw.name.is_empty() { + value["name"] = mw.name.clone().into(); + } if let Some(config) = &mw.config { value["config"] = openshell_core::proto_struct::struct_to_json_value(config); } @@ -1691,7 +1715,7 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St } value["endpoints"] = endpoints; } - value + (name.clone(), value) }) .collect(); @@ -1783,7 +1807,7 @@ mod tests { run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), } } @@ -2701,7 +2725,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: Vec::new(), + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto_with_pid_and_binary_identity_required(&proto, 0, false) .expect("engine from relaxed proto"); @@ -3244,7 +3268,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3316,7 +3340,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -3389,7 +3413,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4266,7 +4290,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4324,7 +4348,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4383,7 +4407,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4444,7 +4468,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -4504,7 +4528,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); @@ -5494,7 +5518,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); let input = NetworkInput { @@ -5549,7 +5573,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); let input = NetworkInput { @@ -5620,7 +5644,7 @@ process: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).expect("Failed to create engine from proto"); @@ -5851,7 +5875,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; let engine = OpaEngine::from_proto(&proto).unwrap(); // Port 443 @@ -6729,11 +6753,13 @@ network_policies: .expect("install last-known-good runtime"); let mut invalid = proto; - invalid.network_middlewares.push(NetworkMiddlewareConfig { - name: String::new(), - middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), - ..Default::default() - }); + invalid.network_middlewares.insert( + String::new(), + NetworkMiddlewareConfig { + middleware: openshell_supervisor_middleware_builtins::BUILTIN_REGEX.into(), + ..Default::default() + }, + ); let empty_registry = MiddlewareRegistry::connect_services(Vec::new(), Vec::new()) .await .expect("empty registry"); @@ -6986,7 +7012,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; // Build engine with our PID (symlink resolution will work via /proc/self/root/) @@ -7064,7 +7090,7 @@ network_policies: run_as_group: "sandbox".to_string(), }), network_policies, - network_middlewares: vec![], + network_middlewares: std::collections::HashMap::default(), }; // Initial load at pid=0 — no symlink expansion @@ -7108,22 +7134,22 @@ network_policies: } #[test] - fn middleware_chain_uses_configured_order_and_name_tie_breaker() { + fn middleware_chain_uses_configured_order() { let data = r#" network_middlewares: - - name: global-redactor + global-redactor: middleware: openshell/regex order: 20 endpoints: include: ["api.example.com"] - - name: policy-redactor + policy-redactor: middleware: openshell/regex order: 10 endpoints: include: ["api.example.com"] - - name: endpoint-redactor + endpoint-redactor: middleware: openshell/regex - order: 10 + order: 5 endpoints: include: ["api.example.com"] network_policies: @@ -7203,18 +7229,18 @@ network_policies: fn middleware_chain_uses_dns_label_glob_semantics() { let data = r#" network_middlewares: - - name: single-label + single-label: middleware: openshell/regex order: 10 endpoints: include: ["*.Example.COM"] exclude: ["trusted.example.com"] - - name: recursive + recursive: middleware: openshell/regex order: 20 endpoints: include: ["**.example.com"] - - name: intra-label + intra-label: middleware: openshell/regex order: 30 endpoints: @@ -7324,7 +7350,7 @@ host_match if { "invalid on_error", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex on_error: maybe endpoints: @@ -7333,25 +7359,27 @@ network_middlewares: "invalid on_error", ), ( - "duplicate names", + "duplicate order", r#" network_middlewares: - - name: redactor + alpha: middleware: openshell/regex + order: 10 endpoints: include: ["api.example.com"] - - name: redactor + beta: middleware: openshell/regex + order: 10 endpoints: - include: ["api.example.com"] + include: ["other.example.com"] "#, - "duplicate middleware config 'redactor'", + "duplicate order 10", ), ( "missing selector", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex "#, "endpoint selector is required", @@ -7360,7 +7388,7 @@ network_middlewares: "malformed selector", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex endpoints: include: ["api[.example.com"] @@ -7371,7 +7399,7 @@ network_middlewares: "tls skip selector", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex endpoints: include: ["api.example.com"] @@ -7390,7 +7418,7 @@ network_policies: "tls skip wildcard overlap", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex endpoints: include: ["api.example.com"] @@ -7430,7 +7458,7 @@ network_policies: "unknown built-in", r#" network_middlewares: - - name: unknown + unknown: middleware: openshell/unknown endpoints: include: ["api.example.com"] @@ -7441,7 +7469,7 @@ network_middlewares: "invalid regex config", r#" network_middlewares: - - name: redactor + redactor: middleware: openshell/regex config: mode: allow @@ -7466,15 +7494,17 @@ network_middlewares: #[test] fn from_proto_revalidates_middleware_policy() { let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_middlewares.push(NetworkMiddlewareConfig { - name: "redactor".into(), - middleware: "openshell/regex".into(), - endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { - include: vec!["api[.example.com".into()], - exclude: Vec::new(), - }), - ..Default::default() - }); + policy.network_middlewares.insert( + "redactor".into(), + NetworkMiddlewareConfig { + middleware: "openshell/regex".into(), + endpoints: Some(openshell_core::proto::MiddlewareEndpointSelector { + include: vec!["api[.example.com".into()], + exclude: Vec::new(), + }), + ..Default::default() + }, + ); let error = OpaEngine::from_proto(&policy) .err() diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 8f05a9c543..2bceb5c78f 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4963,7 +4963,7 @@ mod tests { let policy = include_str!("../data/sandbox-policy.rego"); let data = r#" network_middlewares: - - name: guard + guard: middleware: openshell/regex endpoints: include: ["api.example.com"] diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 830d091431..1f8b538e59 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -18,7 +18,7 @@ For each inspected HTTP request, the supervisor: 1. Evaluates network and L7 policy. 2. Selects middleware whose host selectors match the admitted destination. 3. Buffers the request body using the largest body limit in the selected chain. -4. Runs matching middleware by ascending `order`, using the policy-local name to break ties. +4. Runs matching middleware by ascending `order`. Policy validation rejects duplicate order values. 5. Re-checks body-aware protocol policy (GraphQL, JSON-RPC, MCP) after each stage that replaces the body. Every middleware receives a payload the policy admits, and a transformation cannot smuggle a denied or unparseable operation to a later stage or the upstream. 6. Applies allowed transformations, injects provider credentials, and forwards the request. @@ -66,11 +66,12 @@ Registration is static. Restart the gateway after adding, removing, or changing ## Apply Middleware with Policy -Add middleware configs to the top-level `network_middlewares` list: +Add middleware configs to the top-level `network_middlewares` map. Each key is the policy-local config name: ```yaml network_middlewares: - - name: regex-redactor + regex-redactor: + name: Redact API tokens middleware: openshell/regex order: 10 config: @@ -81,11 +82,11 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Each config has a policy-local `name`, a built-in or operator-owned registration name in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. A policy accepts at most 10 middleware configs. +Each config has a stable policy-local identity from its map key, an optional human-readable `name` that defaults to that key, a built-in or operator-owned registration name in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. The optional name does not replace the map key for attachment or future keyed updates. A policy accepts at most 10 middleware configs. `include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Each config accepts at most 32 combined include and exclude patterns. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. -Matching configs run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Different config names may attach the same middleware and run as separate stages. Config names must be unique. Runtime selection defensively rejects chains with more than 10 stages. +Matching configs run once each by ascending `order`; lower values run first. Order values must be unique across the complete policy, even when endpoint selectors do not overlap. The default order is `0`, so policies with multiple configs normally set explicit values. Different map keys may attach the same middleware and run as separate stages. Map keys are structurally unique. Runtime selection defensively rejects chains with more than 10 stages. See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 44a3dab0c9..721f16109d 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -20,7 +20,7 @@ filesystem_policy: { ... } landlock: { ... } process: { ... } network_policies: { ... } -network_middlewares: [ ... ] +network_middlewares: { ... } ``` | Field | Type | Required | Category | Description | @@ -30,7 +30,7 @@ network_middlewares: [ ... ] | `landlock` | object | No | Static | Configures Landlock LSM enforcement behavior. | | `process` | object | No | Static | Sets the user and group the agent process runs as. | | `network_policies` | map | No | Dynamic | Declares which binaries can reach which network endpoints. | -| `network_middlewares` | list | No | Dynamic | Selects ordered HTTP request middleware by destination host. | +| `network_middlewares` | map | No | Dynamic | Selects ordered HTTP request middleware by destination host. | Static fields are set at sandbox creation time. Changing them requires destroying and recreating the sandbox. Dynamic fields can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. @@ -478,11 +478,12 @@ Identifies an executable that is permitted to use the associated endpoints. **Category:** Dynamic -An ordered list of up to 10 middleware configs selected after network and L7 policy admit an HTTP request. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once by ascending `order`, with the policy-local name breaking ties, before provider credential injection. Runtime selection also enforces the 10-stage maximum. +A map of up to 10 middleware configs selected after network and L7 policy admit an HTTP request. Each map key is the stable policy-local config identity. Middleware selection is independent of the network policy entry that admitted the request. Every matching config runs once by ascending `order` before provider credential injection. Order values must be unique across the policy, and runtime selection also enforces the 10-stage maximum. ```yaml showLineNumbers={false} network_middlewares: - - name: regex-redactor + regex-redactor: + name: Redact API tokens middleware: openshell/regex order: 10 config: @@ -495,9 +496,9 @@ network_middlewares: | Field | Type | Required | Description | |---|---|---|---| -| `name` | string | Yes | Policy-local config name. Names must be unique within the list. | +| `name` | string | No | Human-readable name for the middleware config. Defaults to the map key, which remains its stable identity. | | `middleware` | string | Yes | Built-in middleware name or operator-owned registration name. `openshell/` is reserved for built-ins. | -| `order` | integer | No | Execution priority. Lower values run first; names break ties. Defaults to `0`. | +| `order` | integer | No | Execution priority. Lower values run first, and values must be unique across the policy. Defaults to `0`; therefore, policies with multiple configs normally specify it explicitly. | | `config` | object | No | Implementation-owned configuration validated by the selected middleware. | | `on_error` | string | No | `fail_closed` denies the request when the stage fails; `fail_open` skips the failed stage. Defaults to `fail_closed`. | | `endpoints` | object | Yes | Host selector with required non-empty `include` and optional `exclude` lists, limited to 32 combined patterns. Exclusions take precedence. | diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 72a0ed1729..990f260cea 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -46,7 +46,8 @@ network_policies: # Dynamic: ordered middleware selected independently by admitted host. network_middlewares: - - name: regex-redactor + regex-redactor: + name: Redact API tokens middleware: openshell/regex order: 10 config: @@ -69,15 +70,16 @@ Raw streams are connection-scoped and outside L7 live-reload guarantees. This in | `landlock` | Static | Configures Landlock LSM enforcement behavior. Set `compatibility` to `best_effort` (skip individual inaccessible paths while applying remaining rules) or `hard_requirement` (fail if any path is inaccessible or the required kernel ABI is unavailable). Refer to the [Policy Schema Reference](/reference/policy-schema#landlock) for the full behavior table. | | `process` | Static | Sets the OS-level identity for the agent process. `run_as_user` and `run_as_group` default to `sandbox`. Root (`root` or `0`) is rejected. The agent also runs with seccomp filters that block dangerous system calls. | | `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` goes through the proxy, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints without `protocol` allow the TCP stream through without inspecting payloads.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | -| `network_middlewares` | Dynamic | Declares ordered HTTP request middleware configs. After network and L7 policy admit a request, OpenShell matches each config's host selectors independently and runs matching entries by ascending `order`, using the policy-local name to break ties, before credential injection. | +| `network_middlewares` | Dynamic | Declares keyed HTTP request middleware configs. After network and L7 policy admit a request, OpenShell matches each config's host selectors independently and runs matching entries by their unique ascending `order` before credential injection. | ## Supervisor Middleware -Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. +Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the request: each keyed `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. ```yaml network_middlewares: - - name: regex-redactor + regex-redactor: + name: Redact API tokens middleware: openshell/regex order: 10 config: @@ -88,7 +90,7 @@ network_middlewares: exclude: ["trusted.example.com"] ``` -Matching entries run once each by ascending `order`; lower values run first and policy-local names break ties. The default order is `0`. Config names must be unique. Different config names may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. +Matching entries run once each by ascending `order`; lower values run first, and duplicate order values are rejected. The default order is `0`, so policies with multiple entries normally set it explicitly. Map keys are structurally unique. An optional `name` provides a human-readable label, defaults to the map key, and does not change the key used as the config identity. Different keys may use the same implementation and run as distinct stages. `exclude` takes precedence over `include`. `openshell/regex` is an example built into the supervisor. It applies fixed regular expressions as a best-effort UTF-8 text transformation, without guarantees that sensitive values will be detected or fully removed. Custom expressions are not configurable yet. Operator-run middleware must be registered by name before a policy can reference it. The gateway validates implementation-owned config before accepting the policy. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index a77b683dc2..837554d33d 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -27,9 +27,10 @@ message SandboxPolicy { ProcessPolicy process = 4; // Network access policies keyed by name (e.g. "claude_code", "gitlab"). map network_policies = 5; - // Reusable supervisor middleware configs for network egress. At most 10 - // configs are accepted, and at most 10 stages can be selected per request. - repeated NetworkMiddlewareConfig network_middlewares = 6; + // Reusable supervisor middleware configs for network egress, keyed by their + // policy-local names. At most 10 configs are accepted, and at most 10 stages + // can be selected per request. + map network_middlewares = 6; } // Filesystem access policy. @@ -58,7 +59,7 @@ message ProcessPolicy { // A named network access policy rule. message NetworkPolicyRule { - // Human-readable name for this policy rule. + // Human-readable name for this middleware config. string name = 1; // Allowed endpoint (host:port) pairs. repeated NetworkEndpoint endpoints = 2; @@ -68,7 +69,7 @@ message NetworkPolicyRule { // A reusable middleware config selected for admitted egress by host. message NetworkMiddlewareConfig { - // Policy-local config name. + // Human-readable name for this middleware config. string name = 1; // Built-in middleware name or operator-owned registration name. string middleware = 2; @@ -78,7 +79,7 @@ message NetworkMiddlewareConfig { string on_error = 4; // Host selector controlling which admitted destinations use this config. MiddlewareEndpointSelector endpoints = 5; - // Deterministic execution order. Lower values run first; names break ties. + // Execution order. Values must be unique within a policy; lower values run first. int32 order = 6; } From 01b260a94bfa4d26bb3051cb7dfe5621e45687f7 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 16 Jul 2026 10:01:33 -0700 Subject: [PATCH 56/59] docs(policy): restore policy rule name comment Signed-off-by: Piotr Mlocek --- proto/sandbox.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 837554d33d..9e32b2d306 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -59,7 +59,7 @@ message ProcessPolicy { // A named network access policy rule. message NetworkPolicyRule { - // Human-readable name for this middleware config. + // Human-readable name for this policy rule. string name = 1; // Allowed endpoint (host:port) pairs. repeated NetworkEndpoint endpoints = 2; From d7b29e0fb74f2bd61d022072d490936f7e2b89f9 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 16 Jul 2026 11:35:55 -0700 Subject: [PATCH 57/59] fix(middleware): return dedicated denial responses Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 12 +- .../src/regex.rs | 1 + .../src/lib.rs | 123 ++++++++++- .../src/l7/middleware.rs | 116 +++++++++- .../src/l7/relay.rs | 209 ++++++++---------- .../src/l7/rest.rs | 104 +++++++++ .../openshell-supervisor-network/src/proxy.rs | 91 ++++++-- docs/extensibility/supervisor-middleware.mdx | 16 +- docs/observability/logging.mdx | 6 +- docs/reference/gateway-config.mdx | 2 +- proto/supervisor_middleware.proto | 10 +- 11 files changed, 537 insertions(+), 153 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 050dc7d123..62fcfce4f1 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -80,7 +80,7 @@ by the operator-owned registration name. Each attachment selects the binding for the active operation and phase. A manifest cannot declare two bindings for the same operation and phase pair. The platform caps middleware bodies at 4 MiB and derives the gRPC transport -limit from bounded request and response components, reserving 292 KiB for the +limit from bounded request and response components, reserving 293 KiB for the largest valid protobuf envelope. Config, context, target, headers, mutations, reasons, findings, and metadata each have explicit size or cardinality limits. Policies contain at most 10 middleware configs with 32 combined host selector @@ -93,10 +93,12 @@ timeout for each binding in its `Describe` manifest. Both use integer `ms` or so binding calls use the smaller value; `Describe` uses the service value because bindings have not been discovered yet. Registration and manifest body limits above the platform boundary fail before -request-body allocation. External per-request reason, finding text, mutation -errors, and diagnostic metadata are untrusted: the supervisor maps logged -findings to the validated operator registration name and uses platform-owned -failure codes. +request-body allocation. Per-request free-form reason text, external finding +text, mutation errors, and external diagnostic metadata are untrusted. Direct +denials identify the policy-local config and may expose only a tightly validated +stable reason code; responses and logs use platform-owned text. The supervisor +maps logged external findings to the validated operator registration name and +uses platform-owned failure codes. At startup, the supervisor installs the in-process registry before connecting to external services, so an external outage cannot remove built-in bindings. For a gateway policy snapshot, the supervisor prepares replacements off to the diff --git a/crates/openshell-supervisor-middleware-builtins/src/regex.rs b/crates/openshell-supervisor-middleware-builtins/src/regex.rs index bde6c94442..34e727430a 100644 --- a/crates/openshell-supervisor-middleware-builtins/src/regex.rs +++ b/crates/openshell-supervisor-middleware-builtins/src/regex.rs @@ -96,6 +96,7 @@ pub fn evaluate_http_request(evaluation: &HttpRequestEvaluation) -> Result, pub metadata: BTreeMap>, pub applied: Vec, + /// Present only when a middleware completed successfully and explicitly + /// denied the request. Fail-closed service errors and transformed-body + /// policy denials are not represented as middleware decisions. + pub denial: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MiddlewareDenial { + /// Stable policy-local middleware config identity. + pub config_name: String, + /// Validated service-defined code. Free-form service reason text is never + /// carried into client responses or security logs. + pub reason_code: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -435,6 +451,32 @@ fn is_stable_identifier(value: &str) -> bool { .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'/')) } +fn is_stable_reason_code(value: &str) -> bool { + value.len() <= MAX_MIDDLEWARE_REASON_CODE_BYTES + && value.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn middleware_denial_reason(config_name: &str, reason_code: Option<&str>) -> String { + let config_id: String = config_name + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character + } else { + '_' + } + }) + .take(MAX_STABLE_IDENTIFIER_BYTES) + .collect(); + reason_code.map_or_else( + || format!("middleware_denied:{config_id}"), + |code| format!("middleware_denied:{config_id}:{code}"), + ) +} + fn validate_body_limit(source: &str, binding: &MiddlewareBinding) -> Result { if binding.max_body_bytes == 0 { return Err(miette!("{source} must advertise a non-zero body limit")); @@ -582,6 +624,9 @@ fn validate_response_envelope( if result.reason.len() > MAX_MIDDLEWARE_REASON_BYTES { return Err("response_reason_over_capacity"); } + if !result.reason_code.is_empty() && !is_stable_reason_code(&result.reason_code) { + return Err("response_reason_code_invalid"); + } if result.header_mutations.len() > headers::MAX_HEADER_MUTATIONS { return Err("header_mutation_count_over_capacity"); } @@ -1018,6 +1063,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1034,6 +1080,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1051,6 +1098,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1085,6 +1133,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1103,6 +1152,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1126,6 +1176,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1133,6 +1184,12 @@ impl ChainRunner { }; if decision == Decision::Deny { + let reason_code = + (!result.reason_code.is_empty()).then(|| result.reason_code.clone()); + let denial = MiddlewareDenial { + config_name: entry.entry.name.clone(), + reason_code, + }; for finding in result.findings { findings.push(NamespacedFinding { middleware: entry.entry.name.clone(), @@ -1154,12 +1211,16 @@ impl ChainRunner { }); return Ok(ChainOutcome { allowed: false, - reason: safe_reason(&result.reason), + reason: middleware_denial_reason( + &denial.config_name, + denial.reason_code.as_deref(), + ), body, header_mutations, findings, metadata, applied, + denial: Some(denial), }); } @@ -1175,6 +1236,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1204,6 +1266,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1260,6 +1323,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }); } } @@ -1273,6 +1337,7 @@ impl ChainRunner { findings, metadata, applied, + denial: None, }) } } @@ -1964,6 +2029,7 @@ mod tests { header_mutations: Vec::new(), findings: Vec::new(), metadata: HashMap::new(), + reason_code: String::new(), } } @@ -2824,6 +2890,7 @@ mod tests { max_body_bytes: MAX_MIDDLEWARE_BODY_BYTES as u64, result: openshell_core::proto::HttpRequestResult { reason: "r".repeat(MAX_MIDDLEWARE_REASON_BYTES - 128), + reason_code: "r".repeat(MAX_MIDDLEWARE_REASON_CODE_BYTES), body: vec![b'x'; MAX_MIDDLEWARE_BODY_BYTES], has_body: true, header_mutations: vec![write_header( @@ -2900,10 +2967,10 @@ mod tests { #[test] fn grpc_envelope_headroom_matches_bounded_components() { - assert_eq!(MIDDLEWARE_GRPC_ENVELOPE_BYTES, 292 * 1024); + assert_eq!(MIDDLEWARE_GRPC_ENVELOPE_BYTES, 292 * 1024 + 64); assert_eq!( MIDDLEWARE_GRPC_MESSAGE_BYTES, - MAX_MIDDLEWARE_BODY_BYTES + 292 * 1024 + MAX_MIDDLEWARE_BODY_BYTES + 292 * 1024 + 64 ); } @@ -2917,6 +2984,7 @@ mod tests { result: openshell_core::proto::HttpRequestResult { decision: Decision::Deny as i32, reason: format!("denied body={secret}\nFINDING:FORGED"), + reason_code: "content_match".into(), findings: vec![Finding { r#type: format!("secret.{secret}\nforged"), label: format!("matched {secret}\nFINDING:FORGED"), @@ -2943,7 +3011,14 @@ mod tests { .await .expect("evaluate external middleware"); - assert_eq!(outcome.reason, "middleware_denied:local-guard-service"); + assert_eq!(outcome.reason, "middleware_denied:guard:content_match"); + assert_eq!( + outcome.denial, + Some(MiddlewareDenial { + config_name: "guard".into(), + reason_code: Some("content_match".into()), + }) + ); assert_eq!( outcome.findings[0].finding.r#type, "local-guard-service.finding" @@ -2955,6 +3030,32 @@ mod tests { assert!(!format!("{outcome:?}").contains("FINDING:FORGED")); } + #[tokio::test] + async fn invalid_reason_code_is_a_middleware_failure() { + let runner = ChainRunner::new(Arc::new(scripted_service( + openshell_core::proto::HttpRequestResult { + decision: Decision::Deny as i32, + reason_code: "Secret value!".into(), + ..allow_result() + }, + ))); + let outcome = runner + .evaluate( + &[entry("content-guard", OnError::FailClosed)], + input("hello"), + ) + .await + .expect("evaluate invalid reason code"); + + assert!(!outcome.allowed); + assert_eq!( + outcome.reason, + "middleware_failed: response_reason_code_invalid" + ); + assert!(outcome.denial.is_none()); + assert!(outcome.applied[0].failed); + } + #[tokio::test] async fn external_header_mutation_failure_uses_platform_reason() { let secret = "sk-secret-request-value"; @@ -3159,7 +3260,15 @@ mod tests { .await .expect("evaluate"); assert!(!outcome.allowed); - assert_eq!(outcome.reason, "blocked_by_policy"); + assert_eq!(outcome.reason, "middleware_denied:first"); + assert_eq!( + outcome.denial, + Some(MiddlewareDenial { + config_name: "first".into(), + reason_code: None, + }) + ); + assert!(!format!("{outcome:?}").contains("blocked_by_policy")); // The deny short-circuits the chain: the second middleware never runs. assert_eq!(outcome.applied.len(), 1); assert_eq!(outcome.applied[0].decision, Decision::Deny); @@ -3187,7 +3296,7 @@ mod tests { .expect("evaluate"); assert!(!outcome.allowed); - assert_eq!(outcome.reason, "blocked_by_policy"); + assert_eq!(outcome.reason, "middleware_denied:guard"); assert!(outcome.header_mutations.is_empty()); assert_eq!(outcome.applied.len(), 1); assert_eq!(outcome.applied[0].decision, Decision::Deny); @@ -3214,7 +3323,7 @@ mod tests { .expect("evaluate"); assert!(!outcome.allowed); - assert_eq!(outcome.reason, "blocked_by_policy"); + assert_eq!(outcome.reason, "middleware_denied:guard"); assert_eq!(outcome.body, b"safe"); assert_eq!(outcome.applied.len(), 1); assert_eq!(outcome.applied[0].decision, Decision::Deny); diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 3998460db0..47a2cde010 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -15,7 +15,10 @@ use tokio::io::{AsyncRead, AsyncWrite}; pub enum MiddlewareApplyResult { Allowed(crate::l7::provider::L7Request), - Denied(String), + Denied { + reason: String, + denial: Option, + }, } /// How traffic a middleware chain can never inspect (h2c, non-HTTP TCP, @@ -156,7 +159,10 @@ pub async fn apply_middleware_chain_for_scheme( + req: &crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + reason: &str, + denial: Option<&openshell_supervisor_middleware::MiddlewareDenial>, + redacted_target: &str, +) -> Result<()> { + let context = Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }); + if let Some(denial) = denial { + crate::l7::rest::send_middleware_deny_response( + req, + &ctx.policy_name, + denial, + client, + Some(redacted_target), + context, + ) + .await + } else { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + req, + &ctx.policy_name, + reason, + client, + Some(redacted_target), + context, + ) + .await + } +} + pub(super) fn middleware_request_input( scheme: &str, req: &crate::l7::provider::L7Request, @@ -258,7 +304,10 @@ pub(super) fn resolve_unbuffered_body( return MiddlewareApplyResult::Allowed(req); } emit_middleware_body_unavailable(ctx, true); - MiddlewareApplyResult::Denied("middleware_failed: request_body_over_capacity".into()) + MiddlewareApplyResult::Denied { + reason: "middleware_failed: request_body_over_capacity".into(), + denial: None, + } } fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { @@ -508,7 +557,64 @@ fn emit_middleware_events( #[cfg(test)] mod tests { - use super::safe_middleware_headers; + use super::{safe_middleware_headers, send_middleware_rejection_response}; + use crate::l7::relay::L7EvalContext; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn direct_denial_uses_middleware_response_without_service_text() { + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "api-policy".into(), + binary_path: "/usr/bin/curl".into(), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let req = crate::l7::provider::L7Request { + action: "POST".into(), + target: "/v1/messages".into(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + let denial = openshell_supervisor_middleware::MiddlewareDenial { + config_name: "prototype-content-guard".into(), + reason_code: Some("content_match".into()), + }; + let (mut client, mut server) = tokio::io::duplex(4096); + + send_middleware_rejection_response( + &req, + &mut server, + &ctx, + "request body matched configured content secret-value", + Some(&denial), + "/v1/messages", + ) + .await + .expect("send denial"); + drop(server); + + let mut response = Vec::new(); + client + .read_to_end(&mut response) + .await + .expect("read denial"); + let response = String::from_utf8(response).expect("UTF-8 response"); + let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); + let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); + assert_eq!(body["error"], "middleware_denied"); + assert_eq!(body["middleware"], "prototype-content-guard"); + assert_eq!(body["reason_code"], "content_match"); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + assert!(!body.to_string().contains("secret-value")); + } #[test] fn middleware_headers_exclude_origin_and_proxy_credentials() { diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 599cf70491..bab4b066eb 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -484,27 +484,23 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; + MiddlewareApplyResult::Denied { reason, denial } => { + let denied_request = crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + crate::l7::middleware::send_middleware_rejection_response( + &denied_request, + client, + ctx, + &reason, + denial.as_ref(), + &redacted_target, + ) + .await?; return Ok(()); } }; @@ -981,27 +977,23 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied(reason) => { - provider - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; + MiddlewareApplyResult::Denied { reason, denial } => { + let denied_request = crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + crate::l7::middleware::send_middleware_rejection_response( + &denied_request, + client, + ctx, + &reason, + denial.as_ref(), + &redacted_target, + ) + .await?; return Ok(()); } }; @@ -1255,27 +1247,23 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; + MiddlewareApplyResult::Denied { reason, denial } => { + let denied_request = crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + crate::l7::middleware::send_middleware_rejection_response( + &denied_request, + client, + ctx, + &reason, + denial.as_ref(), + &redacted_target, + ) + .await?; return Ok(()); } }; @@ -1488,27 +1476,23 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: request_info.action.clone(), - target: redacted_target.clone(), - query_params: request_info.query_params.clone(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; + MiddlewareApplyResult::Denied { reason, denial } => { + let denied_request = crate::l7::provider::L7Request { + action: request_info.action.clone(), + target: redacted_target.clone(), + query_params: request_info.query_params.clone(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + crate::l7::middleware::send_middleware_rejection_response( + &denied_request, + client, + ctx, + &reason, + denial.as_ref(), + &redacted_target, + ) + .await?; return Ok(()); } }; @@ -2129,27 +2113,23 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied(reason) => { - crate::l7::rest::RestProvider::default() - .deny_with_redacted_target( - &crate::l7::provider::L7Request { - action: "HTTP".into(), - target: redacted_target.clone(), - query_params: std::collections::HashMap::new(), - raw_header: Vec::new(), - body_length: crate::l7::provider::BodyLength::None, - }, - &ctx.policy_name, - &reason, - client, - Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext { - host: Some(&ctx.host), - port: Some(ctx.port), - binary: Some(&ctx.binary_path), - }), - ) - .await?; + MiddlewareApplyResult::Denied { reason, denial } => { + let denied_request = crate::l7::provider::L7Request { + action: "HTTP".into(), + target: redacted_target.clone(), + query_params: std::collections::HashMap::new(), + raw_header: Vec::new(), + body_length: crate::l7::provider::BodyLength::None, + }; + crate::l7::middleware::send_middleware_rejection_response( + &denied_request, + client, + ctx, + &reason, + denial.as_ref(), + &redacted_target, + ) + .await?; return Ok(()); } } @@ -3170,13 +3150,13 @@ network_policies: // Any fail-closed entry -> deny. assert!(matches!( resolve_unbuffered_body(&ctx, req(), &mixed_chain, true), - MiddlewareApplyResult::Denied(_) + MiddlewareApplyResult::Denied { .. } )); // Not recoverable (chunked overflow already consumed bytes) -> deny even // when every entry is fail-open. assert!(matches!( resolve_unbuffered_body(&ctx, req(), &open_chain, false), - MiddlewareApplyResult::Denied(_) + MiddlewareApplyResult::Denied { .. } )); } @@ -3879,7 +3859,7 @@ network_policies: "redactor must replace the body: {raw}" ); } - MiddlewareApplyResult::Denied(reason) => { + MiddlewareApplyResult::Denied { reason, .. } => { panic!("body within the largest stage limit must not fail the chain: {reason}") } } @@ -4043,6 +4023,7 @@ network_policies: transformed: true, failed: false, }], + denial: None, }; // Build the events directly rather than routing through the global @@ -4112,7 +4093,7 @@ network_policies: let denied_outcome = ChainOutcome { allowed: false, - reason: "request matched configured policy".into(), + reason: "middleware_denied:content-guard:content_match".into(), body: Vec::new(), header_mutations: Vec::new(), findings: Vec::new(), @@ -4124,6 +4105,10 @@ network_policies: transformed: false, failed: false, }], + denial: Some(openshell_supervisor_middleware::MiddlewareDenial { + config_name: "content-guard".into(), + reason_code: Some("content_match".into()), + }), }; let denied_events = middleware_events(&ctx, &req, &denied_outcome); let denied_http = denied_events @@ -4132,7 +4117,7 @@ network_policies: .expect("expected denied HTTP Activity event"); assert_eq!( denied_http.base().status_detail.as_deref(), - Some("request matched configured policy") + Some("middleware_denied:content-guard:content_match") ); let denied_json = denied_http.to_json().expect("serialize denied event"); assert_eq!(denied_json["unmapped"]["transformed"], false); @@ -4141,11 +4126,13 @@ network_policies: denied_http.format_shorthand(), "HTTP:POST [MED] DENIED POST http://api.example.test:443/v1/messages \ [policy:rest_api engine:middleware] \ - [failed:false transformed:false reason:request matched configured policy]" + [failed:false transformed:false \ + reason:middleware_denied:content-guard:content_match]" ); let external_failure_outcome = ChainOutcome { reason: "middleware_failed: header_mutation_invalid_name".into(), + denial: None, applied: vec![MiddlewareInvocation { failed: true, ..denied_outcome.applied[0].clone() diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 2542e4ddc3..a80d31ae7e 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -2161,6 +2161,30 @@ async fn send_deny_response( context: Option>, ) -> Result<()> { let body = deny_response_body(req, policy_name, reason, redacted_target, context); + send_forbidden_json(policy_name, body, client).await +} + +/// Send a middleware-specific 403 response without policy-advisor guidance. +/// +/// The policy config identity and optional reason code are validated platform +/// inputs. Free-form middleware reason text is intentionally absent. +pub(crate) async fn send_middleware_deny_response( + req: &L7Request, + policy_name: &str, + denial: &openshell_supervisor_middleware::MiddlewareDenial, + client: &mut C, + redacted_target: Option<&str>, + context: Option>, +) -> Result<()> { + let body = middleware_deny_response_body(req, policy_name, denial, redacted_target, context); + send_forbidden_json(policy_name, body, client).await +} + +async fn send_forbidden_json( + policy_name: &str, + body: serde_json::Value, + client: &mut C, +) -> Result<()> { let body_bytes = body.to_string(); let response = format!( "HTTP/1.1 403 Forbidden\r\n\ @@ -2182,6 +2206,45 @@ async fn send_deny_response( Ok(()) } +fn middleware_deny_response_body( + req: &L7Request, + policy_name: &str, + denial: &openshell_supervisor_middleware::MiddlewareDenial, + redacted_target: Option<&str>, + context: Option>, +) -> serde_json::Value { + let target = redacted_target.unwrap_or(&req.target); + let context = context.unwrap_or_default(); + let mut body = serde_json::Map::new(); + body.insert("error".to_string(), serde_json::json!("middleware_denied")); + body.insert( + "detail".to_string(), + serde_json::json!("Request rejected by configured middleware"), + ); + body.insert("policy".to_string(), serde_json::json!(policy_name)); + body.insert( + "middleware".to_string(), + serde_json::json!(denial.config_name), + ); + if let Some(reason_code) = &denial.reason_code { + body.insert("reason_code".to_string(), serde_json::json!(reason_code)); + } + body.insert("layer".to_string(), serde_json::json!("l7")); + body.insert("method".to_string(), serde_json::json!(req.action)); + body.insert("path".to_string(), serde_json::json!(target)); + if let Some(host) = non_empty(context.host) { + body.insert("host".to_string(), serde_json::json!(host)); + } + if let Some(port) = context.port { + body.insert("port".to_string(), serde_json::json!(port)); + } + if let Some(binary) = non_empty(context.binary) { + body.insert("binary".to_string(), serde_json::json!(binary)); + } + + serde_json::Value::Object(body) +} + fn deny_response_body( req: &L7Request, policy_name: &str, @@ -3542,6 +3605,47 @@ mod tests { ); } + #[test] + fn middleware_deny_response_identifies_config_without_policy_remediation() { + let _proposals = + openshell_core::proposals::test_helpers::ProposalsFlagGuard::set_blocking(true); + let req = L7Request { + action: "POST".to_string(), + target: "/v1/messages?token=secret-token".to_string(), + query_params: HashMap::new(), + raw_header: Vec::new(), + body_length: BodyLength::ContentLength(64), + }; + let denial = openshell_supervisor_middleware::MiddlewareDenial { + config_name: "prototype-content-guard".into(), + reason_code: Some("content_match".into()), + }; + + let body = middleware_deny_response_body( + &req, + "api-policy", + &denial, + Some("/v1/messages"), + Some(DenyResponseContext { + host: Some("api.example.com"), + port: Some(443), + binary: Some("/usr/bin/curl"), + }), + ); + + assert_eq!(body["error"], "middleware_denied"); + assert_eq!(body["detail"], "Request rejected by configured middleware"); + assert_eq!(body["middleware"], "prototype-content-guard"); + assert_eq!(body["reason_code"], "content_match"); + assert_eq!(body["policy"], "api-policy"); + assert_eq!(body["path"], "/v1/messages"); + assert!(body.get("rule").is_none()); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + assert!(body.get("agent_guidance").is_none()); + assert!(!body.to_string().contains("secret-token")); + } + #[tokio::test] async fn send_deny_response_writes_structured_json_403() { // Agent-readable next_steps is gated on the proposals feature flag. diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 2bceb5c78f..fbb5708bde 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4664,18 +4664,22 @@ async fn handle_forward_proxy( }; forward_request_bytes = match pipeline.apply(request, client, chain).await? { crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header, - crate::l7::middleware::MiddlewareApplyResult::Denied(reason) => { + crate::l7::middleware::MiddlewareApplyResult::Denied { reason, denial } => { emit_activity_simple(activity_tx, true, "middleware"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "middleware_denied", - &format!("{method} {host_lc}:{port}{path} denied by middleware: {reason}"), - ), - ) - .await?; + let response = denial.as_ref().map_or_else( + || { + build_json_error_response( + 403, + "Forbidden", + "middleware_denied", + &format!( + "{method} {host_lc}:{port}{path} denied by middleware: {reason}" + ), + ) + }, + |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + ); + respond(client, &response).await?; return Ok(()); } }; @@ -4883,6 +4887,38 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +fn build_middleware_deny_response( + policy_name: &str, + denial: &openshell_supervisor_middleware::MiddlewareDenial, +) -> Vec { + let mut body = serde_json::Map::new(); + body.insert("error".to_string(), serde_json::json!("middleware_denied")); + body.insert( + "detail".to_string(), + serde_json::json!("Request rejected by configured middleware"), + ); + body.insert("policy".to_string(), serde_json::json!(policy_name)); + body.insert( + "middleware".to_string(), + serde_json::json!(denial.config_name), + ); + if let Some(reason_code) = &denial.reason_code { + body.insert("reason_code".to_string(), serde_json::json!(reason_code)); + } + let body_str = serde_json::Value::Object(body).to_string(); + format!( + "HTTP/1.1 403 Forbidden\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + body_str.len(), + body_str, + ) + .into_bytes() +} + /// Detail shared by the fail-closed 503 body, the OCSF denial event, and the /// denial notification when a terminating CONNECT route has no TLS termination /// state available. @@ -5489,10 +5525,13 @@ network_policies: .expect("forward middleware pipeline"); match outcome { - crate::l7::middleware::MiddlewareApplyResult::Denied(reason) => assert!( - reason.contains("middleware transformation denied by policy"), - "{reason}" - ), + crate::l7::middleware::MiddlewareApplyResult::Denied { reason, denial } => { + assert!( + reason.contains("middleware transformation denied by policy"), + "{reason}" + ); + assert!(denial.is_none()); + } crate::l7::middleware::MiddlewareApplyResult::Allowed(_) => { panic!("policy-invalid transformed request must be denied") } @@ -9084,6 +9123,28 @@ network_policies: assert_eq!(body["detail"], "connection to api.example.com:443 failed"); } + #[test] + fn middleware_deny_response_uses_policy_config_identity() { + let resp = build_middleware_deny_response( + "api-policy", + &openshell_supervisor_middleware::MiddlewareDenial { + config_name: "prototype-content-guard".into(), + reason_code: Some("content_match".into()), + }, + ); + let resp_str = String::from_utf8(resp).unwrap(); + let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; + let body: serde_json::Value = serde_json::from_str(&resp_str[body_start..]).unwrap(); + + assert_eq!(body["error"], "middleware_denied"); + assert_eq!(body["detail"], "Request rejected by configured middleware"); + assert_eq!(body["middleware"], "prototype-content-guard"); + assert_eq!(body["reason_code"], "content_match"); + assert_eq!(body["policy"], "api-policy"); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + } + /// Locks the fail-closed response the CONNECT handler sends when TLS is /// detected but no termination state exists (ephemeral CA setup failed). /// The proxy must refuse the connection with a 503 instead of raw-tunneling diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 1f8b538e59..3ecd416065 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -101,7 +101,17 @@ See [Policy Schema](/reference/policy-schema#network-middleware) for the complet Use `fail_open` only when bypassing the middleware preserves the intended security policy. OpenShell emits a detection finding when a failed stage is bypassed. -An explicit deny decision always stops the chain and denies the request, regardless of `on_error`. +An explicit deny decision always stops the chain and denies the request, regardless of `on_error`. The HTTP response uses `error: middleware_denied`, identifies the policy-local middleware config, and omits policy-advisor remediation because the network and L7 allow rules already matched. OpenShell never copies the free-form middleware `reason` into the response or security logs. A service can instead return an optional stable `reason_code`: 1–64 bytes, starting with a lowercase ASCII letter and containing only lowercase ASCII letters, digits, and underscores. Invalid codes make the result a middleware failure governed by `on_error`. + +```json +{ + "error": "middleware_denied", + "detail": "Request rejected by configured middleware", + "policy": "api-policy", + "middleware": "prototype-content-guard", + "reason_code": "content_match" +} +``` Middleware decisions are enforced regardless of the endpoint's `enforcement` mode. `enforcement: audit` applies to an endpoint's network and L7 policy rules and does not bypass middleware: a middleware deny, or a failed `fail_closed` stage, blocks the request even on an audit endpoint. A middleware service that needs to observe traffic without blocking should return an allow decision with findings, which OpenShell emits as detection findings. @@ -114,7 +124,7 @@ Every middleware binding declares the largest request or replacement body it sup - A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. - The same per-stage limit applies to request bodies and replacement bodies. -The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-body protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB reason, 64 header mutations totaling at most 64 KiB encoded, 32 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 292 KiB so every platform-valid envelope fits. +The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-body protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB discarded free-form reason, a 64-byte validated reason code, 64 header mutations totaling at most 64 KiB encoded, 32 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 293 KiB so every platform-valid envelope fits. At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. OpenShell can apply `fail_open` to an oversized `Content-Length` before consuming body bytes. A chunked body can cross the limit only after bytes have been consumed, so OpenShell denies that request because it cannot safely resume the original stream. @@ -149,7 +159,7 @@ When the effective sandbox configuration changes, a running supervisor validates Middleware activity is emitted through OpenShell's OCSF logging: - Each invocation records its policy-local config name, attached middleware name, decision, transformation state, and failure state. -- A denied built-in invocation records its audit-safe reason. For operator-run services, OpenShell records a stable reason derived from the operator-owned registration name instead of service-provided per-request text. +- A denied invocation records a platform-owned reason derived from the policy-local config name and optional validated reason code. OpenShell does not record service-provided free-form reason text. - A bypass under `fail_open` emits a detection finding. - A required stage that fails closed emits a high-severity detection finding. - Built-in findings include their type, label, and aggregate count. Operator-run findings use the operator-owned registration name and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 32 findings. Exceeding the per-stage cap is an invalid response handled through `on_error`. A maximum 10-stage chain retains and emits up to 320 findings without silently dropping findings from later stages. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index bac936ee21..8c30c5cf04 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -167,7 +167,7 @@ OCSF CONFIG:LOADED [INFO] Policy reloaded successfully [policy_hash:0cc0c2b52557 Denied `NET:` and `HTTP:` events carry a `[reason:...]` suffix that surfaces the decision detail from the event's `status_detail` field. The reason helps distinguish between policy misses, SSRF hardening, and L7 enforcement without inspecting the full OCSF JSONL record. -For supervisor middleware denials, `status_detail` contains an audit-safe reason. Built-in middleware can provide that reason directly. Operator-run denial reasons are derived from the operator-owned registration name, and operator-run failure details use platform-owned error codes; OpenShell does not copy per-request service text into logs. +For supervisor middleware denials, `status_detail` contains a platform-owned reason derived from the policy-local middleware config name and optional validated reason code. Middleware failure details also use platform-owned error codes. OpenShell does not copy per-request service text into logs. Common reason phrases emitted by the sandbox include: @@ -205,9 +205,9 @@ An upstream that the proxy cannot reach returns `502 Bad Gateway`: } ``` -The `error` field is a short machine-readable code (`policy_denied`, `ssrf_denied`, `upstream_unreachable`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. +The `error` field is a short machine-readable code (`policy_denied`, `middleware_denied`, `ssrf_denied`, `upstream_unreachable`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. -For L7 REST denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When policy advisor is enabled, it also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. +For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When policy advisor is enabled, it also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware`, optionally includes a validated `reason_code`, and omits `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. ## Filesystem Sandbox Logs diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 6e2531b807..06e7bb2361 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -191,7 +191,7 @@ Each service implements the supervisor middleware gRPC contract and exposes bind The gateway connects to every registered service and validates `Describe` before it starts. The service must therefore be running before the gateway. Policy creation and full policy updates call `ValidateConfig`; an unavailable service or invalid middleware configuration rejects the policy before persistence. -`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 292 KiB so a maximum-size body and its protobuf envelope fit on the transport. +`max_body_bytes` is the operator limit for every binding exposed by the service. It must be greater than zero, no larger than each binding's advertised limit, and no larger than the 4 MiB platform maximum. OpenShell rejects an oversized value instead of silently clamping it. Middleware gRPC servers should allow messages of at least 4 MiB plus 293 KiB so a maximum-size body and its protobuf envelope fit on the transport. `timeout` is the operator-configured service-wide RPC timeout. It accepts the same compact duration syntax as gateway interceptors: an integer followed by `ms` or `s`, such as `500ms` or `2s`. Values must be between `10ms` and `30s`, inclusive. Omit the field to use the 500 ms platform default. A binding may advertise a shorter `timeout` in the `Describe` manifest, but it cannot extend the operator-configured deadline; OpenShell uses the smaller value. OpenShell validates both levels before accepting the service. The effective timeout covers `ValidateConfig` and `EvaluateHttpRequest`; `Describe` uses the service timeout because binding metadata is not available yet. diff --git a/proto/supervisor_middleware.proto b/proto/supervisor_middleware.proto index 85d5518445..dbde411c9f 100644 --- a/proto/supervisor_middleware.proto +++ b/proto/supervisor_middleware.proto @@ -209,9 +209,8 @@ message HeaderMutation { message HttpRequestResult { // Allow or deny decision for this request. Decision decision = 1; - // Audit-safe human-readable reason. OpenShell does not relay text from an - // operator-run service into denied responses or security logs. Limited to - // 4 KiB before normalization. + // Free-form service diagnostic. OpenShell does not relay this text into + // denied responses or security logs. Limited to 4 KiB before discarding. string reason = 2; // Replacement request body when has_body is true. Limited to 4 MiB. bytes body = 3; @@ -235,4 +234,9 @@ message HttpRequestResult { // Non-secret service-defined metadata included in diagnostics. At most 64 // entries and 32 KiB of combined key/value data are accepted. map metadata = 7; + // Optional stable machine-readable code for a deny decision. Codes must + // start with a lowercase ASCII letter and contain only lowercase ASCII + // letters, digits, and underscores, with a maximum length of 64 bytes. + // OpenShell may return this code to the requester, unlike free-form reason. + string reason_code = 8; } From d03dae6fe8f8e01af1393c09e7306c260b86c96a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 16 Jul 2026 11:40:03 -0700 Subject: [PATCH 58/59] docs(architecture): simplify middleware overview Signed-off-by: Piotr Mlocek --- architecture/sandbox.md | 112 +++++++++------------------------------- 1 file changed, 23 insertions(+), 89 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 62fcfce4f1..d2cb44d7b8 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -67,95 +67,29 @@ JSON-RPC responses and server-to-client MCP messages on response or SSE streams are relayed but are not currently parsed for policy enforcement. For admitted HTTP requests, the proxy can run an ordered supervisor middleware -chain before credential injection. Host selectors choose the chain independently -of the network rule that admitted the request. Policies key middleware configs by -stable policy-local identities, allow optional human-readable names, and require -unique integer order values. The gRPC contract represents operations and phases as enums. Built-ins run -in-process; -operator-registered services are called directly from the supervisor -over the common middleware gRPC contract. The gateway validates external -service capabilities and implementation-owned config before delivery. -Policy entries resolve built-ins by their platform name and external middleware -by the operator-owned registration name. Each attachment selects the binding -for the active operation and phase. A manifest cannot declare two bindings for -the same operation and phase pair. -The platform caps middleware bodies at 4 MiB and derives the gRPC transport -limit from bounded request and response components, reserving 293 KiB for the -largest valid protobuf envelope. Config, context, target, headers, mutations, -reasons, findings, and metadata each have explicit size or cardinality limits. -Policies contain at most 10 middleware configs with 32 combined host selector -patterns each. Runtime selection defensively allows at most 10 stages, each of -which can return 32 findings, for a derived chain maximum of 320 findings. -Middleware RPCs default to a 500 ms deadline. Operators can set a service-wide -`timeout` in gateway configuration, and a service can advertise a shorter -timeout for each binding in its `Describe` manifest. Both use integer `ms` or -`s` values bounded from 10 ms through 30 s. The operator value is a ceiling, -so binding calls use the smaller value; `Describe` uses the service value -because bindings have not been discovered yet. -Registration and manifest body limits above the platform boundary fail before -request-body allocation. Per-request free-form reason text, external finding -text, mutation errors, and external diagnostic metadata are untrusted. Direct -denials identify the policy-local config and may expose only a tightly validated -stable reason code; responses and logs use platform-owned text. The supervisor -maps logged external findings to the validated operator registration name and -uses platform-owned failure codes. -At startup, the supervisor installs the in-process registry before connecting -to external services, so an external outage cannot remove built-in bindings. -For a gateway policy snapshot, the supervisor prepares replacements off to the -side and installs them as one runtime generation. A policy-only change swaps -the policy engine alone and reuses the connected registry, so middleware -reachability cannot block it. The registry is rebuilt, reconnecting every -delivered service, only when the delivered service set changes or the installed -registry is degraded; that rebuild commits together with the policy engine. -A failure preserves the complete last-known-good runtime and leaves its applied -hash and registrations unchanged so the snapshot is retried. The generic -registry and chain runner live in -`openshell-supervisor-middleware`; -first-party implementations and their config schemas live in -`openshell-supervisor-middleware-builtins`. The initial `openshell/regex` -implementation is only a best-effort example for simple, self-contained token -patterns; it does not infer values from keyword assignments, is not a -parser-aware secret scanner, and makes no redaction guarantee. The -gateway and supervisor inject -those services explicitly and discover their operation/phase capabilities -through the same `Describe` contract used by external services. Reusable compiled DNS host -patterns and selectors remain in `openshell-core::host_pattern`. - -`openshell-policy` validates policy-owned middleware structure without knowing -which implementations are installed. The active middleware registry validates -implementation-owned config before gateway admission. The supervisor's -local-file path supplies its built-in catalog to the same JSON projection so it -retains early config validation. Rego exposes the keyed middleware configs as -policy data, but Rust performs selector validation, overlap detection, matching, chain -ordering, implementation discovery, and config validation. When provider layers -are composed just in time, the gateway reruns structural validation on the -complete effective policy before delivering it to a supervisor. - -The Rust HTTP pipeline makes transformed-body handling explicit for every -middleware invocation: body-independent protocols select a no-recheck mode, -while GraphQL, JSON-RPC, and MCP carry a policy evaluator bound to the captured -policy generation. Each replacement is reclassified and evaluated before the -next stage runs. Hard-deny classification is shared by CONNECT relays, -route-selected relays, and forward proxying; evaluator failures become -structured fail-closed outcomes instead of escaping the middleware pipeline. -The shared HTTP/1 request parser validates raw headers at ingress and rejects -obsolete folded continuations, colonless fields, whitespace before field-name -colons, invalid field-name tokens, invalid UTF-8, and control bytes in field -values before policy evaluation, middleware projection, or forwarding. Request -framing accepts either `Content-Length` or exactly one `chunked` transfer coding; -unsupported transfer codings and every CL/TE combination fail closed. -Each middleware stage can also return ordered header write and remove -operations. Rust validates and applies a stage atomically to the logical header -state before the next stage runs, then replays the validated operations against -the raw request before credential injection. Credential, routing, framing, and -hop-by-hop headers remain supervisor-owned and cannot be mutated. -Names dynamically nominated by the original request's `Connection` header are -carried separately after header filtering so write and remove mutations cannot -reintroduce them. The proxy also removes nominated fields before forwarding; -only a validated WebSocket handshake preserves a canonical `Connection: -Upgrade` and `Upgrade: websocket` pair. When middleware buffers a request body, -the proxy consumes `Expect: 100-continue` locally, acknowledges it only if more -client bytes are required, and never forwards it with the normalized body. +chain after L7 policy evaluation and before credential injection. Destination +host selectors choose the chain independently of the network rule that admitted +the request. Policy-local map keys identify configs, while built-in names or +operator-owned registration names identify implementations. + +Built-ins run in-process; operator services use the same bounded gRPC contract. +`openshell-policy` validates policy-owned structure, and the active middleware +registry validates implementation-owned config. The generic registry and chain +runner live in `openshell-supervisor-middleware`; first-party implementations +live in `openshell-supervisor-middleware-builtins`. + +The supervisor installs policy and middleware registry changes as one runtime +generation and preserves the last-known-good generation if preparation fails. +Policy-only updates reuse the connected registry, so an external middleware +outage cannot block unrelated policy changes. + +Middleware cannot observe injected credentials or mutate supervisor-owned +credential, routing, or framing headers. Body transformations are re-evaluated +against body-aware L7 policy before later stages or the upstream can observe +them. Requests, results, chain length, execution time, and diagnostics are +bounded; external free-form diagnostic text is not exposed in responses or +security logs. See [Supervisor Middleware](../docs/extensibility/supervisor-middleware.mdx) +for configuration and protocol details. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: From fdc5c34e765381c9d35eeb7597de49b15b203e03 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 16 Jul 2026 13:10:14 -0700 Subject: [PATCH 59/59] fix(middleware): separate runtime failure responses Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/grpc/policy.rs | 57 ++++++++++++ .../src/l7/middleware.rs | 89 +++++++++++++------ .../src/l7/relay.rs | 69 +++++++------- .../src/l7/rest.rs | 47 ++++++++++ .../openshell-supervisor-network/src/proxy.rs | 63 +++++++------ docs/extensibility/supervisor-middleware.mdx | 2 + docs/observability/logging.mdx | 4 +- 7 files changed, 244 insertions(+), 87 deletions(-) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index fd74215a38..68822d1505 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -10209,6 +10209,63 @@ mod tests { ); } + #[test] + fn policy_hash_is_stable_across_middleware_config_field_insertion_order() { + use prost_types::{Struct, Value, value::Kind}; + use std::collections::BTreeMap; + + fn string_value(value: &str) -> Value { + Value { + kind: Some(Kind::StringValue(value.into())), + } + } + + fn middleware_config(reverse: bool) -> Struct { + let mut nested = BTreeMap::new(); + let mut fields = BTreeMap::new(); + if reverse { + nested.insert("second".into(), string_value("two")); + nested.insert("first".into(), string_value("one")); + fields.insert( + "nested".into(), + Value { + kind: Some(Kind::StructValue(Struct { fields: nested })), + }, + ); + fields.insert("mode".into(), string_value("redact")); + } else { + nested.insert("first".into(), string_value("one")); + nested.insert("second".into(), string_value("two")); + fields.insert("mode".into(), string_value("redact")); + fields.insert( + "nested".into(), + Value { + kind: Some(Kind::StructValue(Struct { fields: nested })), + }, + ); + } + Struct { fields } + } + + let policy = |reverse| ProtoSandboxPolicy { + network_middlewares: HashMap::from([( + "regex-redactor".into(), + openshell_core::proto::NetworkMiddlewareConfig { + middleware: "openshell/regex".into(), + config: Some(middleware_config(reverse)), + ..Default::default() + }, + )]), + ..Default::default() + }; + + assert_eq!( + deterministic_policy_hash(&policy(false)), + deterministic_policy_hash(&policy(true)), + "equivalent middleware configs must hash identically regardless of field insertion order" + ); + } + #[test] fn config_revision_changes_when_policy_source_changes() { let policy = ProtoSandboxPolicy::default(); diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 47a2cde010..263d886f28 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -16,7 +16,6 @@ use tokio::io::{AsyncRead, AsyncWrite}; pub enum MiddlewareApplyResult { Allowed(crate::l7::provider::L7Request), Denied { - reason: String, denial: Option, }, } @@ -160,7 +159,6 @@ pub async fn apply_middleware_chain_for_scheme, redacted_target: &str, ) -> Result<()> { @@ -235,16 +231,14 @@ pub async fn send_middleware_rejection_response request, - MiddlewareApplyResult::Denied { reason, denial } => { + MiddlewareApplyResult::Denied { denial, .. } => { let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -496,7 +496,6 @@ where &denied_request, client, ctx, - &reason, denial.as_ref(), &redacted_target, ) @@ -977,7 +976,7 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied { reason, denial } => { + MiddlewareApplyResult::Denied { denial, .. } => { let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -989,7 +988,6 @@ where &denied_request, client, ctx, - &reason, denial.as_ref(), &redacted_target, ) @@ -1247,7 +1245,7 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied { reason, denial } => { + MiddlewareApplyResult::Denied { denial, .. } => { let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1259,7 +1257,6 @@ where &denied_request, client, ctx, - &reason, denial.as_ref(), &redacted_target, ) @@ -1476,7 +1473,7 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied { reason, denial } => { + MiddlewareApplyResult::Denied { denial, .. } => { let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1488,7 +1485,6 @@ where &denied_request, client, ctx, - &reason, denial.as_ref(), &redacted_target, ) @@ -2113,7 +2109,7 @@ where .await? { MiddlewareApplyResult::Allowed(request) => request, - MiddlewareApplyResult::Denied { reason, denial } => { + MiddlewareApplyResult::Denied { denial, .. } => { let denied_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -2125,7 +2121,6 @@ where &denied_request, client, ctx, - &reason, denial.as_ref(), &redacted_target, ) @@ -2222,6 +2217,22 @@ mod tests { )); } + fn assert_middleware_failure_response(response: &str, policy_name: &str) { + assert!(response.contains("403 Forbidden"), "{response}"); + let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); + let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); + assert_eq!(body["error"], "middleware_failed"); + assert_eq!( + body["detail"], + "Request could not be processed by configured middleware" + ); + assert_eq!(body["policy"], policy_name); + assert!(body.get("rule").is_none()); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + assert!(body.get("agent_guidance").is_none()); + } + fn rest_token_grant_relay_context( resolver_response: std::result::Result<&str, &str>, ) -> ( @@ -2808,7 +2819,18 @@ network_policies: .unwrap(); let response = String::from_utf8_lossy(&response[..n]); assert!(response.contains("403 Forbidden")); - assert!(response.contains("middleware_failed")); + let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); + let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); + assert_eq!(body["error"], "middleware_failed"); + assert_eq!( + body["detail"], + "Request could not be processed by configured middleware" + ); + assert_eq!(body["policy"], "rest_api"); + assert!(body.get("rule").is_none()); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + assert!(body.get("agent_guidance").is_none()); let mut upstream_request = [0u8; 32]; let result = tokio::time::timeout( @@ -3074,8 +3096,7 @@ network_policies: .expect("denial should reach client") .unwrap(); let response = String::from_utf8_lossy(&response[..n]); - assert!(response.contains("403 Forbidden")); - assert!(response.contains("request_body_over_capacity")); + assert_middleware_failure_response(&response, "rest_api"); let mut upstream_request = [0u8; 32]; let result = tokio::time::timeout( @@ -3403,11 +3424,7 @@ network_policies: br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, ) .await; - assert!(response.contains("403 Forbidden"), "{response}"); - assert!( - response.contains("middleware transformation denied by policy"), - "{response}" - ); + assert_middleware_failure_response(&response, "jsonrpc_api"); assert!(upstream_seen.is_none(), "upstream must not see the request"); } @@ -3431,11 +3448,7 @@ network_policies: // An unparseable replacement mirrors force_deny for original parse // errors: denied even on an audit endpoint. let (response, upstream_seen) = run_jsonrpc_transform_case("audit", b"not json").await; - assert!(response.contains("403 Forbidden"), "{response}"); - assert!( - response.contains("middleware transformation rejected"), - "{response}" - ); + assert_middleware_failure_response(&response, "jsonrpc_api"); assert!(upstream_seen.is_none(), "upstream must not see the request"); } @@ -3524,11 +3537,7 @@ network_policies: .expect("denial should reach client") .unwrap(); let response = String::from_utf8_lossy(&response[..n]); - assert!(response.contains("403 Forbidden"), "{response}"); - assert!( - response.contains("middleware transformation denied by policy"), - "{response}" - ); + assert_middleware_failure_response(&response, "graphql_api"); let mut upstream_request = [0u8; 32]; let result = tokio::time::timeout( @@ -3859,8 +3868,8 @@ network_policies: "redactor must replace the body: {raw}" ); } - MiddlewareApplyResult::Denied { reason, .. } => { - panic!("body within the largest stage limit must not fail the chain: {reason}") + MiddlewareApplyResult::Denied { .. } => { + panic!("body within the largest stage limit must not fail the chain") } } } diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index a80d31ae7e..4d1fe92cd4 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -2180,6 +2180,22 @@ pub(crate) async fn send_middleware_deny_response( send_forbidden_json(policy_name, body, client).await } +/// Send a platform-owned response for fail-closed middleware failures. +/// +/// Unlike an L7 policy denial, a middleware failure does not indicate a +/// missing policy rule. The response therefore omits policy-advisor guidance +/// and does not include the middleware runtime's diagnostic text. +pub(crate) async fn send_middleware_failure_response( + req: &L7Request, + policy_name: &str, + client: &mut C, + redacted_target: Option<&str>, + context: Option>, +) -> Result<()> { + let body = middleware_failure_response_body(req, policy_name, redacted_target, context); + send_forbidden_json(policy_name, body, client).await +} + async fn send_forbidden_json( policy_name: &str, body: serde_json::Value, @@ -2245,6 +2261,37 @@ fn middleware_deny_response_body( serde_json::Value::Object(body) } +fn middleware_failure_response_body( + req: &L7Request, + policy_name: &str, + redacted_target: Option<&str>, + context: Option>, +) -> serde_json::Value { + let target = redacted_target.unwrap_or(&req.target); + let context = context.unwrap_or_default(); + let mut body = serde_json::Map::new(); + body.insert("error".to_string(), serde_json::json!("middleware_failed")); + body.insert( + "detail".to_string(), + serde_json::json!("Request could not be processed by configured middleware"), + ); + body.insert("policy".to_string(), serde_json::json!(policy_name)); + body.insert("layer".to_string(), serde_json::json!("l7")); + body.insert("method".to_string(), serde_json::json!(req.action)); + body.insert("path".to_string(), serde_json::json!(target)); + if let Some(host) = non_empty(context.host) { + body.insert("host".to_string(), serde_json::json!(host)); + } + if let Some(port) = context.port { + body.insert("port".to_string(), serde_json::json!(port)); + } + if let Some(binary) = non_empty(context.binary) { + body.insert("binary".to_string(), serde_json::json!(binary)); + } + + serde_json::Value::Object(body) +} + fn deny_response_body( req: &L7Request, policy_name: &str, diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index fbb5708bde..ab2313b890 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -4664,19 +4664,10 @@ async fn handle_forward_proxy( }; forward_request_bytes = match pipeline.apply(request, client, chain).await? { crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header, - crate::l7::middleware::MiddlewareApplyResult::Denied { reason, denial } => { + crate::l7::middleware::MiddlewareApplyResult::Denied { denial, .. } => { emit_activity_simple(activity_tx, true, "middleware"); let response = denial.as_ref().map_or_else( - || { - build_json_error_response( - 403, - "Forbidden", - "middleware_denied", - &format!( - "{method} {host_lc}:{port}{path} denied by middleware: {reason}" - ), - ) - }, + || build_middleware_failure_response(&l7_ctx.policy_name), |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), ); respond(client, &response).await?; @@ -4919,6 +4910,26 @@ fn build_middleware_deny_response( .into_bytes() } +fn build_middleware_failure_response(policy_name: &str) -> Vec { + let body = serde_json::json!({ + "error": "middleware_failed", + "detail": "Request could not be processed by configured middleware", + "policy": policy_name, + }); + let body_str = body.to_string(); + format!( + "HTTP/1.1 403 Forbidden\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + body_str.len(), + body_str, + ) + .into_bytes() +} + /// Detail shared by the fail-closed 503 body, the OCSF denial event, and the /// denial notification when a terminating CONNECT route has no TLS termination /// state available. @@ -5099,19 +5110,21 @@ network_policies: {} } #[test] - fn middleware_denial_response_contains_only_platform_mutation_reason() { - const RAW_SECRET: &str = "sk-secret-request-value"; - let reason = "middleware_failed: header_mutation_invalid_name"; - let response = build_json_error_response( - 403, - "Forbidden", - "middleware_denied", - &format!("POST api.example.test:443/v1 denied by middleware: {reason}"), - ); + fn middleware_failure_response_uses_platform_text_without_policy_guidance() { + let response = build_middleware_failure_response("api-policy"); let response = String::from_utf8(response).expect("UTF-8 error response"); + let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); + let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); - assert!(response.contains(reason)); - assert!(!response.contains(RAW_SECRET)); + assert_eq!(body["error"], "middleware_failed"); + assert_eq!( + body["detail"], + "Request could not be processed by configured middleware" + ); + assert_eq!(body["policy"], "api-policy"); + assert!(body.get("rule_missing").is_none()); + assert!(body.get("next_steps").is_none()); + assert!(body.get("agent_guidance").is_none()); } #[test] @@ -5525,11 +5538,7 @@ network_policies: .expect("forward middleware pipeline"); match outcome { - crate::l7::middleware::MiddlewareApplyResult::Denied { reason, denial } => { - assert!( - reason.contains("middleware transformation denied by policy"), - "{reason}" - ); + crate::l7::middleware::MiddlewareApplyResult::Denied { denial } => { assert!(denial.is_none()); } crate::l7::middleware::MiddlewareApplyResult::Allowed(_) => { diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx index 3ecd416065..f3bdcac9bb 100644 --- a/docs/extensibility/supervisor-middleware.mdx +++ b/docs/extensibility/supervisor-middleware.mdx @@ -113,6 +113,8 @@ An explicit deny decision always stops the chain and denies the request, regardl } ``` +A failed `fail_closed` stage uses `error: middleware_failed` and a platform-owned `detail`. It also omits `rule_missing`, `next_steps`, and `agent_guidance`: the failure did not result from a missing network or L7 policy rule, and changing policy cannot repair it. Runtime diagnostic text is available only through sanitized operator telemetry. + Middleware decisions are enforced regardless of the endpoint's `enforcement` mode. `enforcement: audit` applies to an endpoint's network and L7 policy rules and does not bypass middleware: a middleware deny, or a failed `fail_closed` stage, blocks the request even on an audit endpoint. A middleware service that needs to observe traffic without blocking should return an allow decision with findings, which OpenShell emits as detection findings. ## Set Body Limits diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 8c30c5cf04..7a13ffea0b 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -205,9 +205,9 @@ An upstream that the proxy cannot reach returns `502 Bad Gateway`: } ``` -The `error` field is a short machine-readable code (`policy_denied`, `middleware_denied`, `ssrf_denied`, `upstream_unreachable`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. +The `error` field is a short machine-readable code (`policy_denied`, `middleware_denied`, `middleware_failed`, `ssrf_denied`, `upstream_unreachable`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. -For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When policy advisor is enabled, it also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware`, optionally includes a validated `reason_code`, and omits `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. +For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When policy advisor is enabled, it also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware` and can include a validated `reason_code`. A fail-closed runtime failure uses `middleware_failed` with platform-owned text. Both middleware responses omit `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. ## Filesystem Sandbox Logs