From 019c62e1bafc35b31137ddaa44cd3c3dbeaad04e Mon Sep 17 00:00:00 2001 From: Blake Covarrubias Date: Tue, 19 Sep 2023 14:32:28 -0700 Subject: [PATCH] xds: Use downstream protocol when connecting to local app (#18573) Configure Envoy to use the same HTTP protocol version used by the downstream caller when forwarding requests to a local application that is configured with the protocol set to either `http2` or `grpc`. This allows upstream applications that support both HTTP/1.1 and HTTP/2 on a single port to receive requests using either protocol. This is beneficial when the application primarily communicates using HTTP/2, but also needs to support HTTP/1.1, such as to respond to Kubernetes HTTP readiness/liveness probes. Co-authored-by: Derek Menteer --- .changelog/18573.txt | 3 ++ agent/xds/clusters.go | 33 +++++++++++++++++-- ...paths-grpc-new-cluster-http1.latest.golden | 5 +-- agent/xdsv2/cluster_resources.go | 32 +++++++++++++++++- 4 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 .changelog/18573.txt diff --git a/.changelog/18573.txt b/.changelog/18573.txt new file mode 100644 index 000000000000..ce03f1c55baf --- /dev/null +++ b/.changelog/18573.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +xds: Use downstream protocol when connecting to local app +``` diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 908aa746bc9c..3f0ba0d7f31b 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -1096,8 +1096,14 @@ func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, nam protocol = cfg.Protocol } if protocol == "http2" || protocol == "grpc" { - if err := s.setHttp2ProtocolOptions(c); err != nil { - return c, err + if name == xdscommon.LocalAppClusterName { + if err := s.setLocalAppHttpProtocolOptions(c); err != nil { + return c, err + } + } else { + if err := s.setHttp2ProtocolOptions(c); err != nil { + return c, err + } } } if cfg.MaxInboundConnections > 0 { @@ -2016,6 +2022,29 @@ func (s *ResourceGenerator) setHttp2ProtocolOptions(c *envoy_cluster_v3.Cluster) return nil } +// Allows forwarding either HTTP/1.1 or HTTP/2 traffic to the local application. +// The protocol used depends on the protocol received from the downstream service +// on the public listener. +func (s *ResourceGenerator) setLocalAppHttpProtocolOptions(c *envoy_cluster_v3.Cluster) error { + cfg := &envoy_upstreams_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamProtocolConfig{ + UseDownstreamProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamHttpConfig{ + HttpProtocolOptions: &envoy_core_v3.Http1ProtocolOptions{}, + Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{}, + }, + }, + } + any, err := anypb.New(cfg) + if err != nil { + return err + } + c.TypedExtensionProtocolOptions = map[string]*anypb.Any{ + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any, + } + + return nil +} + // generatePeeredClusterName returns an SNI-like cluster name which mimics PeeredServiceSNI // but excludes partition information which could be ambiguous (local vs remote partition). func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTrustBundle) string { diff --git a/agent/xds/testdata/clusters/expose-paths-grpc-new-cluster-http1.latest.golden b/agent/xds/testdata/clusters/expose-paths-grpc-new-cluster-http1.latest.golden index 3eb573197758..9bc45fc80295 100644 --- a/agent/xds/testdata/clusters/expose-paths-grpc-new-cluster-http1.latest.golden +++ b/agent/xds/testdata/clusters/expose-paths-grpc-new-cluster-http1.latest.golden @@ -53,8 +53,9 @@ "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", - "explicitHttpConfig": { - "http2ProtocolOptions": {} + "useDownstreamProtocolConfig": { + "http2ProtocolOptions": {}, + "httpProtocolOptions": {} } } } diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index b0a79fd0aa13..dc19027338e5 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -164,7 +164,13 @@ func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, s if ok { cluster.LoadAssignment = makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints) } - err := addHttpProtocolOptions(protocol, cluster) + + var err error + if name == xdscommon.LocalAppClusterName { + err = addLocalAppHttpProtocolOptions(protocol, cluster) + } else { + err = addHttpProtocolOptions(protocol, cluster) + } if err != nil { return nil, err } @@ -243,6 +249,30 @@ func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string return clusters, nil } +func addLocalAppHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error { + if !(protocol == "http2" || protocol == "grpc") { + // do not error. returning nil means it won't get set. + return nil + } + cfg := &envoy_upstreams_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamProtocolConfig{ + UseDownstreamProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamHttpConfig{ + HttpProtocolOptions: &envoy_core_v3.Http1ProtocolOptions{}, + Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{}, + }, + }, + } + any, err := anypb.New(cfg) + if err != nil { + return err + } + c.TypedExtensionProtocolOptions = map[string]*anypb.Any{ + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any, + } + + return nil +} + func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error { if !(protocol == "http2" || protocol == "grpc") { // do not error. returning nil means it won't get set.