UPSTREAM: <carry>: Prevent rapid reset http2 DOS on API server

enj · ncdc · commit 059e9a4690ef · 2023-10-13T09:09:00.000-04:00
This change fully addresses CVE-2023-44487 and CVE-2023-39325 for the API server when the client is unauthenticated. The changes to util/runtime are required because otherwise a large number of requests can get blocked on the time.Sleep calls. For unauthenticated clients (either via 401 or the anonymous user), we simply no longer allow such clients to hold open http2 connections. They can use http2, but with the performance of http1 (with keep-alive disabled). Since this change has the potential to cause issues, the UnauthenticatedHTTP2DOSMitigation feature gate can be disabled to remove this protection (it is enabled by default). For example, when the API server is fronted by an L7 load balancer that is set up to mitigate http2 attacks, unauthenticated clients could force disable connection reuse between the load balancer and the API server (many incoming connections could share the same backend connection). An API server that is on a private network may opt to disable this protection to prevent performance regressions for unauthenticated clients. For all other clients, we rely on the golang.org/x/net fix in golang/net@b225e7c That change is not sufficient to adequately protect against a motivated client - future changes to Kube and/or golang.org/x/net will be explored to address this gap. The Kube API server now uses a max stream of 100 instead of 250 (this matches the Go http2 client default). This lowers the abuse limit from 1000 to 400. Signed-off-by: Monis Khan <mok@microsoft.com> (cherry picked from commit d40f08f)
diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go
@@ -126,14 +126,17 @@ type rudimentaryErrorBackoff struct {
 // OnError will block if it is called more often than the embedded period time.
 // This will prevent overly tight hot error loops.
 func (r *rudimentaryErrorBackoff) OnError(error) {
+	now := time.Now() // start the timer before acquiring the lock
 	r.lastErrorTimeLock.Lock()
-	defer r.lastErrorTimeLock.Unlock()
-	d := time.Since(r.lastErrorTime)
-	if d < r.minPeriod {
-		// If the time moves backwards for any reason, do nothing
-		time.Sleep(r.minPeriod - d)
-	}
+	d := now.Sub(r.lastErrorTime)
 	r.lastErrorTime = time.Now()
+	r.lastErrorTimeLock.Unlock()
+
+	// Do not sleep with the lock held because that causes all callers of HandleError to block.
+	// We only want the current goroutine to block.
+	// A negative or zero duration causes time.Sleep to return immediately.
+	// If the time moves backwards for any reason, do nothing.
+	time.Sleep(r.minPeriod - d)
 }
 
 // GetCaller returns the caller of the function that calls it.
diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime_test.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime_test.go
@@ -24,7 +24,9 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
+	"time"
 )
 
 func TestHandleCrash(t *testing.T) {
@@ -156,3 +158,27 @@ func captureStderr(f func()) (string, error) {
 
 	return <-resultCh, nil
 }
+
+func Test_rudimentaryErrorBackoff_OnError_ParallelSleep(t *testing.T) {
+	r := &rudimentaryErrorBackoff{
+		minPeriod: time.Second,
+	}
+
+	start := make(chan struct{})
+	var wg sync.WaitGroup
+	for i := 0; i < 30; i++ {
+		wg.Add(1)
+		go func() {
+			<-start
+			r.OnError(nil) // input error is ignored
+			wg.Done()
+		}()
+	}
+	st := time.Now()
+	close(start)
+	wg.Wait()
+
+	if since := time.Since(st); since > 5*time.Second {
+		t.Errorf("OnError slept for too long: %s", since)
+	}
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go
@@ -29,8 +29,11 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 )
 
@@ -101,13 +104,36 @@ func withAuthentication(handler http.Handler, auth authenticator.Request, failed
 			)
 		}
 
+		// http2 is an expensive protocol that is prone to abuse,
+		// see CVE-2023-44487 and CVE-2023-39325 for an example.
+		// Do not allow unauthenticated clients to keep these
+		// connections open (i.e. basically degrade them to the
+		// performance of http1 with keep-alive disabled).
+		if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.UnauthenticatedHTTP2DOSMitigation) && req.ProtoMajor == 2 && isAnonymousUser(resp.User) {
+			// limit this connection to just this request,
+			// and then send a GOAWAY and tear down the TCP connection
+			// https://github.com/golang/net/commit/97aa3a539ec716117a9d15a4659a911f50d13c3c
+			w.Header().Set("Connection", "close")
+		}
+
 		req = req.WithContext(genericapirequest.WithUser(req.Context(), resp.User))
 		handler.ServeHTTP(w, req)
 	})
 }
 
 func Unauthorized(s runtime.NegotiatedSerializer) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		// http2 is an expensive protocol that is prone to abuse,
+		// see CVE-2023-44487 and CVE-2023-39325 for an example.
+		// Do not allow unauthenticated clients to keep these
+		// connections open (i.e. basically degrade them to the
+		// performance of http1 with keep-alive disabled).
+		if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.UnauthenticatedHTTP2DOSMitigation) && req.ProtoMajor == 2 {
+			// limit this connection to just this request,
+			// and then send a GOAWAY and tear down the TCP connection
+			// https://github.com/golang/net/commit/97aa3a539ec716117a9d15a4659a911f50d13c3c
+			w.Header().Set("Connection", "close")
+		}
 		ctx := req.Context()
 		requestInfo, found := genericapirequest.RequestInfoFrom(ctx)
 		if !found {
@@ -127,3 +153,15 @@ func audiencesAreAcceptable(apiAuds, responseAudiences authenticator.Audiences)
 
 	return len(apiAuds.Intersect(responseAudiences)) > 0
 }
+
+func isAnonymousUser(u user.Info) bool {
+	if u.GetName() == user.Anonymous {
+		return true
+	}
+	for _, group := range u.GetGroups() {
+		if group == user.AllUnauthenticated {
+			return true
+		}
+	}
+	return false
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication_test.go
@@ -18,20 +18,31 @@ package filters
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"errors"
+	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/http2"
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/anonymous"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/authentication/user"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/kubernetes/scheme"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 )
 
 func TestAuthenticateRequestWithAud(t *testing.T) {
@@ -465,3 +476,191 @@ func TestAuthenticateRequestClearHeaders(t *testing.T) {
 		})
 	}
 }
+
+func TestUnauthenticatedHTTP2ClientConnectionClose(t *testing.T) {
+	s := httptest.NewUnstartedServer(WithAuthentication(
+		http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) }),
+		authenticator.RequestFunc(func(r *http.Request) (*authenticator.Response, bool, error) {
+			switch r.Header.Get("Authorization") {
+			case "known":
+				return &authenticator.Response{User: &user.DefaultInfo{Name: "panda"}}, true, nil
+			case "error":
+				return nil, false, errors.New("authn err")
+			case "anonymous":
+				return anonymous.NewAuthenticator().AuthenticateRequest(r)
+			case "anonymous_group":
+				return &authenticator.Response{User: &user.DefaultInfo{Groups: []string{user.AllUnauthenticated}}}, true, nil
+			default:
+				return nil, false, nil
+			}
+		}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			r = r.WithContext(genericapirequest.WithRequestInfo(r.Context(), &genericapirequest.RequestInfo{}))
+			Unauthorized(scheme.Codecs).ServeHTTP(w, r)
+		}),
+		nil,
+		nil,
+	))
+
+	http2Options := &http2.Server{}
+
+	if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
+		t.Fatal(err)
+	}
+
+	s.TLS = s.Config.TLSConfig
+
+	s.StartTLS()
+	t.Cleanup(s.Close)
+
+	const reqs = 4
+
+	cases := []struct {
+		name                   string
+		authorizationHeader    string
+		skipHTTP2DOSMitigation bool
+		expectConnections      uint64
+	}{
+		{
+			name:                   "known",
+			authorizationHeader:    "known",
+			skipHTTP2DOSMitigation: false,
+			expectConnections:      1,
+		},
+		{
+			name:                   "error",
+			authorizationHeader:    "error",
+			skipHTTP2DOSMitigation: false,
+			expectConnections:      reqs,
+		},
+		{
+			name:                   "anonymous",
+			authorizationHeader:    "anonymous",
+			skipHTTP2DOSMitigation: false,
+			expectConnections:      reqs,
+		},
+		{
+			name:                   "anonymous_group",
+			authorizationHeader:    "anonymous_group",
+			skipHTTP2DOSMitigation: false,
+			expectConnections:      reqs,
+		},
+		{
+			name:                   "other",
+			authorizationHeader:    "other",
+			skipHTTP2DOSMitigation: false,
+			expectConnections:      reqs,
+		},
+
+		{
+			name:                   "known skip=true",
+			authorizationHeader:    "known",
+			skipHTTP2DOSMitigation: true,
+			expectConnections:      1,
+		},
+		{
+			name:                   "error skip=true",
+			authorizationHeader:    "error",
+			skipHTTP2DOSMitigation: true,
+			expectConnections:      1,
+		},
+		{
+			name:                   "anonymous skip=true",
+			authorizationHeader:    "anonymous",
+			skipHTTP2DOSMitigation: true,
+			expectConnections:      1,
+		},
+		{
+			name:                   "anonymous_group skip=true",
+			authorizationHeader:    "anonymous_group",
+			skipHTTP2DOSMitigation: true,
+			expectConnections:      1,
+		},
+		{
+			name:                   "other skip=true",
+			authorizationHeader:    "other",
+			skipHTTP2DOSMitigation: true,
+			expectConnections:      1,
+		},
+	}
+
+	rootCAs := x509.NewCertPool()
+	rootCAs.AddCert(s.Certificate())
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := func(t *testing.T, nextProto string, expectConnections uint64) {
+				defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.UnauthenticatedHTTP2DOSMitigation, !tc.skipHTTP2DOSMitigation)()
+
+				var localAddrs atomic.Uint64 // indicates how many TCP connection set up
+
+				tlsConfig := &tls.Config{
+					RootCAs:    rootCAs,
+					NextProtos: []string{nextProto},
+				}
+
+				dailer := tls.Dialer{
+					Config: tlsConfig,
+				}
+
+				tr := &http.Transport{
+					TLSHandshakeTimeout: 10 * time.Second,
+					TLSClientConfig:     tlsConfig,
+					DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+						conn, err := dailer.DialContext(ctx, network, addr)
+						if err != nil {
+							return nil, err
+						}
+
+						localAddrs.Add(1)
+
+						return conn, nil
+					},
+				}
+
+				tr.MaxIdleConnsPerHost = 1 // allow http1 to have keep alive connections open
+				if nextProto == http2.NextProtoTLS {
+					// Disable connection pooling to avoid additional connections
+					// that cause the test to flake
+					tr.MaxIdleConnsPerHost = -1
+					if err := http2.ConfigureTransport(tr); err != nil {
+						t.Fatal(err)
+					}
+				}
+
+				client := &http.Client{
+					Transport: tr,
+				}
+
+				for i := 0; i < reqs; i++ {
+					req, err := http.NewRequest(http.MethodGet, s.URL, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if len(tc.authorizationHeader) > 0 {
+						req.Header.Set("Authorization", tc.authorizationHeader)
+					}
+
+					resp, err := client.Do(req)
+					if err != nil {
+						t.Fatal(err)
+					}
+					_, _ = io.Copy(io.Discard, resp.Body)
+					_ = resp.Body.Close()
+				}
+
+				if expectConnections != localAddrs.Load() {
+					t.Fatalf("expect TCP connection: %d, actual: %d", expectConnections, localAddrs.Load())
+				}
+			}
+
+			t.Run(http2.NextProtoTLS, func(t *testing.T) {
+				f(t, http2.NextProtoTLS, tc.expectConnections)
+			})
+
+			t.Run("http/1.1", func(t *testing.T) {
+				f(t, "http/1.1", 1)
+			})
+		})
+	}
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go
@@ -95,6 +95,24 @@ const (
 	// Enable the storage version API.
 	StorageVersionAPI featuregate.Feature = "StorageVersionAPI"
 
+	// owner: @enj
+	// beta: v1.29
+	//
+	// Enables http2 DOS mitigations for unauthenticated clients.
+	//
+	// Some known reasons to disable these mitigations:
+	//
+	// An API server that is fronted by an L7 load balancer that is set up
+	// to mitigate http2 attacks may opt to disable this protection to prevent
+	// unauthenticated clients from disabling connection reuse between the load
+	// balancer and the API server (many incoming connections could share the
+	// same backend connection).
+	//
+	// An API server that is on a private network may opt to disable this
+	// protection to prevent performance regressions for unauthenticated
+	// clients.
+	UnauthenticatedHTTP2DOSMitigation featuregate.Feature = "UnauthenticatedHTTP2DOSMitigation"
+
 	// owner: @wojtek-t
 	// alpha: v1.15
 	// beta: v1.16
@@ -196,6 +214,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	ServerSideApply:                     {Default: true, PreRelease: featuregate.GA},
 	StorageVersionHash:                  {Default: true, PreRelease: featuregate.Beta},
 	StorageVersionAPI:                   {Default: false, PreRelease: featuregate.Alpha},
+	UnauthenticatedHTTP2DOSMitigation:   {Default: true, PreRelease: featuregate.Beta},
 	WatchBookmark:                       {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 	APIPriorityAndFairness:              {Default: true, PreRelease: featuregate.Beta},
 	RemoveSelfLink:                      {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go
@@ -189,7 +189,10 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
 	if s.HTTP2MaxStreamsPerConnection > 0 {
 		http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection)
 	} else {
-		http2Options.MaxConcurrentStreams = 250
+		// match http2.initialMaxConcurrentStreams used by clients
+		// this makes it so that a malicious client can only open 400 streams before we forcibly close the connection
+		// https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd
+		http2Options.MaxConcurrentStreams = 100
 	}
 
 	// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
