diff --git a/docs/content/configuration/policy-resource.md b/docs/content/configuration/policy-resource.md index b5f6cc403c..32e7db9b4d 100644 --- a/docs/content/configuration/policy-resource.md +++ b/docs/content/configuration/policy-resource.md @@ -235,11 +235,14 @@ jwt: |Field | Description | Type | Required | | ---| ---| ---| --- | |``jwksURI`` | The remote URI where the request will be sent to retrieve JSON Web Key set| ``string`` | Yes | -|``keyCache`` | Enables the caching of keys that are obtained from the ``jwksURI`` and sets a valid time for expiration | ``string`` | Yes | +|``keyCache`` | Enables in-memory caching of JWKS (JSON Web Key Sets) that are obtained from the ``jwksURI`` and sets a valid time for expiration. | ``string`` | Yes | |``realm`` | The realm of the JWT. | ``string`` | Yes | |``token`` | The token specifies a variable that contains the JSON Web Token. By default the JWT is passed in the ``Authorization`` header as a Bearer Token. JWT may be also passed as a cookie or a part of a query string, for example: ``$cookie_auth_token``. Accepted variables are ``$http_``, ``$arg_``, ``$cookie_``. | ``string`` | No | {{% /table %}} +> Note: Content caching is enabled by default for each JWT policy with a default time of 12 hours. +> This is done to ensure to improve resiliency by allowing the JWKS (JSON Web Key Set) to be retrieved from the cache even when it has expired. + #### JWT Merging Behavior This behavior is similar to using a local Kubernetes secret where a VirtualServer/VirtualServerRoute can reference multiple JWT policies. However, only one can be applied: every subsequent reference will be ignored. For example, here we reference two policies: diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index 8da6938a79..34a6c09c34 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -69,6 +69,8 @@ type Server struct { LimitReqOptions LimitReqOptions LimitReqs []LimitReq JWTAuth *JWTAuth + JWTAuthList map[string]*JWTAuth + JWKSAuthEnabled bool BasicAuth *BasicAuth IngressMTLS *IngressMTLS EgressMTLS *EgressMTLS @@ -356,6 +358,7 @@ func (rl LimitReqOptions) String() string { // JWTAuth holds JWT authentication configuration. type JWTAuth struct { + Key string Secret string Realm string Token string diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 72648d11f9..1b13ee419c 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -57,14 +57,9 @@ match {{ $m.Name }} { {{ end }} {{ $s := .Server }} -{{ with $s.JWTAuth }} -{{ if .KeyCache }}proxy_cache_path /var/cache/nginx/jwks_uri levels=1 keys_zone=jwks_uri:1m max_size=10m; {{ end }} -{{ end }} -{{ range $l := $s.Locations }} -{{ with $l.JWTAuth }} -{{ if .KeyCache }}proxy_cache_path /var/cache/nginx{{ $l.Path }}_jwks_uri levels=1 keys_zone={{ $l.Path }}_jwks_uri:1m max_size=10m; {{ end }} -{{ end }} +{{ with $s.JWKSAuthEnabled }} +proxy_cache_path /var/cache/nginx/jwks_uri_{{$s.VSName}} levels=1 keys_zone=jwks_uri_{{$s.VSName}}:1m max_size=10m; {{ end }} server { @@ -190,24 +185,27 @@ server { {{ if .Secret}}auth_jwt_key_file {{ .Secret }};{{ end }} {{ if .JwksURI.JwksHost }} {{ if .KeyCache }}auth_jwt_key_cache {{ .KeyCache }};{{ end }} - auth_jwt_key_request /_jwks_uri_server; + auth_jwt_key_request /_jwks_uri_server_{{ .Key }}; + + {{ end }} + {{ end }} - location = /_jwks_uri_server { + {{ range $index, $element := $s.JWTAuthList }} + location = /_jwks_uri_server_{{ .Key }} { internal; proxy_method GET; proxy_set_header Content-Length ""; {{ if .KeyCache }} - proxy_cache jwks_uri; + proxy_cache jwks_uri_{{ $s.VSName }}; proxy_cache_valid 200 12h; {{ end }} - {{ with $s.JWTAuth.JwksURI }} + {{ with .JwksURI }} proxy_set_header Host {{ .JwksHost }}; set $idp_backend {{ .JwksHost }}; proxy_pass {{ .JwksScheme}}://$idp_backend{{ if .JwksPort }}:{{ .JwksPort }}{{ end }}{{ .JwksPath }}; {{ end }} } - {{ end }} {{ end }} {{ with $s.BasicAuth }} @@ -391,22 +389,7 @@ server { {{ if .Secret}}auth_jwt_key_file {{ .Secret }};{{ end }} {{ if .JwksURI.JwksHost }} {{ if .KeyCache }}auth_jwt_key_cache {{ .KeyCache }};{{ end }} - auth_jwt_key_request {{ $l.Path }}_jwks_uri; - - location = {{ $l.Path }}_jwks_uri { - internal; - proxy_method GET; - proxy_set_header Content-Length ""; - {{ if .KeyCache }} - proxy_cache {{ $l.Path }}_jwks_uri; - proxy_cache_valid 200 12h; - {{ end }} - {{ with $l.JWTAuth.JwksURI }} - proxy_set_header Host {{ .JwksHost }}; - set $idp_backend {{ .JwksHost }}; - proxy_pass {{ .JwksScheme}}://$idp_backend{{ if .JwksPort }}:{{ .JwksPort }}{{ end }}{{ .JwksPath }}; - {{ end }} - } + auth_jwt_key_request /_jwks_uri_server_{{ .Key }}; {{ end }} {{ end }} diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index 4d2c3e95fb..e588358f2d 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -327,6 +327,12 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( } policiesCfg := vsc.generatePolicies(ownerDetails, vsEx.VirtualServer.Spec.Policies, vsEx.Policies, specContext, policyOpts) + if policiesCfg.JWKSAuthEnabled { + jwtAuthKey := policiesCfg.JWTAuth.Key + policiesCfg.JWTAuthList = make(map[string]*version2.JWTAuth) + policiesCfg.JWTAuthList[jwtAuthKey] = policiesCfg.JWTAuth + } + dosCfg := generateDosCfg(dosResources[""]) // enabledInternalRoutes controls if a virtual server is configured as an internal route. @@ -469,6 +475,18 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( if policiesCfg.OIDC { routePoliciesCfg.OIDC = policiesCfg.OIDC } + if routePoliciesCfg.JWKSAuthEnabled { + policiesCfg.JWKSAuthEnabled = routePoliciesCfg.JWKSAuthEnabled + + if policiesCfg.JWTAuthList == nil { + policiesCfg.JWTAuthList = make(map[string]*version2.JWTAuth) + } + + jwtAuthKey := routePoliciesCfg.JWTAuth.Key + if _, exists := policiesCfg.JWTAuthList[jwtAuthKey]; !exists { + policiesCfg.JWTAuthList[jwtAuthKey] = routePoliciesCfg.JWTAuth + } + } limitReqZones = append(limitReqZones, routePoliciesCfg.LimitReqZones...) dosRouteCfg := generateDosCfg(dosResources[r.Path]) @@ -579,6 +597,18 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( if policiesCfg.OIDC { routePoliciesCfg.OIDC = policiesCfg.OIDC } + if routePoliciesCfg.JWKSAuthEnabled { + policiesCfg.JWKSAuthEnabled = routePoliciesCfg.JWKSAuthEnabled + + if policiesCfg.JWTAuthList == nil { + policiesCfg.JWTAuthList = make(map[string]*version2.JWTAuth) + } + + jwtAuthKey := routePoliciesCfg.JWTAuth.Key + if _, exists := policiesCfg.JWTAuthList[jwtAuthKey]; !exists { + policiesCfg.JWTAuthList[jwtAuthKey] = routePoliciesCfg.JWTAuth + } + } limitReqZones = append(limitReqZones, routePoliciesCfg.LimitReqZones...) dosRouteCfg := generateDosCfg(dosResources[r.Path]) @@ -675,6 +705,8 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( LimitReqs: policiesCfg.LimitReqs, JWTAuth: policiesCfg.JWTAuth, BasicAuth: policiesCfg.BasicAuth, + JWTAuthList: policiesCfg.JWTAuthList, + JWKSAuthEnabled: policiesCfg.JWKSAuthEnabled, IngressMTLS: policiesCfg.IngressMTLS, EgressMTLS: policiesCfg.EgressMTLS, OIDC: vsc.oidcPolCfg.oidc, @@ -699,6 +731,8 @@ type policiesCfg struct { LimitReqZones []version2.LimitReqZone LimitReqs []version2.LimitReq JWTAuth *version2.JWTAuth + JWTAuthList map[string]*version2.JWTAuth + JWKSAuthEnabled bool BasicAuth *version2.BasicAuth IngressMTLS *version2.IngressMTLS EgressMTLS *version2.EgressMTLS @@ -858,11 +892,13 @@ func (p *policiesCfg) addJWTAuthConfig( } p.JWTAuth = &version2.JWTAuth{ + Key: polKey, JwksURI: *JwksURI, Realm: jwtAuth.Realm, Token: jwtAuth.Token, KeyCache: jwtAuth.KeyCache, } + p.JWKSAuthEnabled = true return res } return res diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index 933d4931d1..2227ad98fc 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -2909,6 +2909,261 @@ func TestGenerateVirtualServerConfigForVirtualServerWithReturns(t *testing.T) { } } +func TestGenerateVirtualServerConfigJWKSPolicy(t *testing.T) { + t.Parallel() + + virtualServerEx := VirtualServerEx{ + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + Policies: []conf_v1.PolicyReference{ + { + Name: "jwt-policy", + }, + }, + Upstreams: []conf_v1.Upstream{ + { + Name: "tea", + Service: "tea-svc", + Port: 80, + }, + { + Name: "coffee", + Service: "coffee-svc", + Port: 80, + }, + }, + Routes: []conf_v1.Route{ + { + Path: "/tea", + Action: &conf_v1.Action{ + Pass: "tea", + }, + Policies: []conf_v1.PolicyReference{ + { + Name: "jwt-policy-route", + }, + }, + }, + { + Path: "/coffee", + Action: &conf_v1.Action{ + Pass: "coffee", + }, + Policies: []conf_v1.PolicyReference{ + { + Name: "jwt-policy-route", + }, + }, + }, + }, + }, + }, + Policies: map[string]*conf_v1.Policy{ + "default/jwt-policy": { + ObjectMeta: meta_v1.ObjectMeta{ + Name: "jwt-policy", + Namespace: "default", + }, + Spec: conf_v1.PolicySpec{ + JWTAuth: &conf_v1.JWTAuth{ + Realm: "Spec Realm API", + JwksURI: "https://idp.spec.example.com:443/spec-keys", + KeyCache: "1h", + }, + }, + }, + "default/jwt-policy-route": { + ObjectMeta: meta_v1.ObjectMeta{ + Name: "jwt-policy-route", + Namespace: "default", + }, + Spec: conf_v1.PolicySpec{ + JWTAuth: &conf_v1.JWTAuth{ + Realm: "Route Realm API", + JwksURI: "http://idp.route.example.com:80/route-keys", + KeyCache: "1h", + }, + }, + }, + }, + Endpoints: map[string][]string{ + "default/tea-svc:80": { + "10.0.0.20:80", + }, + "default/coffee-svc:80": { + "10.0.0.30:80", + }, + }, + } + + expected := version2.VirtualServerConfig{ + Upstreams: []version2.Upstream{ + { + UpstreamLabels: version2.UpstreamLabels{ + Service: "tea-svc", + ResourceType: "virtualserver", + ResourceName: "cafe", + ResourceNamespace: "default", + }, + Name: "vs_default_cafe_tea", + Servers: []version2.UpstreamServer{ + { + Address: "10.0.0.20:80", + }, + }, + Keepalive: 16, + }, + { + UpstreamLabels: version2.UpstreamLabels{ + Service: "coffee-svc", + ResourceType: "virtualserver", + ResourceName: "cafe", + ResourceNamespace: "default", + }, + Name: "vs_default_cafe_coffee", + Servers: []version2.UpstreamServer{ + { + Address: "10.0.0.30:80", + }, + }, + Keepalive: 16, + }, + }, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Server: version2.Server{ + JWTAuthList: map[string]*version2.JWTAuth{ + "default/jwt-policy": { + Key: "default/jwt-policy", + Realm: "Spec Realm API", + KeyCache: "1h", + JwksURI: version2.JwksURI{ + JwksScheme: "https", + JwksHost: "idp.spec.example.com", + JwksPort: "443", + JwksPath: "/spec-keys", + }, + }, + "default/jwt-policy-route": { + Key: "default/jwt-policy-route", + Realm: "Route Realm API", + KeyCache: "1h", + JwksURI: version2.JwksURI{ + JwksScheme: "http", + JwksHost: "idp.route.example.com", + JwksPort: "80", + JwksPath: "/route-keys", + }, + }, + }, + JWTAuth: &version2.JWTAuth{ + Key: "default/jwt-policy", + Realm: "Spec Realm API", + KeyCache: "1h", + JwksURI: version2.JwksURI{ + JwksScheme: "https", + JwksHost: "idp.spec.example.com", + JwksPort: "443", + JwksPath: "/spec-keys", + }, + }, + JWKSAuthEnabled: true, + ServerName: "cafe.example.com", + StatusZone: "cafe.example.com", + ProxyProtocol: true, + ServerTokens: "off", + RealIPHeader: "X-Real-IP", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPRecursive: true, + Snippets: []string{"# server snippet"}, + TLSPassthrough: true, + VSNamespace: "default", + VSName: "cafe", + Locations: []version2.Location{ + { + Path: "/tea", + ProxyPass: "http://vs_default_cafe_tea", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + HasKeepalive: true, + ProxySSLName: "tea-svc.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "tea-svc", + JWTAuth: &version2.JWTAuth{ + Key: "default/jwt-policy-route", + Realm: "Route Realm API", + KeyCache: "1h", + JwksURI: version2.JwksURI{ + JwksScheme: "http", + JwksHost: "idp.route.example.com", + JwksPort: "80", + JwksPath: "/route-keys", + }, + }, + }, + { + Path: "/coffee", + ProxyPass: "http://vs_default_cafe_coffee", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + HasKeepalive: true, + ProxySSLName: "coffee-svc.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "coffee-svc", + JWTAuth: &version2.JWTAuth{ + Key: "default/jwt-policy-route", + Realm: "Route Realm API", + KeyCache: "1h", + JwksURI: version2.JwksURI{ + JwksScheme: "http", + JwksHost: "idp.route.example.com", + JwksPort: "80", + JwksPath: "/route-keys", + }, + }, + }, + }, + }, + } + + baseCfgParams := ConfigParams{ + ServerTokens: "off", + Keepalive: 16, + ServerSnippets: []string{"# server snippet"}, + ProxyProtocol: true, + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + } + + vsc := newVirtualServerConfigurator( + &baseCfgParams, + false, + false, + &StaticConfigParams{TLSPassthrough: true}, + false, + ) + + result, warnings := vsc.GenerateVirtualServerConfig(&virtualServerEx, nil, nil) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff) + } + + if len(warnings) != 0 { + t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings) + } +} + func TestGeneratePolicies(t *testing.T) { t.Parallel() ownerDetails := policyOwnerDetails{ @@ -3187,6 +3442,7 @@ func TestGeneratePolicies(t *testing.T) { Secret: "/etc/nginx/secrets/default-jwt-secret", Realm: "My Test API", }, + JWKSAuthEnabled: false, }, msg: "jwt reference", }, @@ -3200,7 +3456,7 @@ func TestGeneratePolicies(t *testing.T) { policies: map[string]*conf_v1.Policy{ "default/jwt-policy-2": { ObjectMeta: meta_v1.ObjectMeta{ - Name: "jwt-policy", + Name: "jwt-policy-2", Namespace: "default", }, Spec: conf_v1.PolicySpec{ @@ -3214,6 +3470,7 @@ func TestGeneratePolicies(t *testing.T) { }, expected: policiesCfg{ JWTAuth: &version2.JWTAuth{ + Key: "default/jwt-policy-2", Realm: "My Test API", JwksURI: version2.JwksURI{ JwksScheme: "https", @@ -3223,6 +3480,7 @@ func TestGeneratePolicies(t *testing.T) { }, KeyCache: "1h", }, + JWKSAuthEnabled: true, }, msg: "Basic jwks example", }, @@ -3236,7 +3494,7 @@ func TestGeneratePolicies(t *testing.T) { policies: map[string]*conf_v1.Policy{ "default/jwt-policy-2": { ObjectMeta: meta_v1.ObjectMeta{ - Name: "jwt-policy", + Name: "jwt-policy-2", Namespace: "default", }, Spec: conf_v1.PolicySpec{ @@ -3250,6 +3508,7 @@ func TestGeneratePolicies(t *testing.T) { }, expected: policiesCfg{ JWTAuth: &version2.JWTAuth{ + Key: "default/jwt-policy-2", Realm: "My Test API", JwksURI: version2.JwksURI{ JwksScheme: "https", @@ -3259,6 +3518,7 @@ func TestGeneratePolicies(t *testing.T) { }, KeyCache: "1h", }, + JWKSAuthEnabled: true, }, msg: "Basic jwks example, no port in JwksURI", }, diff --git a/tests/data/jwt-policy-jwksuri/policies/jwt-policy-invalid.yaml b/tests/data/jwt-policy-jwksuri/policies/jwt-policy-invalid.yaml new file mode 100644 index 0000000000..2123b69265 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/policies/jwt-policy-invalid.yaml @@ -0,0 +1,10 @@ +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: jwt-policy-invalid +spec: + jwt: + realm: MyProductAPI + token: $$http_token + jwksURI: https://login.microsoftonline.com/dd3dfd2f-6a3b-40d1-9be0-bf8327d81c50/discovery/v2.0/keys + keyCache: 1h diff --git a/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-invalid-policy-subroute.yaml b/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-invalid-policy-subroute.yaml new file mode 100644 index 0000000000..6b4cbb5ada --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-invalid-policy-subroute.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backends +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend3 + service: backend3-svc + port: 80 + subroutes: + - path: "/backends/backend1" + policies: + - name: jwt-policy-valid + action: + pass: backend1 + - path: "/backends/backend3" + action: + pass: backend3 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-policy-subroute.yaml b/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-policy-subroute.yaml new file mode 100644 index 0000000000..6b4cbb5ada --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-policy-subroute.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backends +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend3 + service: backend3-svc + port: 80 + subroutes: + - path: "/backends/backend1" + policies: + - name: jwt-policy-valid + action: + pass: backend1 + - path: "/backends/backend3" + action: + pass: backend3 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route-subpath.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route-subpath.yaml new file mode 100644 index 0000000000..4bde0eb1e9 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route-subpath.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1/subpath1" + action: + pass: backend1 + policies: + - name: jwt-policy-invalid + - path: "/backend2/subpath2" + action: + pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route.yaml new file mode 100644 index 0000000000..c2fed43359 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1" + action: + pass: backend1 + policies: + - name: jwt-policy-invalid + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-spec.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-spec.yaml new file mode 100644 index 0000000000..6351e6b230 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-spec.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + policies: + - name: jwt-policy-invalid + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1" + action: + pass: backend1 + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath-diff-host.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath-diff-host.yaml new file mode 100644 index 0000000000..ca5787f3d4 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath-diff-host.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server-2 +spec: + host: virtual-server-2.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1/subpath1" + action: + pass: backend1 + policies: + - name: jwt-policy-valid + - path: "/backend2/subpath2" + action: + pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath.yaml new file mode 100644 index 0000000000..e70e20f5e9 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1/subpath1" + action: + pass: backend1 + policies: + - name: jwt-policy-valid + - path: "/backend2/subpath2" + action: + pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route.yaml index aa0cb6f44c..18f1917c6d 100644 --- a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route.yaml +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route.yaml @@ -16,7 +16,7 @@ spec: action: pass: backend1 policies: - - name: jwt-policy-valid + - name: jwt-policy-valid - path: "/backend2" action: pass: backend2 diff --git a/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-spec-and-route.yaml b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-spec-and-route.yaml new file mode 100644 index 0000000000..9e79515fe9 --- /dev/null +++ b/tests/data/jwt-policy-jwksuri/virtual-server/virtual-server-policy-spec-and-route.yaml @@ -0,0 +1,24 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + policies: + - name: jwt-policy-valid + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1" + action: + pass: backend1 + policies: + - name: jwt-policy-valid + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/suite/test_jwt_policies_jwksuri.py b/tests/suite/test_jwt_policies_jwksuri.py index 65ef44cd3f..9006de79d6 100644 --- a/tests/suite/test_jwt_policies_jwksuri.py +++ b/tests/suite/test_jwt_policies_jwksuri.py @@ -6,12 +6,28 @@ from settings import TEST_DATA from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy from suite.utils.resources_utils import replace_configmap_from_yaml, wait_before_test -from suite.utils.vs_vsr_resources_utils import delete_and_create_vs_from_yaml, patch_v_s_route_from_yaml +from suite.utils.vs_vsr_resources_utils import ( + create_virtual_server_from_yaml, + delete_and_create_vs_from_yaml, + delete_virtual_server, + patch_v_s_route_from_yaml, +) std_vs_src = f"{TEST_DATA}/virtual-server/standard/virtual-server.yaml" jwt_pol_valid_src = f"{TEST_DATA}/jwt-policy-jwksuri/policies/jwt-policy-valid.yaml" +jwt_pol_invalid_src = f"{TEST_DATA}/jwt-policy-jwksuri/policies/jwt-policy-invalid.yaml" jwt_vs_spec_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-policy-spec.yaml" jwt_vs_route_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route.yaml" +jwt_spec_and_route_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-policy-spec-and-route.yaml" +jwt_vs_route_subpath_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath.yaml" +jwt_vs_route_subpath_diff_host_src = ( + f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-policy-route-subpath-diff-host.yaml" +) +jwt_vs_invalid_pol_spec_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-spec.yaml" +jwt_vs_invalid_pol_route_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route.yaml" +jwt_vs_invalid_pol_route_subpath_src = ( + f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server/virtual-server-invalid-policy-route-subpath.yaml" +) jwt_cm_src = f"{TEST_DATA}/jwt-policy-jwksuri/configmap/nginx-config.yaml" ad_tenant = "dd3dfd2f-6a3b-40d1-9be0-bf8327d81c50" client_id = "8a172a83-a630-41a4-9ca6-1e5ef03cd7e7" @@ -41,7 +57,6 @@ def get_token(request): @pytest.mark.skip_for_nginx_oss @pytest.mark.policies -@pytest.mark.skip(reason="issues with ingressClass") @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ @@ -62,7 +77,7 @@ def get_token(request): indirect=True, ) class TestJWTPoliciesVsJwksuri: - @pytest.mark.parametrize("jwt_virtual_server", [jwt_vs_spec_src, jwt_vs_route_src]) + @pytest.mark.parametrize("jwt_virtual_server", [jwt_vs_spec_src, jwt_vs_route_src, jwt_spec_and_route_src]) def test_jwt_policy_jwksuri( self, request, @@ -74,7 +89,7 @@ def test_jwt_policy_jwksuri( jwt_virtual_server, ): """ - Test jwt-policy in Virtual Server (spec and route) with keys fetched form Azure + Test jwt-policy in Virtual Server (spec, route and both at the same time) with keys fetched form Azure """ replace_configmap_from_yaml( kube_apis.v1, @@ -92,17 +107,75 @@ def test_jwt_policy_jwksuri( jwt_virtual_server, virtual_server_setup.namespace, ) - resp1 = mock.Mock() - resp1.status_code == 502 + resp_no_token = mock.Mock() + resp_no_token.status_code == 502 counter = 0 - while resp1.status_code != 401 and counter < 3: - resp1 = requests.get( + while resp_no_token.status_code != 401 and counter < 10: + resp_no_token = requests.get( virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host}, ) wait_before_test() - counter = +1 + counter += 1 + + token = get_token(request) + + resp_valid_token = requests.get( + virtual_server_setup.backend_1_url, + headers={"host": virtual_server_setup.vs_host, "token": token}, + timeout=5, + ) + + delete_policy(kube_apis.custom_objects, pol_name, test_namespace) + wait_before_test() + + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + std_vs_src, + virtual_server_setup.namespace, + ) + + assert resp_no_token.status_code == 401 and f"Authorization Required" in resp_no_token.text + assert resp_valid_token.status_code == 200 and f"Request ID:" in resp_valid_token.text + + @pytest.mark.parametrize("jwt_virtual_server", [jwt_vs_invalid_pol_spec_src, jwt_vs_invalid_pol_route_src]) + def test_jwt_invalid_policy_jwksuri( + self, + request, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + virtual_server_setup, + test_namespace, + jwt_virtual_server, + ): + """ + Test invalid jwt-policy in Virtual Server (spec and route) with keys fetched form Azure + """ + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + jwt_cm_src, + ) + pol_name = create_policy_from_yaml(kube_apis.custom_objects, jwt_pol_invalid_src, test_namespace) + wait_before_test() + + print(f"Patch vs with policy: {jwt_virtual_server}") + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + jwt_virtual_server, + virtual_server_setup.namespace, + ) + wait_before_test() + + resp1 = requests.get( + virtual_server_setup.backend_1_url, + headers={"host": virtual_server_setup.vs_host}, + ) token = get_token(request) @@ -120,5 +193,167 @@ def test_jwt_policy_jwksuri( virtual_server_setup.namespace, ) - assert resp1.status_code == 401 and f"Authorization Required" in resp1.text - assert resp2.status_code == 200 and f"Request ID:" in resp2.text + assert resp1.status_code == 500 and f"Internal Server Error" in resp1.text + assert resp2.status_code == 500 and f"Internal Server Error" in resp2.text + + @pytest.mark.parametrize("jwt_virtual_server", [jwt_vs_route_subpath_src]) + def test_jwt_policy_subroute_jwksuri( + self, + request, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + virtual_server_setup, + test_namespace, + jwt_virtual_server, + ): + """ + Test jwt-policy in Virtual Server using subpaths with keys fetched form Azure + """ + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + jwt_cm_src, + ) + pol_name = create_policy_from_yaml(kube_apis.custom_objects, jwt_pol_valid_src, test_namespace) + wait_before_test() + + print(f"Patch vs with policy: {jwt_virtual_server}") + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + jwt_virtual_server, + virtual_server_setup.namespace, + ) + resp_no_token = mock.Mock() + resp_no_token.status_code == 502 + counter = 0 + + while resp_no_token.status_code != 401 and counter < 10: + resp_no_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": virtual_server_setup.vs_host}, + ) + wait_before_test() + counter += 1 + + token = get_token(request) + + resp_valid_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": virtual_server_setup.vs_host, "token": token}, + timeout=5, + ) + + delete_policy(kube_apis.custom_objects, pol_name, test_namespace) + wait_before_test() + + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + std_vs_src, + virtual_server_setup.namespace, + ) + + assert resp_no_token.status_code == 401 and f"Authorization Required" in resp_no_token.text + assert resp_valid_token.status_code == 200 and f"Request ID:" in resp_valid_token.text + + def test_jwt_policy_subroute_jwksuri_multiple_vs( + self, + request, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + virtual_server_setup, + test_namespace, + ): + """ + Test jwt-policy applied to two Virtual Servers with different hosts and the same subpaths + """ + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + jwt_cm_src, + ) + pol_name = create_policy_from_yaml(kube_apis.custom_objects, jwt_pol_valid_src, test_namespace) + wait_before_test() + + print(f"Patch first vs with policy: {jwt_vs_route_subpath_src}") + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + jwt_vs_route_subpath_src, + virtual_server_setup.namespace, + ) + + print(f"Create second vs with policy: {jwt_vs_route_subpath_diff_host_src}") + create_virtual_server_from_yaml( + kube_apis.custom_objects, + jwt_vs_route_subpath_diff_host_src, + virtual_server_setup.namespace, + ) + + wait_before_test() + + resp_1_no_token = mock.Mock() + resp_1_no_token.status_code == 502 + + resp_2_no_token = mock.Mock() + resp_2_no_token.status_code == 502 + counter = 0 + + while resp_1_no_token.status_code != 401 and counter < 10: + resp_1_no_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": virtual_server_setup.vs_host}, + ) + wait_before_test() + counter += 1 + + counter = 0 + + while resp_2_no_token.status_code != 401 and counter < 10: + resp_2_no_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": "virtual-server-2.example.com"}, + ) + wait_before_test() + counter += 1 + + token = get_token(request) + + resp_1_valid_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": virtual_server_setup.vs_host, "token": token}, + timeout=5, + ) + + resp_2_valid_token = requests.get( + virtual_server_setup.backend_1_url + "/subpath1", + headers={"host": "virtual-server-2.example.com", "token": token}, + timeout=5, + ) + + delete_policy(kube_apis.custom_objects, pol_name, test_namespace) + wait_before_test() + + delete_and_create_vs_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + std_vs_src, + virtual_server_setup.namespace, + ) + + delete_virtual_server( + kube_apis.custom_objects, + "virtual-server-2", + virtual_server_setup.namespace, + ) + + assert resp_1_no_token.status_code == 401 and f"Authorization Required" in resp_1_no_token.text + assert resp_1_valid_token.status_code == 200 and f"Request ID:" in resp_1_valid_token.text + + assert resp_2_no_token.status_code == 401 and f"Authorization Required" in resp_2_no_token.text + assert resp_2_valid_token.status_code == 200 and f"Request ID:" in resp_2_valid_token.text diff --git a/tests/suite/test_jwt_policies_jwksuri_vsr.py b/tests/suite/test_jwt_policies_jwksuri_vsr.py new file mode 100644 index 0000000000..c3b772f469 --- /dev/null +++ b/tests/suite/test_jwt_policies_jwksuri_vsr.py @@ -0,0 +1,195 @@ +import time +from unittest import mock + +import pytest +import requests +from settings import TEST_DATA +from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy +from suite.utils.resources_utils import replace_configmap_from_yaml, wait_before_test +from suite.utils.vs_vsr_resources_utils import patch_v_s_route_from_yaml + +std_vsr_src = f"{TEST_DATA}/virtual-server-route/route-multiple.yaml" +jwt_pol_valid_src = f"{TEST_DATA}/jwt-policy-jwksuri/policies/jwt-policy-valid.yaml" +jwt_pol_invalid_src = f"{TEST_DATA}/jwt-policy-jwksuri/policies/jwt-policy-invalid.yaml" +jwt_vsr_subroute_src = f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-policy-subroute.yaml" +jwt_vsr_invalid_pol_subroute_src = ( + f"{TEST_DATA}/jwt-policy-jwksuri/virtual-server-route/virtual-server-route-invalid-policy-subroute.yaml" +) +jwt_cm_src = f"{TEST_DATA}/jwt-policy-jwksuri/configmap/nginx-config.yaml" +ad_tenant = "dd3dfd2f-6a3b-40d1-9be0-bf8327d81c50" +client_id = "8a172a83-a630-41a4-9ca6-1e5ef03cd7e7" + + +def get_token(request): + """ + get jwt token from azure ad endpoint + """ + data = { + "client_id": f"{client_id}", + "scope": ".default", + "client_secret": request.config.getoption("--ad-secret"), + "grant_type": "client_credentials", + } + ad_response = requests.get( + f"https://login.microsoftonline.com/{ad_tenant}/oauth2/token", + data=data, + timeout=5, + headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Chrome/76.0.3809.100"}, + ) + + if ad_response.status_code == 200: + return ad_response.json()["access_token"] + pytest.fail("Unable to request Azure token endpoint") + + +@pytest.mark.skip_for_nginx_oss +@pytest.mark.jwks +@pytest.mark.parametrize( + "crd_ingress_controller, v_s_route_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + f"-enable-custom-resources", + f"-enable-leader-election=false", + ], + }, + { + "example": "virtual-server-route", + }, + ) + ], + indirect=True, +) +class TestJWTPoliciesVSRJwksuri: + @pytest.mark.parametrize("jwt_virtual_server_route", [jwt_vsr_subroute_src]) + def test_jwt_policy_jwksuri( + self, + request, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + v_s_route_app_setup, + v_s_route_setup, + test_namespace, + jwt_virtual_server_route, + ): + """ + Test jwt-policy in Virtual Server Route with keys fetched form Azure + """ + req_url = f"http://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}" + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + jwt_cm_src, + ) + pol_name = create_policy_from_yaml( + kube_apis.custom_objects, jwt_pol_valid_src, v_s_route_setup.route_m.namespace + ) + wait_before_test() + + print(f"Patch vsr with policy: {jwt_virtual_server_route}") + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + v_s_route_setup.route_m.name, + jwt_virtual_server_route, + v_s_route_setup.route_m.namespace, + ) + resp_no_token = mock.Mock() + resp_no_token.status_code == 502 + counter = 0 + + while resp_no_token.status_code != 401 and counter < 10: + resp_no_token = requests.get( + f"{req_url}{v_s_route_setup.route_m.paths[0]}", + headers={"host": v_s_route_setup.vs_host}, + ) + wait_before_test() + counter += 1 + + token = get_token(request) + + resp_valid_token = requests.get( + f"{req_url}{v_s_route_setup.route_m.paths[0]}", + headers={"host": v_s_route_setup.vs_host, "token": token}, + ) + + delete_policy(kube_apis.custom_objects, pol_name, v_s_route_setup.route_m.namespace) + wait_before_test() + + resp_pol_deleted = requests.get( + f"{req_url}{v_s_route_setup.route_m.paths[0]}", + headers={"host": v_s_route_setup.vs_host, "token": token}, + ) + + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + v_s_route_setup.route_m.name, + std_vsr_src, + v_s_route_setup.route_m.namespace, + ) + + assert resp_no_token.status_code == 401 and f"Authorization Required" in resp_no_token.text + assert resp_valid_token.status_code == 200 and f"Request ID:" in resp_valid_token.text + assert resp_pol_deleted.status_code == 500 and f"Internal Server Error" in resp_pol_deleted.text + + @pytest.mark.parametrize("jwt_virtual_server_route", [jwt_vsr_invalid_pol_subroute_src]) + def test_jwt_invalid_policy_jwksuri( + self, + request, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + v_s_route_app_setup, + v_s_route_setup, + test_namespace, + jwt_virtual_server_route, + ): + """ + Test invalid jwt-policy in Virtual Server Route with keys fetched form Azure + """ + req_url = f"http://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}" + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + jwt_cm_src, + ) + pol_name = create_policy_from_yaml( + kube_apis.custom_objects, jwt_pol_invalid_src, v_s_route_setup.route_m.namespace + ) + wait_before_test() + + print(f"Patch vsr with policy: {jwt_virtual_server_route}") + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + v_s_route_setup.route_m.name, + jwt_virtual_server_route, + v_s_route_setup.route_m.namespace, + ) + wait_before_test() + + resp1 = requests.get( + f"{req_url}{v_s_route_setup.route_m.paths[0]}", + headers={"host": v_s_route_setup.vs_host}, + ) + + token = get_token(request) + + resp2 = requests.get( + f"{req_url}{v_s_route_setup.route_m.paths[0]}", + headers={"host": v_s_route_setup.vs_host, "token": token}, + ) + + delete_policy(kube_apis.custom_objects, pol_name, v_s_route_setup.route_m.namespace) + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + v_s_route_setup.route_m.name, + std_vsr_src, + v_s_route_setup.route_m.namespace, + ) + + assert resp1.status_code == 500 and f"Internal Server Error" in resp1.text + assert resp2.status_code == 500 and f"Internal Server Error" in resp2.text diff --git a/tests/suite/utils/vs_vsr_resources_utils.py b/tests/suite/utils/vs_vsr_resources_utils.py index c1fe02ef8c..2b2982aa08 100644 --- a/tests/suite/utils/vs_vsr_resources_utils.py +++ b/tests/suite/utils/vs_vsr_resources_utils.py @@ -6,7 +6,7 @@ from kubernetes.client import CoreV1Api, CustomObjectsApi from kubernetes.client.rest import ApiException from suite.utils.custom_resources_utils import read_custom_resource -from suite.utils.resources_utils import ensure_item_removal, get_file_contents +from suite.utils.resources_utils import ensure_item_removal, get_file_contents, wait_before_test def read_vs(custom_objects: CustomObjectsApi, namespace, name) -> object: @@ -144,6 +144,7 @@ def delete_and_create_vs_from_yaml(custom_objects: CustomObjectsApi, name, yaml_ try: delete_virtual_server(custom_objects, name, namespace) create_virtual_server_from_yaml(custom_objects, yaml_manifest, namespace) + wait_before_test() except ApiException: logging.exception(f"Failed with exception while patching VirtualServer: {name}") raise @@ -183,6 +184,7 @@ def patch_v_s_route_from_yaml(custom_objects: CustomObjectsApi, name, yaml_manif custom_objects.patch_namespaced_custom_object( "k8s.nginx.org", "v1", namespace, "virtualserverroutes", name, dep ) + wait_before_test() print(f"VirtualServerRoute updated with name '{dep['metadata']['name']}'") except ApiException: logging.exception(f"Failed with exception while patching VirtualServerRoute: {name}")