From adeab58f58852b8e18c7e1fc8e5b0fd40c04a746 Mon Sep 17 00:00:00 2001 From: Boshi Lian Date: Wed, 11 Oct 2017 21:27:22 +0800 Subject: [PATCH 1/4] introduce go client style config creating functions --- ...ubernetesClientConfiguration.ConfigFile.cs | 202 ++++++++++++++++++ ...KubernetesClientConfiguration.InCluster.cs | 35 +++ src/KubernetesClientConfiguration.cs | 161 +------------- src/Utils.cs | 20 +- 4 files changed, 258 insertions(+), 160 deletions(-) create mode 100644 src/KubernetesClientConfiguration.ConfigFile.cs create mode 100644 src/KubernetesClientConfiguration.InCluster.cs diff --git a/src/KubernetesClientConfiguration.ConfigFile.cs b/src/KubernetesClientConfiguration.ConfigFile.cs new file mode 100644 index 000000000..0817d3bd6 --- /dev/null +++ b/src/KubernetesClientConfiguration.ConfigFile.cs @@ -0,0 +1,202 @@ +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography.X509Certificates; +using k8s.Exceptions; +using k8s.KubeConfigModels; +using YamlDotNet.Serialization; + +namespace k8s +{ + public partial class KubernetesClientConfiguration + { + /// + /// Gets CurrentContext + /// + public string CurrentContext { get; private set; } + + /// + /// kubeconfig Default Location + /// + private static readonly string KubeConfigDefaultLocation = + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Path.Combine(Environment.GetEnvironmentVariable("USERPROFILE"), @".kube\config") + : Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".kube/config"); + + /// + /// Initializes a new instance of the class. + /// + /// kubeconfig file info + /// Context to use from kube config + public KubernetesClientConfiguration(FileInfo kubeconfig = null, string currentContext = null) + { + var k8SConfig = LoadKubeConfig(kubeconfig ?? new FileInfo(KubeConfigDefaultLocation)); + this.Initialize(k8SConfig, currentContext); + } + + /// + /// Initializes a new instance of the from config file + /// + /// kube api server endpoint + /// kubeconfig filepath + /// + public static KubernetesClientConfiguration BuildConfigFromConfigFile(string masterUrl = null, string kubeconfigPath = null) + { + var k8SConfig = LoadKubeConfig(new FileInfo(kubeconfigPath ?? KubeConfigDefaultLocation)); + var k8SConfiguration = new KubernetesClientConfiguration(); + k8SConfiguration.Initialize(k8SConfig); + + if (!string.IsNullOrWhiteSpace(masterUrl)) + { + k8SConfiguration.Host = masterUrl; + } + return k8SConfiguration; + } + + /// + /// Validates and Intializes Client Configuration + /// + /// Kubernetes Configuration + /// Current Context + private void Initialize(K8SConfiguration k8SConfig, string currentContext = null) + { + if (k8SConfig.Contexts == null) + { + throw new KubeConfigException("No contexts found in kubeconfig"); + } + + if (k8SConfig.Clusters == null) + { + throw new KubeConfigException($"No clusters found in kubeconfig"); + } + + if (k8SConfig.Users == null) + { + throw new KubeConfigException($"No users found in kubeconfig"); + } + + // current context + currentContext = currentContext ?? k8SConfig.CurrentContext; + Context activeContext = + k8SConfig.Contexts.FirstOrDefault( + c => c.Name.Equals(currentContext, StringComparison.OrdinalIgnoreCase)); + if (activeContext == null) + { + throw new KubeConfigException($"CurrentContext: {currentContext} not found in contexts in kubeconfig"); + } + + this.CurrentContext = activeContext.Name; + + // cluster + var clusterDetails = + k8SConfig.Clusters.FirstOrDefault(c => c.Name.Equals(activeContext.ContextDetails.Cluster, + StringComparison.OrdinalIgnoreCase)); + if (clusterDetails?.ClusterEndpoint == null) + { + throw new KubeConfigException($"Cluster not found for context {activeContext} in kubeconfig"); + } + + if (string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.Server)) + { + throw new KubeConfigException($"Server not found for current-context {activeContext} in kubeconfig"); + } + + if (!clusterDetails.ClusterEndpoint.SkipTlsVerify && + string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.CertificateAuthorityData) && + string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.CertificateAuthority)) + { + throw new KubeConfigException( + $"neither certificate-authority-data nor certificate-authority not found for current-context :{activeContext} in kubeconfig"); + } + + this.Host = clusterDetails.ClusterEndpoint.Server; + if (!string.IsNullOrEmpty(clusterDetails.ClusterEndpoint.CertificateAuthorityData)) + { + string data = clusterDetails.ClusterEndpoint.CertificateAuthorityData; + this.SslCaCert = new X509Certificate2(System.Text.Encoding.UTF8.GetBytes(Utils.Base64Decode(data))); + } + else if (!string.IsNullOrEmpty(clusterDetails.ClusterEndpoint.CertificateAuthority)) + { + this.SslCaCert = new X509Certificate2(clusterDetails.ClusterEndpoint.CertificateAuthority); + } + this.SkipTlsVerify = clusterDetails.ClusterEndpoint.SkipTlsVerify; + + // user + this.SetUserDetails(k8SConfig, activeContext); + } + + private void SetUserDetails(K8SConfiguration k8SConfig, Context activeContext) + { + var userDetails = k8SConfig.Users.FirstOrDefault(c => c.Name.Equals(activeContext.ContextDetails.User, + StringComparison.OrdinalIgnoreCase)); + + if (userDetails == null) + { + throw new KubeConfigException("User not found for context {activeContext.Name} in kubeconfig"); + } + + if (userDetails.UserCredentials == null) + { + throw new KubeConfigException($"User credentials not found for user: {userDetails.Name} in kubeconfig"); + } + + var userCredentialsFound = false; + + // Basic and bearer tokens are mutually exclusive + if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.Token)) + { + this.AccessToken = userDetails.UserCredentials.Token; + userCredentialsFound = true; + } + else if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.UserName) && + !string.IsNullOrWhiteSpace(userDetails.UserCredentials.Password)) + { + this.Username = userDetails.UserCredentials.UserName; + this.Password = userDetails.UserCredentials.Password; + userCredentialsFound = true; + } + + // Token and cert based auth can co-exist + if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientCertificateData) && + !string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientKeyData)) + { + this.ClientCertificateData = userDetails.UserCredentials.ClientCertificateData; + this.ClientCertificateKey = userDetails.UserCredentials.ClientKeyData; + userCredentialsFound = true; + } + + if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientCertificate) && + !string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientKey)) + { + this.ClientCertificate = userDetails.UserCredentials.ClientCertificate; + this.ClientKey = userDetails.UserCredentials.ClientKey; + userCredentialsFound = true; + } + + if (!userCredentialsFound) + { + throw new KubeConfigException( + $"User: {userDetails.Name} does not have appropriate auth credentials in kubeconfig"); + } + } + + /// + /// Loads Kube Config + /// + /// Kube config file contents + /// Instance of the class + private static K8SConfiguration LoadKubeConfig(FileInfo kubeconfig) + { + if (!kubeconfig.Exists) + { + throw new KubeConfigException($"kubeconfig file not found at {kubeconfig.FullName}"); + } + var kubeconfigContent = File.ReadAllText(kubeconfig.FullName); + + var deserializeBuilder = new DeserializerBuilder(); + var deserializer = deserializeBuilder.Build(); + return deserializer.Deserialize(kubeconfigContent); + } + } +} \ No newline at end of file diff --git a/src/KubernetesClientConfiguration.InCluster.cs b/src/KubernetesClientConfiguration.InCluster.cs new file mode 100644 index 000000000..da70d5f81 --- /dev/null +++ b/src/KubernetesClientConfiguration.InCluster.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; +using System.Security.Cryptography.X509Certificates; +using k8s.Exceptions; + +namespace k8s +{ + public partial class KubernetesClientConfiguration + { + private const string ServiceAccountTokenKey = "token"; + private const string ServiceAccountRootCAKey = "ca.crt"; + + public static KubernetesClientConfiguration InClusterConfig() + { + var host = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST"); + var port = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_PORT"); + + if (string.IsNullOrWhiteSpace(host) || string.IsNullOrWhiteSpace(port)) + { + throw new KubeConfigException( + "unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined"); + } + + var token = File.ReadAllText("/var/run/secrets/kubernetes.io/serviceaccount/" + ServiceAccountTokenKey); + var rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/" + ServiceAccountRootCAKey; + + return new KubernetesClientConfiguration + { + Host = new UriBuilder("https", host, Convert.ToInt32(port)).ToString(), + AccessToken = token, + SslCaCert = Utils.LoadPemFileCert(rootCAFile) + }; + } + } +} \ No newline at end of file diff --git a/src/KubernetesClientConfiguration.cs b/src/KubernetesClientConfiguration.cs index 90ed05eac..441759e88 100644 --- a/src/KubernetesClientConfiguration.cs +++ b/src/KubernetesClientConfiguration.cs @@ -12,31 +12,13 @@ namespace k8s /// /// Represents a set of kubernetes client configuration settings /// - public class KubernetesClientConfiguration + public partial class KubernetesClientConfiguration { - /// - /// Initializes a new instance of the class. - /// - /// kubeconfig file info - /// Context to use from kube config - public KubernetesClientConfiguration(FileInfo kubeconfig = null, string currentContext = null) + private KubernetesClientConfiguration() { - var k8SConfig = this.LoadKubeConfig(kubeconfig ?? new FileInfo(KubeConfigDefaultLocation)); - this.Initialize(k8SConfig, currentContext); + } - /// - /// kubeconfig Default Location - /// - private static readonly string KubeConfigDefaultLocation = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? - Path.Combine(Environment.GetEnvironmentVariable("USERPROFILE"), @".kube\config") : - Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".kube/config"); - - /// - /// Gets CurrentContext - /// - public string CurrentContext { get; private set; } - /// /// Gets Host /// @@ -95,142 +77,5 @@ public KubernetesClientConfiguration(FileInfo kubeconfig = null, string currentC /// /// The access token. public string AccessToken { get; set; } - - /// - /// Validates and Intializes Client Configuration - /// - /// Kubernetes Configuration - /// Current Context - private void Initialize(K8SConfiguration k8SConfig, string currentContext = null) - { - if (k8SConfig.Contexts == null) - { - throw new KubeConfigException("No contexts found in kubeconfig"); - } - - if (k8SConfig.Clusters == null) - { - throw new KubeConfigException($"No clusters found in kubeconfig"); - } - - if (k8SConfig.Users == null) - { - throw new KubeConfigException($"No users found in kubeconfig"); - } - - // current context - currentContext = currentContext ?? k8SConfig.CurrentContext; - Context activeContext = k8SConfig.Contexts.FirstOrDefault(c => c.Name.Equals(currentContext, StringComparison.OrdinalIgnoreCase)); - if (activeContext == null) - { - throw new KubeConfigException($"CurrentContext: {currentContext} not found in contexts in kubeconfig"); - } - - this.CurrentContext = activeContext.Name; - - // cluster - var clusterDetails = k8SConfig.Clusters.FirstOrDefault(c => c.Name.Equals(activeContext.ContextDetails.Cluster, StringComparison.OrdinalIgnoreCase)); - if (clusterDetails?.ClusterEndpoint == null) - { - throw new KubeConfigException($"Cluster not found for context {activeContext} in kubeconfig"); - } - - if (string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.Server)) - { - throw new KubeConfigException($"Server not found for current-context {activeContext} in kubeconfig"); - } - - if (!clusterDetails.ClusterEndpoint.SkipTlsVerify && - string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.CertificateAuthorityData) && - string.IsNullOrWhiteSpace(clusterDetails.ClusterEndpoint.CertificateAuthority)) - { - throw new KubeConfigException($"neither certificate-authority-data nor certificate-authority not found for current-context :{activeContext} in kubeconfig"); - } - - this.Host = clusterDetails.ClusterEndpoint.Server; - if (!string.IsNullOrEmpty(clusterDetails.ClusterEndpoint.CertificateAuthorityData)) - { - string data = clusterDetails.ClusterEndpoint.CertificateAuthorityData; - this.SslCaCert = new X509Certificate2(System.Text.Encoding.UTF8.GetBytes(Utils.Base64Decode(data))); - } - else if (!string.IsNullOrEmpty(clusterDetails.ClusterEndpoint.CertificateAuthority)) - { - this.SslCaCert = new X509Certificate2(clusterDetails.ClusterEndpoint.CertificateAuthority); - } - this.SkipTlsVerify = clusterDetails.ClusterEndpoint.SkipTlsVerify; - - // user - this.SetUserDetails(k8SConfig, activeContext); - } - - private void SetUserDetails(K8SConfiguration k8SConfig, Context activeContext) - { - var userDetails = k8SConfig.Users.FirstOrDefault(c => c.Name.Equals(activeContext.ContextDetails.User, StringComparison.OrdinalIgnoreCase)); - - if (userDetails == null) - { - throw new KubeConfigException("User not found for context {activeContext.Name} in kubeconfig"); - } - - if (userDetails.UserCredentials == null) - { - throw new KubeConfigException($"User credentials not found for user: {userDetails.Name} in kubeconfig"); - } - - var userCredentialsFound = false; - - // Basic and bearer tokens are mutually exclusive - if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.Token)) - { - this.AccessToken = userDetails.UserCredentials.Token; - userCredentialsFound = true; - } - else if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.UserName) && - !string.IsNullOrWhiteSpace(userDetails.UserCredentials.Password)) - { - this.Username = userDetails.UserCredentials.UserName; - this.Password = userDetails.UserCredentials.Password; - userCredentialsFound = true; - } - - // Token and cert based auth can co-exist - if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientCertificateData) && - !string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientKeyData)) - { - this.ClientCertificateData = userDetails.UserCredentials.ClientCertificateData; - this.ClientCertificateKey = userDetails.UserCredentials.ClientKeyData; - userCredentialsFound = true; - } - - if (!string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientCertificate) && - !string.IsNullOrWhiteSpace(userDetails.UserCredentials.ClientKey)) { - this.ClientCertificate = userDetails.UserCredentials.ClientCertificate; - this.ClientKey = userDetails.UserCredentials.ClientKey; - userCredentialsFound = true; - } - - if (!userCredentialsFound) - { - throw new KubeConfigException($"User: {userDetails.Name} does not have appropriate auth credentials in kubeconfig"); - } - } - - /// - /// Loads Kube Config - /// - /// Kube config file contents - /// Instance of the class - private K8SConfiguration LoadKubeConfig(FileInfo kubeconfig) - { - if (!kubeconfig.Exists) - { - throw new KubeConfigException($"kubeconfig file not found at {kubeconfig.FullName}"); - } - var kubeconfigContent = File.ReadAllText(kubeconfig.FullName); - - var deserializeBuilder = new DeserializerBuilder(); - var deserializer = deserializeBuilder.Build(); - return deserializer.Deserialize(kubeconfigContent); - } } } diff --git a/src/Utils.cs b/src/Utils.cs index 81a14c921..bf2573ecf 100644 --- a/src/Utils.cs +++ b/src/Utils.cs @@ -35,7 +35,23 @@ public static string Base64Decode(string text) } /// - /// Generates pfx from client configuration + /// Load pem encoded cert file + /// + /// Path to pem encoded cert file + /// x509 instance. + public static X509Certificate2 LoadPemFileCert(string file) + { + var certdata = File.ReadAllText(file) + .Replace("-----BEGIN CERTIFICATE-----", "") + .Replace("-----END CERTIFICATE-----", "") + .Replace("\r", "") + .Replace("\n", ""); + + return new X509Certificate2(Convert.FromBase64String(certdata)); + } + + /// + /// Generates pfx from client configuration /// /// Kuberentes Client Configuration /// Generated Pfx Path @@ -98,4 +114,4 @@ public static X509Certificate2 GeneratePfx(KubernetesClientConfiguration config) } } } -} \ No newline at end of file +} From 8d7845189003262d34a4535fdcca36d20c5dffc0 Mon Sep 17 00:00:00 2001 From: Boshi Lian Date: Thu, 12 Oct 2017 16:55:59 +0800 Subject: [PATCH 2/4] support anonymous for client --- src/Kubernetes.Auth.cs | 41 ++++++----- ...ubernetesClientConfiguration.ConfigFile.cs | 20 +++++- src/KubernetesClientConfiguration.cs | 17 +++-- src/KubernetesClientCredentials.cs | 70 ------------------- tests/KubernetesClientCredentialsTests.cs | 60 ---------------- 5 files changed, 48 insertions(+), 160 deletions(-) delete mode 100644 src/KubernetesClientCredentials.cs delete mode 100644 tests/KubernetesClientCredentialsTests.cs diff --git a/src/Kubernetes.Auth.cs b/src/Kubernetes.Auth.cs index dd1d1fb40..d3fc3b5c8 100644 --- a/src/Kubernetes.Auth.cs +++ b/src/Kubernetes.Auth.cs @@ -24,21 +24,24 @@ public Kubernetes(KubernetesClientConfiguration config) this.CaCert = config.SslCaCert; this.BaseUri = new Uri(config.Host); - // ssl cert validation - Func sslCertValidationFunc; - if (config.SkipTlsVerify) - { - sslCertValidationFunc = (sender, certificate, chain, sslPolicyErrors) => true; - } - else - { - sslCertValidationFunc = this.CertificateValidationCallBack; - } + var handler = new HttpClientHandler(); - var handler = new HttpClientHandler + if (BaseUri.Scheme == "https") { - ServerCertificateCustomValidationCallback = sslCertValidationFunc - }; + if (config.SkipTlsVerify) + { + handler.ServerCertificateCustomValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; + } + else + { + if (CaCert == null) + { + throw new KubeConfigException("a CA must be set when SkipTlsVerify === false"); + } + + handler.ServerCertificateCustomValidationCallback = CertificateValidationCallBack; + } + } // set credentails for the kubernernet client this.SetCredentials(config, handler); @@ -58,11 +61,15 @@ private void SetCredentials(KubernetesClientConfiguration config, HttpClientHand // set the Credentails for token based auth if (!string.IsNullOrWhiteSpace(config.AccessToken)) { - this.Credentials = new KubernetesClientCredentials(config.AccessToken); + Credentials = new TokenCredentials(config.AccessToken); } else if (!string.IsNullOrWhiteSpace(config.Username) && !string.IsNullOrWhiteSpace(config.Password)) { - this.Credentials = new KubernetesClientCredentials(config.Username, config.Password); + Credentials = new BasicAuthenticationCredentials + { + UserName = config.Username, + Password = config.Password + }; } // othwerwise set handler for clinet cert based auth else if ((!string.IsNullOrWhiteSpace(config.ClientCertificateData) || @@ -73,10 +80,6 @@ private void SetCredentials(KubernetesClientConfiguration config, HttpClientHand var cert = Utils.GeneratePfx(config); handler.ClientCertificates.Add(cert); } - else - { - throw new KubeConfigException("Configuration does not have appropriate auth credentials"); - } } /// diff --git a/src/KubernetesClientConfiguration.ConfigFile.cs b/src/KubernetesClientConfiguration.ConfigFile.cs index 0817d3bd6..e2f24d3d6 100644 --- a/src/KubernetesClientConfiguration.ConfigFile.cs +++ b/src/KubernetesClientConfiguration.ConfigFile.cs @@ -40,10 +40,25 @@ public KubernetesClientConfiguration(FileInfo kubeconfig = null, string currentC /// /// kube api server endpoint /// kubeconfig filepath - /// public static KubernetesClientConfiguration BuildConfigFromConfigFile(string masterUrl = null, string kubeconfigPath = null) { - var k8SConfig = LoadKubeConfig(new FileInfo(kubeconfigPath ?? KubeConfigDefaultLocation)); + return BuildConfigFromConfigFile(new FileInfo(kubeconfigPath ?? KubeConfigDefaultLocation), null, masterUrl); + } + + /// + /// + /// + /// Fileinfo of the kubeconfig, cannot be null + /// override the context in config file, set null if do not want to override + /// overrider kube api server endpoint, set null if do not want to override + public static KubernetesClientConfiguration BuildConfigFromConfigFile(FileInfo kubeconfig, string currentContext = null, string masterUrl = null) + { + if (kubeconfig == null) + { + throw new NullReferenceException(nameof(kubeconfig)); + } + + var k8SConfig = LoadKubeConfig(kubeconfig); var k8SConfiguration = new KubernetesClientConfiguration(); k8SConfiguration.Initialize(k8SConfig); @@ -53,6 +68,7 @@ public static KubernetesClientConfiguration BuildConfigFromConfigFile(string mas } return k8SConfiguration; } + /// /// Validates and Intializes Client Configuration diff --git a/src/KubernetesClientConfiguration.cs b/src/KubernetesClientConfiguration.cs index 441759e88..df4e9225b 100644 --- a/src/KubernetesClientConfiguration.cs +++ b/src/KubernetesClientConfiguration.cs @@ -14,45 +14,44 @@ namespace k8s /// public partial class KubernetesClientConfiguration { - private KubernetesClientConfiguration() + public KubernetesClientConfiguration() { - } /// /// Gets Host /// - public string Host { get; private set; } + public string Host { get; set; } /// /// Gets SslCaCert /// - public X509Certificate2 SslCaCert { get; private set; } + public X509Certificate2 SslCaCert { get; set; } /// /// Gets ClientCertificateData /// - public string ClientCertificateData { get; private set; } + public string ClientCertificateData { get; set; } /// /// Gets ClientCertificate Key /// - public string ClientCertificateKey { get; private set; } + public string ClientCertificateKey { get; set; } /// /// Gets ClientCertificate filename /// - public string ClientCertificate { get; private set; } + public string ClientCertificate { get; set; } /// /// Gets ClientCertificate Key filename /// - public string ClientKey { get; private set; } + public string ClientKey { get; set; } /// /// Gets a value indicating whether to skip ssl server cert validation /// - public bool SkipTlsVerify { get; private set; } + public bool SkipTlsVerify { get; set; } /// /// Gets or sets the HTTP user agent. diff --git a/src/KubernetesClientCredentials.cs b/src/KubernetesClientCredentials.cs deleted file mode 100644 index 8fa6073e2..000000000 --- a/src/KubernetesClientCredentials.cs +++ /dev/null @@ -1,70 +0,0 @@ -namespace k8s -{ - using System; - using System.Globalization; - using System.Net.Http; - using System.Net.Http.Headers; - using System.Threading; - using System.Threading.Tasks; - using k8s.Exceptions; - using Microsoft.Rest; - - /// - /// Class to set the Kubernetes Client Credentials for token based auth - /// - public class KubernetesClientCredentials : ServiceClientCredentials - { - public KubernetesClientCredentials(string token) - { - if (string.IsNullOrWhiteSpace(token)) - { - throw new ArgumentNullException(nameof(token)); - } - - this.AuthenticationToken = token; - this.AuthenticationScheme = "Bearer"; - } - - public KubernetesClientCredentials(string userName, string password) - { - if (string.IsNullOrWhiteSpace(userName)) - { - throw new ArgumentNullException(nameof(userName)); - } - - if (string.IsNullOrWhiteSpace(password)) - { - throw new ArgumentNullException(nameof(password)); - } - - this.AuthenticationToken = Utils.Base64Encode(string.Format(CultureInfo.InvariantCulture, "{0}:{1}", userName, password)); - this.AuthenticationScheme = "Basic"; - } - - private string AuthenticationToken { get; } - - private string AuthenticationScheme { get; } - - public override async Task ProcessHttpRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (request == null) - { - throw new ArgumentNullException(nameof(request)); - } - - if (string.IsNullOrWhiteSpace(this.AuthenticationScheme)) - { - throw new KubernetesClientException("AuthenticationScheme cannot be null. Please set the AuthenticationScheme to Basic/Bearer"); - } - - if (string.IsNullOrWhiteSpace(this.AuthenticationToken)) - { - throw new KubernetesClientException("AuthenticationToken cannot be null. Please set the authentication token"); - } - - request.Headers.Authorization = new AuthenticationHeaderValue(this.AuthenticationScheme, this.AuthenticationToken); - - await base.ProcessHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - } - } -} \ No newline at end of file diff --git a/tests/KubernetesClientCredentialsTests.cs b/tests/KubernetesClientCredentialsTests.cs deleted file mode 100644 index 1f36728a9..000000000 --- a/tests/KubernetesClientCredentialsTests.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; -using Xunit; -using k8s; -using System.IO; - -namespace k8s.Tests -{ - public class KubernetesClientCredentialsTests - { - /// - /// Checks that a ArgumentNullException is thrown when trying to create a KubernetesClientCredentials with null token - /// - [Fact] - public void TokenNull() - { - Assert.Throws(() => new KubernetesClientCredentials(null)); - } - - /// - /// Checks that a ArgumentNullException is thrown when trying to create a KubernetesClientCredentials with null username - /// - [Fact] - public void UsernameNull() - { - Assert.Throws(() => new KubernetesClientCredentials(null,"password")); - } - - /// - /// Checks that a ArgumentNullException is thrown when trying to create a KubernetesClientCredentials with null password - /// - [Fact] - public void PasswordNull() - { - Assert.Throws(() => new KubernetesClientCredentials("username", null)); - } - - /// - /// Checks that the Token is set with no exceptions - /// - [Fact] - public void ValidTokenIsSet() - { - var token = "mytoken"; - var credentials = new KubernetesClientCredentials(token); - Assert.NotNull(credentials); - } - - /// - /// Checks that the Username and Password is set with no exceptions - /// - [Fact] - public void ValidUserPasswordIsSet() - { - var username = "myuser"; - var password = "mypassword"; - var credentials = new KubernetesClientCredentials(username, password); - Assert.NotNull(credentials); - } - } -} \ No newline at end of file From 7096b1560440e8dc9f4cd42b6de0f55bf5193f1a Mon Sep 17 00:00:00 2001 From: Boshi Lian Date: Sat, 21 Oct 2017 21:37:38 +0800 Subject: [PATCH 3/4] extract path to shared const --- src/KubernetesClientConfiguration.InCluster.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/KubernetesClientConfiguration.InCluster.cs b/src/KubernetesClientConfiguration.InCluster.cs index da70d5f81..d2a61895c 100644 --- a/src/KubernetesClientConfiguration.InCluster.cs +++ b/src/KubernetesClientConfiguration.InCluster.cs @@ -1,14 +1,14 @@ using System; using System.IO; -using System.Security.Cryptography.X509Certificates; using k8s.Exceptions; namespace k8s { public partial class KubernetesClientConfiguration { - private const string ServiceAccountTokenKey = "token"; - private const string ServiceAccountRootCAKey = "ca.crt"; + private const string ServiceaccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/"; + private const string ServiceAccountTokenKeyFileName = "token"; + private const string ServiceAccountRootCAKeyFileName = "ca.crt"; public static KubernetesClientConfiguration InClusterConfig() { @@ -21,8 +21,8 @@ public static KubernetesClientConfiguration InClusterConfig() "unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined"); } - var token = File.ReadAllText("/var/run/secrets/kubernetes.io/serviceaccount/" + ServiceAccountTokenKey); - var rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/" + ServiceAccountRootCAKey; + var token = File.ReadAllText(Path.Combine(ServiceaccountPath, ServiceAccountTokenKeyFileName)); + var rootCAFile = Path.Combine(ServiceaccountPath, ServiceAccountRootCAKeyFileName); return new KubernetesClientConfiguration { From 8c93486aa2b693bb428a8ac9b86f26aba426fcca Mon Sep 17 00:00:00 2001 From: Boshi Lian Date: Sat, 21 Oct 2017 21:28:08 +0800 Subject: [PATCH 4/4] introduce auth test using asp.net core --- tests/AuthTests.cs | 316 ++++++++++++++++++++++++++++ tests/Mock/MockKubeApiServer.cs | 54 +++++ tests/assets/apiserver-pfx-data.txt | 1 + tests/tests.csproj | 3 +- 4 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 tests/AuthTests.cs create mode 100644 tests/Mock/MockKubeApiServer.cs create mode 100644 tests/assets/apiserver-pfx-data.txt diff --git a/tests/AuthTests.cs b/tests/AuthTests.cs new file mode 100644 index 000000000..62115675d --- /dev/null +++ b/tests/AuthTests.cs @@ -0,0 +1,316 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography.X509Certificates; +using k8s.Models; +using k8s.Tests.Mock; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Https; +using Microsoft.Rest; +using Xunit; + +namespace k8s.Tests +{ + public class AuthTests + { + private static HttpOperationResponse ExecuteListPods(IKubernetes client) + { + return client.ListNamespacedPodWithHttpMessagesAsync("default").Result; + } + + [Fact] + public void TestAnonymous() + { + using (var server = new MockKubeApiServer()) + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + + var listTask = ExecuteListPods(client); + + Assert.True(listTask.Response.IsSuccessStatusCode); + Assert.Equal(1, listTask.Body.Items.Count); + } + + using (var server = new MockKubeApiServer(cxt => + { + cxt.Response.StatusCode = (int) HttpStatusCode.Unauthorized; + return false; + })) + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + } + + [Fact] + public void TestBasicAuth() + { + const string testName = "test_name"; + const string testPassword = "test_password"; + + using (var server = new MockKubeApiServer(cxt => + { + var header = cxt.Request.Headers["Authorization"].FirstOrDefault(); + + var expect = new AuthenticationHeaderValue("Basic", Utils.Base64Encode($"{testName}:{testPassword}")) + .ToString(); + + if (header != expect) + { + cxt.Response.StatusCode = (int) HttpStatusCode.Unauthorized; + return false; + } + + return true; + })) + { + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = testName, + Password = testPassword + }); + + var listTask = ExecuteListPods(client); + Assert.True(listTask.Response.IsSuccessStatusCode); + Assert.Equal(1, listTask.Body.Items.Count); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = "wrong name", + Password = testPassword + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = testName, + Password = "wrong password" + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = "both wrong", + Password = "wrong password" + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = "xx" + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + } + } + + [Fact] + public void TestCert() + { + var serverCertificateData = File.ReadAllText("assets/apiserver-pfx-data.txt"); + + var clientCertificateKeyData = File.ReadAllText("assets/client-key-data.txt"); + var clientCertificateData = File.ReadAllText("assets/client-certificate-data.txt"); + + var serverCertificate = new X509Certificate2(Convert.FromBase64String(serverCertificateData)); + var clientCertificate = new X509Certificate2(Convert.FromBase64String(clientCertificateData)); + + var clientCertificateValidationCalled = false; + + using (var server = new MockKubeApiServer(listenConfigure: options => + { + options.UseHttps(new HttpsConnectionAdapterOptions + { + ServerCertificate = serverCertificate, + ClientCertificateMode = ClientCertificateMode.RequireCertificate, + ClientCertificateValidation = (certificate, chain, valid) => + { + clientCertificateValidationCalled = true; + return clientCertificate.Equals(certificate); + } + }); + })) + { + { + clientCertificateValidationCalled = false; + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + ClientCertificateData = clientCertificateData, + ClientCertificateKey = clientCertificateKeyData, + SslCaCert = serverCertificate, + SkipTlsVerify = false + }); + + var listTask = ExecuteListPods(client); + + Assert.True(clientCertificateValidationCalled); + Assert.True(listTask.Response.IsSuccessStatusCode); + Assert.Equal(1, listTask.Body.Items.Count); + } + + { + clientCertificateValidationCalled = false; + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + ClientCertificateData = clientCertificateData, + ClientCertificateKey = clientCertificateKeyData, + SkipTlsVerify = true + }); + + var listTask = ExecuteListPods(client); + + Assert.True(clientCertificateValidationCalled); + Assert.True(listTask.Response.IsSuccessStatusCode); + Assert.Equal(1, listTask.Body.Items.Count); + } + + { + clientCertificateValidationCalled = false; + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + ClientCertificate = "assets/client.crt", // TODO amazoning why client.crt != client-data.txt + ClientKey = "assets/client.key", // TODO bad naming param + SkipTlsVerify = true + }); + + Assert.ThrowsAny(() => ExecuteListPods(client)); + Assert.True(clientCertificateValidationCalled); + } + + { + clientCertificateValidationCalled = false; + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + SkipTlsVerify = true + }); + + Assert.ThrowsAny(() => ExecuteListPods(client)); + Assert.False(clientCertificateValidationCalled); + } + } + } + + [Fact] + public void TestToken() + { + const string token = "testingtoken"; + + using (var server = new MockKubeApiServer(cxt => + { + var header = cxt.Request.Headers["Authorization"].FirstOrDefault(); + + var expect = new AuthenticationHeaderValue("Bearer", token).ToString(); + + if (header != expect) + { + cxt.Response.StatusCode = (int) HttpStatusCode.Unauthorized; + return false; + } + + return true; + })) + { + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + AccessToken = token + }); + + var listTask = ExecuteListPods(client); + Assert.True(listTask.Response.IsSuccessStatusCode); + Assert.Equal(1, listTask.Body.Items.Count); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + AccessToken = "wrong token" + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString(), + Username = "wrong name", + Password = "same password" + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + + { + var client = new Kubernetes(new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + + var listTask = ExecuteListPods(client); + + Assert.Equal(HttpStatusCode.Unauthorized, listTask.Response.StatusCode); + } + } + } + } +} \ No newline at end of file diff --git a/tests/Mock/MockKubeApiServer.cs b/tests/Mock/MockKubeApiServer.cs new file mode 100644 index 000000000..aa0831037 --- /dev/null +++ b/tests/Mock/MockKubeApiServer.cs @@ -0,0 +1,54 @@ +using System; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core; + +namespace k8s.Tests.Mock +{ + public class MockKubeApiServer : IDisposable + { + // paste from minikube /api/v1/namespaces/default/pods + public const string MockPodResponse = + "{\r\n \"kind\": \"PodList\",\r\n \"apiVersion\": \"v1\",\r\n \"metadata\": {\r\n \"selfLink\": \"/api/v1/namespaces/default/pods\",\r\n \"resourceVersion\": \"1762810\"\r\n },\r\n \"items\": [\r\n {\r\n \"metadata\": {\r\n \"name\": \"nginx-1493591563-xb2v4\",\r\n \"generateName\": \"nginx-1493591563-\",\r\n \"namespace\": \"default\",\r\n \"selfLink\": \"/api/v1/namespaces/default/pods/nginx-1493591563-xb2v4\",\r\n \"uid\": \"ac1abb94-9c58-11e7-aaf5-00155d744505\",\r\n \"resourceVersion\": \"1737928\",\r\n \"creationTimestamp\": \"2017-09-18T10:03:51Z\",\r\n \"labels\": {\r\n \"app\": \"nginx\",\r\n \"pod-template-hash\": \"1493591563\"\r\n },\r\n \"annotations\": {\r\n \"kubernetes.io/created-by\": \"{\\\"kind\\\":\\\"SerializedReference\\\",\\\"apiVersion\\\":\\\"v1\\\",\\\"reference\\\":{\\\"kind\\\":\\\"ReplicaSet\\\",\\\"namespace\\\":\\\"default\\\",\\\"name\\\":\\\"nginx-1493591563\\\",\\\"uid\\\":\\\"ac013b63-9c58-11e7-aaf5-00155d744505\\\",\\\"apiVersion\\\":\\\"extensions\\\",\\\"resourceVersion\\\":\\\"5306\\\"}}\\n\"\r\n },\r\n \"ownerReferences\": [\r\n {\r\n \"apiVersion\": \"extensions/v1beta1\",\r\n \"kind\": \"ReplicaSet\",\r\n \"name\": \"nginx-1493591563\",\r\n \"uid\": \"ac013b63-9c58-11e7-aaf5-00155d744505\",\r\n \"controller\": true,\r\n \"blockOwnerDeletion\": true\r\n }\r\n ]\r\n },\r\n \"spec\": {\r\n \"volumes\": [\r\n {\r\n \"name\": \"default-token-3zzcj\",\r\n \"secret\": {\r\n \"secretName\": \"default-token-3zzcj\",\r\n \"defaultMode\": 420\r\n }\r\n }\r\n ],\r\n \"containers\": [\r\n {\r\n \"name\": \"nginx\",\r\n \"image\": \"nginx\",\r\n \"resources\": {},\r\n \"volumeMounts\": [\r\n {\r\n \"name\": \"default-token-3zzcj\",\r\n \"readOnly\": true,\r\n \"mountPath\": \"/var/run/secrets/kubernetes.io/serviceaccount\"\r\n }\r\n ],\r\n \"terminationMessagePath\": \"/dev/termination-log\",\r\n \"terminationMessagePolicy\": \"File\",\r\n \"imagePullPolicy\": \"Always\"\r\n }\r\n ],\r\n \"restartPolicy\": \"Always\",\r\n \"terminationGracePeriodSeconds\": 30,\r\n \"dnsPolicy\": \"ClusterFirst\",\r\n \"serviceAccountName\": \"default\",\r\n \"serviceAccount\": \"default\",\r\n \"nodeName\": \"ubuntu\",\r\n \"securityContext\": {},\r\n \"schedulerName\": \"default-scheduler\"\r\n },\r\n \"status\": {\r\n \"phase\": \"Running\",\r\n \"conditions\": [\r\n {\r\n \"type\": \"Initialized\",\r\n \"status\": \"True\",\r\n \"lastProbeTime\": null,\r\n \"lastTransitionTime\": \"2017-09-18T10:03:51Z\"\r\n },\r\n {\r\n \"type\": \"Ready\",\r\n \"status\": \"True\",\r\n \"lastProbeTime\": null,\r\n \"lastTransitionTime\": \"2017-10-12T07:09:21Z\"\r\n },\r\n {\r\n \"type\": \"PodScheduled\",\r\n \"status\": \"True\",\r\n \"lastProbeTime\": null,\r\n \"lastTransitionTime\": \"2017-09-18T10:03:51Z\"\r\n }\r\n ],\r\n \"hostIP\": \"192.168.188.42\",\r\n \"podIP\": \"172.17.0.5\",\r\n \"startTime\": \"2017-09-18T10:03:51Z\",\r\n \"containerStatuses\": [\r\n {\r\n \"name\": \"nginx\",\r\n \"state\": {\r\n \"running\": {\r\n \"startedAt\": \"2017-10-12T07:09:20Z\"\r\n }\r\n },\r\n \"lastState\": {\r\n \"terminated\": {\r\n \"exitCode\": 0,\r\n \"reason\": \"Completed\",\r\n \"startedAt\": \"2017-10-10T21:35:51Z\",\r\n \"finishedAt\": \"2017-10-12T07:07:37Z\",\r\n \"containerID\": \"docker://94df3f3965807421ad6dc76618e00b76cb15d024919c4946f3eb46a92659c62a\"\r\n }\r\n },\r\n \"ready\": true,\r\n \"restartCount\": 7,\r\n \"image\": \"nginx:latest\",\r\n \"imageID\": \"docker-pullable://nginx@sha256:004ac1d5e791e705f12a17c80d7bb1e8f7f01aa7dca7deee6e65a03465392072\",\r\n \"containerID\": \"docker://fa11bdd48c9b7d3a6c4c3f9b6d7319743c3455ab8d00c57d59c083b319b88194\"\r\n }\r\n ],\r\n \"qosClass\": \"BestEffort\"\r\n }\r\n }\r\n ]\r\n}" + ; + + private readonly IWebHost _webHost; + + public MockKubeApiServer(Func shouldNext = null, Action listenConfigure = null, + string resp = MockPodResponse) + { + shouldNext = shouldNext ?? (_ => true); + listenConfigure = listenConfigure ?? (_ => { }); + + _webHost = WebHost.CreateDefaultBuilder() + .Configure(app => app.Run(httpContext => + { + if (shouldNext(httpContext)) + { + httpContext.Response.WriteAsync(resp); + } + + return Task.Delay(0); + })) + .UseKestrel(options => { options.Listen(IPAddress.Loopback, 0, listenConfigure); }) + .Build(); + + _webHost.Start(); + } + + public Uri Uri => _webHost.ServerFeatures.Get().Addresses + .Select(a => new Uri(a)).First(); + + public void Dispose() + { + _webHost.StopAsync(); + _webHost.WaitForShutdown(); + } + } +} \ No newline at end of file diff --git a/tests/assets/apiserver-pfx-data.txt b/tests/assets/apiserver-pfx-data.txt new file mode 100644 index 000000000..97bc6e18b --- /dev/null +++ b/tests/assets/apiserver-pfx-data.txt @@ -0,0 +1 @@ +MIIJwQIBAzCCCYcGCSqGSIb3DQEHAaCCCXgEggl0MIIJcDCCBCcGCSqGSIb3DQEHBqCCBBgwggQUAgEAMIIEDQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQI4bbBAeH9ZRUCAggAgIID4N1CiG8n9Oc6+68ZYhPdpZCl3Lo2KxQLsuiY9raGi6qiwboWIMwepTGGsO1uLJqSGIvNcPqPfhAotDpwgSNexE2LYeUYvuYLFy2o/NuS6w1DaKUneikT0h/tjaVFBTzFmhJN4QsaZo3JhX+WfVdv8BLCNISsVAFVc3Aisi8+uuYMK++i+gDccAJA1MbVFJsUtl82d3ZIQEQDtOG4qLXn2yDGf/lKP15H3bYAw6H+6CwVyEAahwTDDDX8q6u/DKvsVyHhWf7BPNPm1/v6b1uLX0Zi9zj4QxLTklrIhNp6a/FrcOmejf8eJFIhDLdemTrmAd3b1KaLsmVZZUPLzfoub18TihzKVI0Wqk5N6kvRERFUNCfFcXy2v2Re0U/3haKubG8nHrFErsXTHuycsmj+BjFPY3LlffkeMi1WCU1CpIaAq07/bpUYAKwmgY21v7c1jss2Yob1vH+4LPZLVp/drFCa/INcpYez4CnxIElarkjfdnOtqEfQLEwWJCdCbxw3pCCnLWC9vM89mkoYXYohqYx5xEDDdHphyGNKwSnxlf87l1HulTOJ55GwczFlL+GqEp9GM7P9AbZWcLvIPje7TsyIS+DNPflA0eORzoSC0Vuy1e3PGwZsv6eulAOKfXbqn6OSSMwwyBCSw1meb8+Za3B2pNb7mZ5bnbjNId7mUOwjV1x2gLb2ROKd7YRdAelLdAzMudcbY/wqkREzQ5RsEwiEK9EkMY7Y+VjYZJHSoCECtuylx8yCpnPJ95BIGGsnmUDXioYc1fQP1+z8jsSe2uzo/6SS5DSRsNYG8xivrM0tKHipKPvMD895I3mY+OPps6b8wTtUyYNG4Lz+tq8ZkKnWvn7QWDphAzqvBIlVUHOsym2ft4/xO7fgNUL/AiLg2qcNcrzj6tQEoL4Qq9P/zlajhW24VNfYyFVa/BdTVzKdc/YQ8puaVPzB26gAXJCE6b75ytgH+dq08XjVuFG/G97n4uMxsxxI8DP0ZUWpCgzdF0SvxhFfB+ZYp+ELK1YbVrqcThMZuSH6r/KCU1N3Z6GFqpHMK9ky7aC/7HEJ84Ed1vvBncOMGRyfKbnn3f4aRuxkt6/KJ9+riIyUW0N+fG0kFuV0ADTmIvy3a1tu0is7E8xNiujsuO1/KnbVLyf0OvoSkOM7A7UWGuNnQkPTDmgq+hJEV7Y/Zgz7nbc376slisTMq4kSgElWZT1fzLwvrC5DTsnxw1ypqLeBQWx1TvF7kZaPAPEnaHOu1EcSPdDA6UkIGJpU+XpNextC//u5De53Ro54ViYIF/h3Em4+l7CPkk0KVjf+rWlFkIbrXSVAMIIFQQYJKoZIhvcNAQcBoIIFMgSCBS4wggUqMIIFJgYLKoZIhvcNAQwKAQKgggTuMIIE6jAcBgoqhkiG9w0BDAEDMA4ECCUYjeFdLaumAgIIAASCBMi1GALxlEly1sJJ5JShpXwMc8Xw5lHfBbYvovJ8AVZeVjAtea+oaV17yD8fgCfhoPaD7sG+QC6TS+Qx8Ja4N03fjRGv+FeAlKbT/WtwjXpT1iIXETRtfE6z+uKbNun/XTD90+blYyAETVxKtyvWBxm3RA4e67RiEKYhGHl6n2DKjn023iasnShZ2C1HCI3ewD402syT2r5BI+faTOUe4OitHNYMLg46jPQzKQ29qkl1SmPbS8wMF4/fh1FiuEsF/W/6TYFdZ022dXxOPDM1wmpISo3+De+wqEAlETowPrkouYs3c8j1rWpNBpILXgCrFsZlvUhMpoRDs6gzUgYWnwBSZI3PwfbDfYfZ8hjSUoeiaHxpHp1dQHM4i9l4+r7FsV9fJErkgbe+/6M6YpPAs749yqOUHmMhlvu4mpPKBPbivF6o8rrTdPZahuqHWqe8l5fFxmw/PBiOnUhv1pT3VF8tFR0c9+q3KnzCgz+h47VlC+ZJyAdGMhFORaaijVnoaYvNsvmambpcll9+t/9qHB4r9i7oGLxCpUFEXa2nvxJStZdKt9vXO04+TrEwFWESyuB4zlmzUxWJrKcm/t44sMIOhpmf5lEcVfC8J9zcOzP9YPaH8Uivdg25tAwGkUVjVss3U9WNc+RHhexfWppiRtrzRgx9ZTaJjnOkREHijrIMJ3THi75kvTA4dk5MprdTGq7R8NgXt6ZwuPkeeuDd6c+d9nniMs17L8aX+cKq+ih++d9PzBlp7EB/reSL4IAxYdwwlJKmGHqtIbmoOifaVNDOaz7fduzo4EqQ//dw6vEkORb+FJ9t8p/bXHd+AmcAVbk9xD8U/TB/WQUyAbQQuh1+s7TfYP6D0kR+0FRhQiw2L2zJS29XG7qM37As27O0sCcm7DHMEG08i2SYMgssjsCJqlyvQnnOizuKL6+B5czjgA+oMAuRk4BF3Mjz9KpB3/EhFkvdl2Qzf3V2IghUdguPE2SgKZaZreKFQCzCFkbGYxZjDkyxI4Bgt3eZ605lW89RmW5ievRP2aG+8O5TrDL4fMn5PYDS9Y5uZlMfPI5kSjgg1Skv3ER/KhdrJAvDfH17WlkBa7E7gZGbNwk89VcB6EgqftF6kRjF+VtpmZFBnmcmxXORrzgnQ6mp6M3BXoAxrf7BH5IOxZL6UVslz1MLpjBteHxDfyPs24OgLLQcd82uWj5CkEZ2WKgM0eO5QP3V5nUnjmwehmib/sSB1e7w2kbqQDKiMcWmr0Y+XyHd/xaGqXo2ou8xe1n9atwg4iGeYd1zf8OLQZrOdNfA9mPu/84sMooAOdr632ewUNkpOIOuvn5wkz4BiTo7lMBUveFr7YxPqziGOq1wk0fGUWC5s/ykB/Dcj/dSJJ0ui/l5wuhUMkog+mG4XMgQStVHfLpeNhMm8Sz6xasi0dKto+omA96QHmcylbprSDpFm+iaMRSu0K9h95l2p9FMjqaffDG6ydDnYQNgOo7cLAqHWvEcZyeMEU8k/+BWANjRs+zQaWFTaHqZNObwLVlfVJORQj2NrB/o6rnlqmW/MG9CLFBml679Rsj5Uv8Zb0SgL0A+ui/dsyAYIb8NUiL/WOmsWx999ztdpKA9BkTOijHPAfkAYndSqcIoBBMxJTAjBgkqhkiG9w0BCRUxFgQUleDEgeLMxqDSYaW50lkmHFwjYv8wMTAhMAkGBSsOAwIaBQAEFBtlW3NKzWYbFpQosWzQZqthWnriBAjoLbUohvvqcAICCAA= \ No newline at end of file diff --git a/tests/tests.csproj b/tests/tests.csproj index e9c9306f5..7726e9322 100755 --- a/tests/tests.csproj +++ b/tests/tests.csproj @@ -5,10 +5,11 @@ k8s.tests + - +