Merge pull request #352 from charlieegan3/prometheus-mode

Prometheus Install Modes

Author: simonswine

Gopkg.lock

@@ -79,3 +79,4 @@ overprovisioning
 autoscaling
 preempted
 millicores
+yaml

docs/spelling_wordlist.txt

@@ -156,7 +156,7 @@ Cluster Autoscaler
 Tarmak supports deploying `Cluster Autoscaler
 <https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler>`_ when
 spinning up a Kubernetes cluster. The following `tarmak.yaml` snippet shows how
-you would enable Cluster Autoscaler. 
+you would enable Cluster Autoscaler.
 
 .. code-block:: yaml
 
@@ -323,9 +323,9 @@ Elasticsearch endpoint and the policy that allow shipping to it:
 ::
 
   Apply complete! Resources: 2 added, 0 changed, 0 destroyed.
-  
+
   Outputs:
-  
+
   elasticsearch_endpoint = search-tarmak-logs-xyz.eu-west-1.es.amazonaws.com
   elasticsearch_shipping_policy_arn = arn:aws:iam::1234:policy/tarmak-logs-shipping
 
@@ -441,29 +441,42 @@ allows to override the deployed version:
 Prometheus
 ~~~~~~~~~~
 
-By default Tarmak will deploy a `Prometheus <https://prometheus.io/>`_ and some
-exporters into the ``monitoring`` namespace. Using this config Prometheus could
-be disabled all together:
+By default Tarmak will deploy a `Prometheus <https://prometheus.io/>`_
+installation and some exporters into the ``monitoring`` namespace.
+
+This can be completely disabled with the following cluster configuration:
 
 .. code-block:: yaml
 
   kubernetes:
     prometheus:
       enabled: false
 
-Another possibility would be to use The Tarmak provisioned Prometheus only for
+Another possibility would be to use the Tarmak provisioned Prometheus only for
 scraping exporters on instances that are not part of the Kubernetes cluster.
-Using federation those metrics could then be integrated into an already
-existing Prometheus deployment. Get that behaviour you needs to set the
-configuration like that:
+Using federation, those metrics could then be integrated into an existing
+Prometheus deployment.
+
+To have Prometheus only monitor nodes external to the cluster, use the
+following configuration instead:
 
 .. code-block:: yaml
 
   kubernetes:
     prometheus:
       enabled: true
-      externalScrapeTargetsOnly: true
+      mode: ExternalScrapeTargetsOnly
+
+Finally, you may wish to have Tarmak only install the exporters on the external
+nodes. If this is your desired configuration, then set the following mode in
+the yaml:
 
+.. code-block:: yaml
+
+  kubernetes:
+    prometheus:
+      enabled: true
+      mode: ExternalExportersOnly
 
 API Server
 ~~~~~~~~~~~

docs/user-guide.rst

@@ -38,6 +38,12 @@ const (
 	StateDestroy = "destroy"
 )
 
+const (
+	PrometheusModeFull                      = "Full"
+	PrometheusModeExternalExportersOnly     = "ExternalExportersOnly"
+	PrometheusModeExternalScrapeTargetsOnly = "ExternalScrapeTargetsOnly"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +resource:path=clusters
 
@@ -165,8 +171,8 @@ type ClusterPodSecurityPolicy struct {
 type ClusterKubernetesPrometheus struct {
 	// Enable a cluster internal prometheus deployment, default: true
 	Enabled bool `json:"enabled,omitempty"`
-	// Only deploy/scrape Kubernetes external exporters, default: false
-	ExternalScrapeTargetsOnly bool `json:"externalScrapeTargetsOnly,omitempty"`
+	// Mode defines which components are installed
+	Mode string `json:"mode,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/cluster/v1alpha1/cluster.go

@@ -143,14 +143,13 @@ func kubernetesClusterConfig(conf *clusterv1alpha1.ClusterKubernetes, hieraData
 		}
 	}
 
-	// enabling prometheus if requested, default is to enable prometheus
+	// enable prometheus if set, default: enabled
 	if conf.Prometheus == nil || conf.Prometheus.Enabled {
-		// check if mode external scrape targets only is enabled
-		if conf.Prometheus != nil && conf.Prometheus.ExternalScrapeTargetsOnly {
-			hieraData.variables = append(hieraData.variables, "prometheus::external_scrape_targets_only: true")
-		} else {
-			hieraData.variables = append(hieraData.variables, "prometheus::external_scrape_targets_only: false")
+		mode := clusterv1alpha1.PrometheusModeFull
+		if conf.Prometheus != nil && conf.Prometheus.Mode != "" {
+			mode = conf.Prometheus.Mode
 		}
+		hieraData.variables = append(hieraData.variables, fmt.Sprintf("prometheus::mode: %s", mode))
 		hieraData.classes = append(hieraData.classes, `prometheus`)
 	}
 

pkg/puppet/puppet.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 	"github.com/jetstack/tarmak/pkg/tarmak/instance_pool"
@@ -206,6 +207,13 @@ func (c *Cluster) Validate() (result error) {
 				result = multierror.Append(result, err)
 			}
 		}
+
+		//validate prometheus mode
+		if c.Config().Kubernetes.Prometheus != nil {
+			if err := c.validatePrometheusMode(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
 	}
 
 	return result
@@ -297,6 +305,25 @@ func (c *Cluster) validateAPIServer() (result error) {
 	return result
 }
 
+func (c *Cluster) validatePrometheusMode() error {
+	var result error
+
+	allowedModes := sets.NewString(
+		clusterv1alpha1.PrometheusModeFull,
+		clusterv1alpha1.PrometheusModeExternalScrapeTargetsOnly,
+		clusterv1alpha1.PrometheusModeExternalExportersOnly,
+	)
+
+	modeString := c.Config().Kubernetes.Prometheus.Mode
+	if c.Config().Kubernetes.Prometheus != nil && modeString != "" {
+		if !allowedModes.Has(modeString) {
+			return fmt.Errorf("%s is not a valid Prometheus mode, allowed modes: %s", modeString, allowedModes.List())
+		}
+	}
+
+	return result
+}
+
 // Determine if this Cluster is a cluster or hub, single or multi environment
 func (c *Cluster) Type() string {
 	if c.conf.Type != "" {

pkg/tarmak/cluster/cluster.go

@@ -10,13 +10,11 @@
   $namespace = $::prometheus::namespace
 
   # Setup deployment for blackbox exporter in cluster
-  if $::prometheus::role == 'master' {
-    kubernetes::apply{'blackbox-exporter':
-      manifests => [
-        template('prometheus/prometheus-ns.yaml.erb'),
-        template('prometheus/blackbox-exporter-deployment.yaml.erb'),
-        template('prometheus/blackbox-exporter-svc.yaml.erb'),
-      ],
-    }
+  kubernetes::apply{'blackbox-exporter':
+    manifests => [
+      template('prometheus/prometheus-ns.yaml.erb'),
+      template('prometheus/blackbox-exporter-deployment.yaml.erb'),
+      template('prometheus/blackbox-exporter-svc.yaml.erb'),
+    ],
   }
 }

puppet/modules/prometheus/manifests/blackbox_exporter.pp

@@ -19,16 +19,13 @@
   if $::prometheus::role == 'master' {
     include ::prometheus::server
 
-    $blackbox_exporters = $::prometheus::etcd_cluster.map |$node| { "${node}:${port}" }
-
     prometheus::scrape_config { 'etcd-k8s-main':
       order  =>  140,
       config => {
         'metrics_path'    => '/probe',
         'params'          => { 'module' => ['etcd_k8s_main_proxy'] },
-        'static_configs'  => [{
-          'targets' => $blackbox_exporters,
-          'labels'  => {'role' => 'etcd'},
+        'dns_sd_configs'  => [{
+          'names' => $tarmak::etcd_cluster_exporters,
         }],
         'relabel_configs' => [{
           'source_labels' => [],
@@ -44,9 +41,8 @@
       config => {
         'metrics_path'    => '/probe',
         'params'          => { 'module' => ['etcd_k8s_events_proxy'] },
-        'static_configs'  => [{
-          'targets' => $blackbox_exporters,
-          'labels'  => {'role' => 'etcd'},
+        'dns_sd_configs'  => [{
+          'names' => $tarmak::etcd_cluster_exporters,
         }],
         'relabel_configs' => [{
           'source_labels' => [],
@@ -62,9 +58,8 @@
       config => {
         'metrics_path'    => '/probe',
         'params'          => { 'module' => ['etcd_overlay_proxy'] },
-        'static_configs'  => [{
-          'targets' => $blackbox_exporters,
-          'labels'  => {'role' => 'etcd'},
+        'dns_sd_configs'  => [{
+          'names' => $tarmak::etcd_cluster_exporters,
         }],
         'relabel_configs' => [{
           'source_labels' => [],

puppet/modules/prometheus/manifests/blackbox_exporter_etcd.pp

@@ -2,22 +2,29 @@
   String $systemd_path = '/etc/systemd/system',
   String $namespace = 'monitoring',
   Optional[Enum['etcd','master','worker']] $role = $::prometheus::params::role,
-  $etcd_cluster = $::prometheus::params::etcd_cluster,
+  $etcd_cluster_exporters = $::prometheus::params::etcd_cluster_exporters,
   Integer[1025,65535] $etcd_k8s_main_port = $::prometheus::params::etcd_k8s_main_port,
   Integer[1025,65535] $etcd_k8s_events_port = $::prometheus::params::etcd_k8s_events_port,
   Integer[1024,65535] $etcd_overlay_port = $::prometheus::params::etcd_overlay_port,
-  Boolean $external_scrape_targets_only = false,
+  String $mode = 'Full',
 ) inherits ::prometheus::params
 {
 
   if $role == 'master' {
-    include ::prometheus::server
-    include ::prometheus::blackbox_exporter_etcd
-    include ::prometheus::node_exporter
-    if ! $external_scrape_targets_only {
+    if $mode == 'Full' {
+      include ::prometheus::server
+      include ::prometheus::blackbox_exporter_etcd
+      include ::prometheus::node_exporter
+
       include ::prometheus::kube_state_metrics
       include ::prometheus::blackbox_exporter
     }
+
+    if $mode == 'ExternalScrapeTargetsOnly' {
+      include ::prometheus::server
+      include ::prometheus::blackbox_exporter_etcd
+      include ::prometheus::node_exporter
+    }
   }
 
   if $role == 'etcd' {

puppet/modules/prometheus/manifests/init.pp

@@ -57,19 +57,16 @@
 
 
     # scrape node exporter running on etcd nodes
-    $etcd_node_exporters = $::prometheus::etcd_cluster.map |$node| { "${node}:${port}" }
     prometheus::scrape_config { 'etcd-nodes-exporter':
       order  =>  135,
       config => {
-        'static_configs'  => [{
-          'targets' => $etcd_node_exporters,
-          'labels'  => {'role' => 'etcd'},
+        'dns_sd_configs'  => [{
+          'names' => $tarmak::etcd_cluster_exporters,
         }],
       }
     }
 
-    $external_scrape_targets_only = $::prometheus::external_scrape_targets_only
-    if ! $external_scrape_targets_only {
+    if $::prometheus::mode == 'Full' {
       kubernetes::apply{'node-exporter':
         manifests => [
           template('prometheus/prometheus-ns.yaml.erb'),