From 0e641b7cc0aa59cd2dfb05fafa5c1a744c34fcd9 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 17 Jul 2024 13:30:51 +0200 Subject: [PATCH 1/4] feat: add telemetry dashboard --- config/powersync.yaml | 4 + demos/django/ps-django-backend.yaml | 6 + demos/nodejs/docker-compose.yaml | 3 + demos/supabase/config/powersync.yaml | 4 + demos/supabase/docker-compose.yaml | 3 + .../telemetry-config/collector-config.yaml | 25 + .../grafana/dashboards/example-dashboard.json | 902 ++++++++++++++++++ .../provisioning/dashboards/dashboard.yaml | 11 + .../provisioning/datasources/datasource.yaml | 8 + services/telemetry-config/prometheus.yaml | 11 + services/telemetry.yaml | 34 + 11 files changed, 1011 insertions(+) create mode 100644 services/telemetry-config/collector-config.yaml create mode 100644 services/telemetry-config/grafana/dashboards/example-dashboard.json create mode 100644 services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml create mode 100644 services/telemetry-config/grafana/provisioning/datasources/datasource.yaml create mode 100644 services/telemetry-config/prometheus.yaml create mode 100644 services/telemetry.yaml diff --git a/config/powersync.yaml b/config/powersync.yaml index 64f5bb8..43691fb 100644 --- a/config/powersync.yaml +++ b/config/powersync.yaml @@ -25,6 +25,10 @@ telemetry: # Opt out of reporting anonymized usage metrics to PowerSync telemetry service disable_telemetry_sharing: false + additional_endpoints: + - url: http://telemetry-collector:4318/v1/metrics + # Optionally configure the export interval - defaults to 5 minutes + export_interval_ms: 10000 # 10 seconds # Settings for source database replication replication: diff --git a/demos/django/ps-django-backend.yaml b/demos/django/ps-django-backend.yaml index 380ee1d..6320490 100644 --- a/demos/django/ps-django-backend.yaml +++ b/demos/django/ps-django-backend.yaml @@ -1,3 +1,9 @@ +# Include syntax requires Docker compose > 2.20.3 +# https://docs.docker.com/compose/release-notes/#2203 +include: + # Custom local telemetry service + - path: ../../services/telemetry.yaml + services: # A backend which provides basic authentication and CRUD access to the Postgres DB from the client demo-backend: diff --git a/demos/nodejs/docker-compose.yaml b/demos/nodejs/docker-compose.yaml index f1597bf..bce606c 100644 --- a/demos/nodejs/docker-compose.yaml +++ b/demos/nodejs/docker-compose.yaml @@ -7,6 +7,9 @@ include: # Demo NodeJS backend server and front-end web client - path: ./ps-nodejs-demo.yaml + # Custom local telemetry service + - path: ../../services/telemetry.yaml + services: # Extend PowerSync with Mongo and Postgres healthchecks powersync: diff --git a/demos/supabase/config/powersync.yaml b/demos/supabase/config/powersync.yaml index 326b662..6e360bb 100644 --- a/demos/supabase/config/powersync.yaml +++ b/demos/supabase/config/powersync.yaml @@ -4,6 +4,10 @@ telemetry: # Opt out of reporting anonymized usage metrics to PowerSync telemetry service disable_telemetry_sharing: false + additional_endpoints: + - url: http://telemetry-collector:4318/v1/metrics + # Optionally configure the export interval - defaults to 5 minutes + export_interval_ms: 10000 # 10 seconds # Settings for source database replication replication: diff --git a/demos/supabase/docker-compose.yaml b/demos/supabase/docker-compose.yaml index ecd524d..da0457c 100644 --- a/demos/supabase/docker-compose.yaml +++ b/demos/supabase/docker-compose.yaml @@ -4,6 +4,9 @@ include: # Creates the internal MongoDB replica set - path: ../../services/mongo.yaml + # Custom local telemetry service + - path: ../../services/telemetry.yaml + services: # Extend PowerSync with Mongo healthchecks powersync: diff --git a/services/telemetry-config/collector-config.yaml b/services/telemetry-config/collector-config.yaml new file mode 100644 index 0000000..6c00d68 --- /dev/null +++ b/services/telemetry-config/collector-config.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + protocols: + http: + +exporters: + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 200 + prometheus: + endpoint: "0.0.0.0:7654" + +processors: + batch: + +service: + telemetry: + logs: + level: "debug" + pipelines: + metrics: + receivers: [otlp] + exporters: [debug, prometheus] + processors: [batch] diff --git a/services/telemetry-config/grafana/dashboards/example-dashboard.json b/services/telemetry-config/grafana/dashboards/example-dashboard.json new file mode 100644 index 0000000..ad4c35c --- /dev/null +++ b/services/telemetry-config/grafana/dashboards/example-dashboard.json @@ -0,0 +1,902 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "datasource": "Prometheus", + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_concurrent_connections", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Concurrent Connections", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_rows_replicated_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rows Replicated Total", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_transactions_replicated_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Transactions Replicated Total", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_parameter_storage_size_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Parameter Storage Size (bytes)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_replication_storage_size_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Replication Storage Size (bytes)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_operation_storage_size_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Operation Storage Size (bytes)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "prometheus", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_operations_synced_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Operations Synced Total", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_chunks_replicated_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunks Replicated Total", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "dds0it07m2z28f" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "powersync_data_replicated_bytes_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data Replicated Total (bytes)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Metrics", + "uid": "fds0jamuqczcwa", + "version": 12, + "weekStart": "" +} diff --git a/services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml b/services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml new file mode 100644 index 0000000..0b17189 --- /dev/null +++ b/services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/services/telemetry-config/grafana/provisioning/datasources/datasource.yaml b/services/telemetry-config/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 0000000..86fd346 --- /dev/null +++ b/services/telemetry-config/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/services/telemetry-config/prometheus.yaml b/services/telemetry-config/prometheus.yaml new file mode 100644 index 0000000..27facbe --- /dev/null +++ b/services/telemetry-config/prometheus.yaml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + + - job_name: "otel-collector" + static_configs: + - targets: ["telemetry-collector:7654"] diff --git a/services/telemetry.yaml b/services/telemetry.yaml new file mode 100644 index 0000000..9c0ee6e --- /dev/null +++ b/services/telemetry.yaml @@ -0,0 +1,34 @@ +services: + # Collector + telemetry-collector: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/conf/collector-config.yaml"] + depends_on: + - prometheus + volumes: + - ./telemetry-config/collector-config.yaml:/conf/collector-config.yaml + ports: + - "4318:4318" + - "4317:4317" + + prometheus: + image: prom/prometheus:latest + volumes: + - ./telemetry-config/prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + ports: + - "3121:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana-storage:/var/lib/grafana + - ./telemetry-config/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + - ./telemetry-config/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./telemetry-config/grafana/dashboards:/var/lib/grafana/dashboards + +volumes: + grafana-storage: From a00be93dcb04abe542af870f142a5f9807db7eaa Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 18 Jul 2024 17:30:51 +0200 Subject: [PATCH 2/4] update telemetry --- README.md | 2 +- config/powersync.yaml | 2 ++ demos/django/ps-django-backend.yaml | 6 ------ demos/nodejs/README.md | 18 ++++++++++++++++++ demos/nodejs/docker-compose-telemetry.yaml | 14 ++++++++++++++ demos/nodejs/docker-compose.yaml | 3 --- demos/supabase/docker-compose.yaml | 3 --- services/telemetry/README.md | 3 +++ .../telemetry-config/collector-config.yaml | 0 .../grafana/dashboards/example-dashboard.json | 0 .../provisioning/dashboards/dashboard.yaml | 0 .../provisioning/datasources/datasource.yaml | 0 .../telemetry-config/prometheus.yaml | 0 services/{ => telemetry}/telemetry.yaml | 2 +- 14 files changed, 39 insertions(+), 14 deletions(-) create mode 100644 demos/nodejs/docker-compose-telemetry.yaml create mode 100644 services/telemetry/README.md rename services/{ => telemetry}/telemetry-config/collector-config.yaml (100%) rename services/{ => telemetry}/telemetry-config/grafana/dashboards/example-dashboard.json (100%) rename services/{ => telemetry}/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml (100%) rename services/{ => telemetry}/telemetry-config/grafana/provisioning/datasources/datasource.yaml (100%) rename services/{ => telemetry}/telemetry-config/prometheus.yaml (100%) rename services/{ => telemetry}/telemetry.yaml (94%) diff --git a/README.md b/README.md index fafce23..e2fdaf1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repository contains basic demonstrations in the `demos` folder. - [NodeJS](./demos/nodejs/README.md) - - This can be stared from the repo root with `docker compose -f demos/nodejs/docker-compose.yaml up` + - This can be stared from the repo root with `docker compose -f demos/nodejs/docker-compose.yaml up`. See the readme for additional instructions including telemetry configuration. - [Django](./demos/django/README.md) diff --git a/config/powersync.yaml b/config/powersync.yaml index 43691fb..d9c2e72 100644 --- a/config/powersync.yaml +++ b/config/powersync.yaml @@ -25,6 +25,8 @@ telemetry: # Opt out of reporting anonymized usage metrics to PowerSync telemetry service disable_telemetry_sharing: false + + # This is only available when using the Enterprise edition of the PowerSync self hosted Docker image additional_endpoints: - url: http://telemetry-collector:4318/v1/metrics # Optionally configure the export interval - defaults to 5 minutes diff --git a/demos/django/ps-django-backend.yaml b/demos/django/ps-django-backend.yaml index 6320490..380ee1d 100644 --- a/demos/django/ps-django-backend.yaml +++ b/demos/django/ps-django-backend.yaml @@ -1,9 +1,3 @@ -# Include syntax requires Docker compose > 2.20.3 -# https://docs.docker.com/compose/release-notes/#2203 -include: - # Custom local telemetry service - - path: ../../services/telemetry.yaml - services: # A backend which provides basic authentication and CRUD access to the Postgres DB from the client demo-backend: diff --git a/demos/nodejs/README.md b/demos/nodejs/README.md index 0059e7b..24b37ac 100644 --- a/demos/nodejs/README.md +++ b/demos/nodejs/README.md @@ -22,6 +22,24 @@ docker compose -f demos/nodejs/docker-compose.yaml up The frontend can be accessed at `http://localhost:3030` in a browser. +## Telemetry + +The enterprise edition of the PowerSync self hosting Docker image supports custom telemetry integration. + +This demo can be started with custom telemetry by running + +```bash +docker compose -f docker-compose.yaml -f docker-compose-telemetry.yaml up +``` + +This will additionally start an OpenTelemetry OTLP collector, Prometheus exporter and Grafana service. + +The telemetry can be viewed by accessing `http://localhost:3121` in a browser. + +Login with `admin` for both the username and password. + +The `Telemetry` dashboard contains panels with various metrics. + ## Cleanup The `setup.sql` script only runs on the first initialization of the container. Delete the container and volumes if making changes. diff --git a/demos/nodejs/docker-compose-telemetry.yaml b/demos/nodejs/docker-compose-telemetry.yaml new file mode 100644 index 0000000..ebbcaed --- /dev/null +++ b/demos/nodejs/docker-compose-telemetry.yaml @@ -0,0 +1,14 @@ +# Include syntax requires Docker compose > 2.20.3 +# https://docs.docker.com/compose/release-notes/#2203 +include: + # Creates the internal MongoDB replica set + - path: ../../services/telemetry/telemetry.yaml + +services: + # Include all services from the other file + powersync: + extends: + file: ./docker-compose.yaml + service: powersync + # This image needs to be used for custom telemetry integration + image: container-registry.staging.journeyapps.com/powersync-service:latest diff --git a/demos/nodejs/docker-compose.yaml b/demos/nodejs/docker-compose.yaml index bce606c..f1597bf 100644 --- a/demos/nodejs/docker-compose.yaml +++ b/demos/nodejs/docker-compose.yaml @@ -7,9 +7,6 @@ include: # Demo NodeJS backend server and front-end web client - path: ./ps-nodejs-demo.yaml - # Custom local telemetry service - - path: ../../services/telemetry.yaml - services: # Extend PowerSync with Mongo and Postgres healthchecks powersync: diff --git a/demos/supabase/docker-compose.yaml b/demos/supabase/docker-compose.yaml index da0457c..ecd524d 100644 --- a/demos/supabase/docker-compose.yaml +++ b/demos/supabase/docker-compose.yaml @@ -4,9 +4,6 @@ include: # Creates the internal MongoDB replica set - path: ../../services/mongo.yaml - # Custom local telemetry service - - path: ../../services/telemetry.yaml - services: # Extend PowerSync with Mongo healthchecks powersync: diff --git a/services/telemetry/README.md b/services/telemetry/README.md new file mode 100644 index 0000000..a56c96a --- /dev/null +++ b/services/telemetry/README.md @@ -0,0 +1,3 @@ +# Telemetry + +Note: Custom telemetry integration is only supported in the Enterprise edition of the PowerSync self hosted Docker image. diff --git a/services/telemetry-config/collector-config.yaml b/services/telemetry/telemetry-config/collector-config.yaml similarity index 100% rename from services/telemetry-config/collector-config.yaml rename to services/telemetry/telemetry-config/collector-config.yaml diff --git a/services/telemetry-config/grafana/dashboards/example-dashboard.json b/services/telemetry/telemetry-config/grafana/dashboards/example-dashboard.json similarity index 100% rename from services/telemetry-config/grafana/dashboards/example-dashboard.json rename to services/telemetry/telemetry-config/grafana/dashboards/example-dashboard.json diff --git a/services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml b/services/telemetry/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml similarity index 100% rename from services/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml rename to services/telemetry/telemetry-config/grafana/provisioning/dashboards/dashboard.yaml diff --git a/services/telemetry-config/grafana/provisioning/datasources/datasource.yaml b/services/telemetry/telemetry-config/grafana/provisioning/datasources/datasource.yaml similarity index 100% rename from services/telemetry-config/grafana/provisioning/datasources/datasource.yaml rename to services/telemetry/telemetry-config/grafana/provisioning/datasources/datasource.yaml diff --git a/services/telemetry-config/prometheus.yaml b/services/telemetry/telemetry-config/prometheus.yaml similarity index 100% rename from services/telemetry-config/prometheus.yaml rename to services/telemetry/telemetry-config/prometheus.yaml diff --git a/services/telemetry.yaml b/services/telemetry/telemetry.yaml similarity index 94% rename from services/telemetry.yaml rename to services/telemetry/telemetry.yaml index 9c0ee6e..938c603 100644 --- a/services/telemetry.yaml +++ b/services/telemetry/telemetry.yaml @@ -1,7 +1,7 @@ services: # Collector telemetry-collector: - image: otel/opentelemetry-collector-contrib:latest + image: otel/opentelemetry-collector-contrib:0.92.0 command: ["--config=/conf/collector-config.yaml"] depends_on: - prometheus From 435aae6f4104404fa8cd2cec6c6d45d0576622b6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 18 Jul 2024 17:31:48 +0200 Subject: [PATCH 3/4] readme update --- demos/nodejs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/nodejs/README.md b/demos/nodejs/README.md index 24b37ac..1ccd25a 100644 --- a/demos/nodejs/README.md +++ b/demos/nodejs/README.md @@ -22,7 +22,7 @@ docker compose -f demos/nodejs/docker-compose.yaml up The frontend can be accessed at `http://localhost:3030` in a browser. -## Telemetry +### Telemetry The enterprise edition of the PowerSync self hosting Docker image supports custom telemetry integration. From 3c0925018b4fb262a00b2845d21854e18afb368d Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 18 Jul 2024 17:37:07 +0200 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cd2df..0d7c8bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # PowerSync Self Hosted Example +## v0.5.0 + +- Added custom telemetry integration (Enterprise only feature) to NodeJS demo + ## v0.4.0 - Added Supabase local development demo.