Feat: local Streamlit setup (#90)

Author: thekaveman

.devcontainer/Dockerfile

@@ -1,8 +1,12 @@
 FROM caltrans/pems:app
 
 COPY . .
+
 # install devcontainer requirements
 RUN pip install -e .[dev,test]
 
 # install docs requirements
 RUN pip install --no-cache-dir -r docs/requirements.txt
+
+# install streamlit requirements
+RUN pip install --no-cache-dir -r streamlit_app/requirements.txt

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
   "name": "caltrans/pems",
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
-  "forwardPorts": ["docs:8000", "kibana:5601"],
+  "forwardPorts": ["docs:8000"],
   "workspaceFolder": "/caltrans/app",
   "postStartCommand": ["/bin/bash", "bin/reset_db.sh"],
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],

.env.sample

@@ -8,29 +8,7 @@ DJANGO_DB_RESET=true
 DJANGO_STORAGE_DIR=.
 DJANGO_DB_FILE=django.db
 
-# uncomment to start the elasticstack services with compose
-# COMPOSE_PROFILES=elasticstack
-
-# Version of Elastic products
-ELASTIC_STACK_VERSION=8.16.1
-
-# Set to 'basic' or 'trial' to automatically start the 30-day trial
-ELASTIC_LICENSE=basic
-
-# SAMPLE Predefined Key only to be used in POC environments
-ELASTIC_ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
-
-# Set the cluster name
-ELASTIC_CLUSTER=eslocal
-
-# Increase or decrease based on the available host memory (in bytes)
-ELASTIC_MEM_LIMIT=2147483648
-
-# Password for the 'elastic' user (at least 6 characters)
-ELASTIC_PASSWORD=elastic
-
-# Password for the 'kibana_system' user (at least 6 characters)
-KIBANA_PASSWORD=kibana
-
-# Increase or decrease based on the available host memory (in bytes)
-KIBANA_MEM_LIMIT=1073741824
+# Streamlit
+STREAMLIT_LOCAL_PORT=8501
+# options: hidden, sidebar
+STREAMLIT_NAV=hidden

.github/workflows/tests-pytest.yml

@@ -29,7 +29,9 @@ jobs:
           cache-dependency-path: "**/pyproject.toml"
 
       - name: Install Python dependencies
-        run: pip install -e .[test]
+        run: |
+          pip install -e .[test]
+          pip install -r streamlit_app/requirements.txt
 
       - name: Run setup
         run: ./bin/init.sh

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[browser]
+gatherUsageStats = false
+
+[theme]
+base = "light"

.vscode/launch.json

@@ -27,6 +27,30 @@
         "DJANGO_DEBUG": "false",
         "DJANGO_STATICFILES_STORAGE": "django.contrib.staticfiles.storage.StaticFilesStorage"
       }
+    },
+    {
+      "name": "Streamlit, yes sidebar",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "streamlit",
+      "args": ["run", "streamlit_app/main.py"],
+      "env": {
+        "PYTHONBUFFERED": "1",
+        "PYTHONWARNINGS": "default",
+        "STREAMLIT_NAV": "sidebar"
+      }
+    },
+    {
+      "name": "Streamlit, no sidebar",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "streamlit",
+      "args": ["run", "streamlit_app/main.py"],
+      "env": {
+        "PYTHONBUFFERED": "1",
+        "PYTHONWARNINGS": "default",
+        "STREAMLIT_NAV": "hidden"
+      }
     }
   ]
 }

bin/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eux
+
+docker compose build app
+
+docker compose build dev
+
+docker compose build streamlit

compose.yml

@@ -30,190 +30,13 @@ services:
     volumes:
       - ./:/caltrans/app
 
-  esconfig:
-    profiles: ["elasticstack"]
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
-    user: "0"
-    command: ["bash", "/.local/config/setup.sh"]
-    env_file:
-      - .env
-    volumes:
-      - certs:/usr/share/elasticsearch/config/certs
-      - ./elasticstack/esconfig:/.local/config
-    healthcheck:
-      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
-      interval: 1s
-      timeout: 5s
-      retries: 120
-
-  es01:
-    profiles: ["elasticstack"]
-    depends_on:
-      esconfig:
-        condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
-    labels:
-      co.elastic.logs/module: elasticsearch
-    volumes:
-      - certs:/usr/share/elasticsearch/config/certs
-      - esdata01:/usr/share/elasticsearch/data
-    ports:
-      - "9200"
-    environment:
-      - node.name=es01
-      - cluster.name=${ELASTIC_CLUSTER}
-      - discovery.type=single-node
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - bootstrap.memory_lock=true
-      - xpack.security.enabled=true
-      - xpack.security.http.ssl.enabled=true
-      - xpack.security.http.ssl.key=certs/es01/es01.key
-      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.key=certs/es01/es01.key
-      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
-    mem_limit: ${ELASTIC_MEM_LIMIT}
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-  kibana:
-    profiles: ["elasticstack"]
-    depends_on:
-      es01:
-        condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:${ELASTIC_STACK_VERSION}
-    labels:
-      co.elastic.logs/module: kibana
-    volumes:
-      - certs:/usr/share/kibana/config/certs
-      - kibanadata:/usr/share/kibana/data
-      - ./elasticstack/kibana:/usr/share/kibana/config
-    ports:
-      - "5601"
-    environment:
-      - SERVERNAME=kibana
-      - ELASTICSEARCH_HOSTS=https://es01:9200
-      - ELASTICSEARCH_USERNAME=kibana_system
-      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
-      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
-      - XPACK_SECURITY_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
-      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
-      - XPACK_REPORTING_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
-    mem_limit: ${KIBANA_MEM_LIMIT}
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-  metricbeat01:
-    profiles: ["elasticstack"]
-    depends_on:
-      es01:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    image: caltrans/pems:metricbeat01
-    build:
-      context: .
-      dockerfile: ./elasticstack/metricbeat01/Dockerfile
-      args:
-        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
-    volumes:
-      - certs:/usr/share/metricbeat/certs
-      - metricbeatdata01:/usr/share/metricbeat/data
-      - "/var/run/docker.sock:/var/run/docker.sock:ro"
-      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
-      - "/proc:/hostfs/proc:ro"
-      - "/:/hostfs:ro"
-    environment:
-      - ELASTIC_USER=elastic
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - ELASTIC_HOSTS=https://es01:9200
-      - KIBANA_HOSTS=http://kibana:5601
-      - LOGSTASH_HOSTS=http://logstash01:9600
-    command:
-      - --strict.perms=false
-
-  filebeat01:
-    profiles: ["elasticstack"]
-    depends_on:
-      es01:
-        condition: service_healthy
-    image: caltrans/pems:filebeat01
-    build:
-      context: .
-      dockerfile: ./elasticstack/filebeat01/Dockerfile
-      args:
-        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
-    volumes:
-      - certs:/usr/share/filebeat/certs
-      - filebeatdata01:/usr/share/filebeat/data
-      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
-      - "/var/run/docker.sock:/var/run/docker.sock:ro"
-    environment:
-      - ELASTIC_USER=elastic
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - ELASTIC_HOSTS=https://es01:9200
-      - KIBANA_HOSTS=http://kibana:5601
-      - LOGSTASH_HOSTS=http://logstash01:9600
-    command:
-      - --strict.perms=false
-
-  logstash01:
-    profiles: ["elasticstack"]
-    depends_on:
-      es01:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    image: caltrans/pems:logstash01
+  streamlit:
     build:
       context: .
-      dockerfile: ./elasticstack/logstash01/Dockerfile
-      args:
-        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
-    labels:
-      co.elastic.logs/module: logstash
+      dockerfile: streamlit_app/Dockerfile
+    image: caltrans/pems:streamlit
+    env_file: .env
+    ports:
+      - "${STREAMLIT_LOCAL_PORT:-8501}:8501"
     volumes:
-      - certs:/usr/share/logstash/certs
-      - logstashdata01:/usr/share/logstash/data
-      - ./elasticstack/logstash01:/tmp/logstash01
-    environment:
-      - xpack.monitoring.enabled=false
-      - ELASTIC_USER=elastic
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - ELASTIC_HOSTS=https://es01:9200
-
-volumes:
-  certs:
-    driver: local
-  esdata01:
-    driver: local
-  kibanadata:
-    driver: local
-  metricbeatdata01:
-    driver: local
-  filebeatdata01:
-    driver: local
-  logstashdata01:
-    driver: local
+      - ./:/caltrans/app