refactor(deployment): Move supporting docker-compose YAML scripts into etc/docker-compose.

junhaoliao · junhaoliao · commit 5f8311d8db0d · 2025-11-19T16:42:06.000-05:00
diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py
@@ -588,7 +588,11 @@ def _set_up_env_for_mcp_server(self) -> EnvVarsDict:
         component_name = MCP_SERVER_COMPONENT_NAME
         if self._clp_config.mcp_server is None:
             logger.info(f"The MCP Server is not configured, skipping {component_name} creation...")
-            return EnvVarsDict()
+            return EnvVarsDict(
+                {
+                    "CLP_MCP_ENABLED": "0",
+                }
+            )
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
@@ -797,8 +801,6 @@ def start(self) -> None:
 
         cmd = ["docker", "compose", "--project-name", self._project_name]
         cmd += ["--file", self._get_docker_file_name()]
-        if self._clp_config.mcp_server is not None:
-            cmd += ["--profile", "mcp"]
         cmd += ["up", "--detach", "--wait"]
         subprocess.run(
             cmd,
@@ -853,7 +855,7 @@ def _get_docker_file_name(self) -> str:
         """
         deployment_type = self._clp_config.get_deployment_type()
         if deployment_type == DeploymentType.BASE:
-            return "docker-compose-base.yaml"
+            return "etc/docker-compose/docker-compose-base.yaml"
         return "docker-compose.yaml"
 
 
diff --git a/components/package-template/src/sbin/admin-tools/archive-manager.sh b/components/package-template/src/sbin/admin-tools/archive-manager.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/../.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.archive_manager \
diff --git a/components/package-template/src/sbin/admin-tools/dataset-manager.sh b/components/package-template/src/sbin/admin-tools/dataset-manager.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/../.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.dataset_manager \
diff --git a/components/package-template/src/sbin/compress-from-s3.sh b/components/package-template/src/sbin/compress-from-s3.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.compress_from_s3 \
diff --git a/components/package-template/src/sbin/compress.sh b/components/package-template/src/sbin/compress.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.compress \
diff --git a/components/package-template/src/sbin/decompress.sh b/components/package-template/src/sbin/decompress.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.decompress \
diff --git a/components/package-template/src/sbin/search.sh b/components/package-template/src/sbin/search.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.search \
diff --git a/components/package-template/src/sbin/start-clp.sh b/components/package-template/src/sbin/start-clp.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.start_clp \
diff --git a/components/package-template/src/sbin/stop-clp.sh b/components/package-template/src/sbin/stop-clp.sh
@@ -10,7 +10,7 @@ common_env_path="$script_dir/.common-env.sh"
 # shellcheck source=.common-env.sh
 source "$common_env_path"
 
-docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" \
+docker compose -f "$CLP_HOME/etc/docker-compose/docker-compose.runtime.yaml" \
     run --rm "${CLP_COMPOSE_RUN_EXTRA_FLAGS[@]}" clp-runtime \
     python3 \
     -m clp_package_utils.scripts.stop_clp \
diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml
@@ -1,80 +1,80 @@
 name: "clp-package"
 
 include:
-  - "docker-compose.utils.yaml"
+  - "etc/docker-compose/docker-compose.utils.yaml"
 
 services:
   database:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "database"
 
   db-table-creator:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "db-table-creator"
 
   queue:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "queue"
 
   redis:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "redis"
 
   results-cache:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "results-cache"
 
   results-cache-indices-creator:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "results-cache-indices-creator"
 
   compression-scheduler:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "compression-scheduler"
 
   compression-worker:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "compression-worker"
 
   webui:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "webui"
 
   garbage-collector:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "garbage-collector"
 
   query-scheduler:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "query-scheduler"
 
   query-worker:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "query-worker"
 
   reducer:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "reducer"
 
   mcp-server:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "mcp-server"
 
   api-server:
     extends:
-      file: "docker-compose-all.yaml"
+      file: "etc/docker-compose/docker-compose-all.yaml"
       service: "api-server"
diff --git a/tools/deployment/package/etc/docker-compose/docker-compose-all.yaml b/tools/deployment/package/etc/docker-compose/docker-compose-all.yaml
@@ -32,12 +32,12 @@ x-healthcheck-defaults: &healthcheck_defaults
 x-volume-definitions:
   clp-config-readonly: &volume_clp_config_readonly
     type: "bind"
-    source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yaml"
+    source: "${CLP_LOGS_DIR_HOST:-../../var/log}/.clp-config.yaml"
     target: "/etc/clp-config.yaml"
     read_only: true
   clp-logs: &volume_clp_logs
     type: "bind"
-    source: "${CLP_LOGS_DIR_HOST:-./var/log}"
+    source: "${CLP_LOGS_DIR_HOST:-../../var/log}"
     target: "/var/log"
 
 services:
@@ -57,14 +57,14 @@ services:
         target: 3306
     volumes:
       - type: "bind"
-        source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}"
+        source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-../../etc/mysql/conf.d/logging.cnf}"
         target: "/etc/mysql/conf.d/logging.cnf"
         read_only: true
       - type: "bind"
-        source: "${CLP_DB_DATA_DIR_HOST:-./var/data/database}"
+        source: "${CLP_DB_DATA_DIR_HOST:-../../var/data/database}"
         target: "/var/lib/mysql"
       - type: "bind"
-        source: "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}"
+        source: "${CLP_DB_LOGS_DIR_HOST:-../../var/log/database}"
         target: "/var/log/mysql"
     healthcheck:
       <<: *healthcheck_defaults
@@ -112,7 +112,7 @@ services:
         target: 5672
     volumes:
       - type: "bind"
-        source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}"
+        source: "${CLP_QUEUE_LOGS_DIR_HOST:-../../var/log/queue}"
         target: "/var/log/rabbitmq"
     healthcheck:
       <<: *healthcheck_defaults
@@ -132,14 +132,14 @@ services:
         target: 6379
     volumes:
       - type: "bind"
-        source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}"
+        source: "${CLP_REDIS_CONF_FILE_HOST:-../../etc/redis/redis.conf}"
         target: "/usr/local/etc/redis/redis.conf"
         read_only: true
       - type: "bind"
-        source: "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}"
+        source: "${CLP_REDIS_DATA_DIR_HOST:-../../var/data/redis}"
         target: "/data"
       - type: "bind"
-        source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}"
+        source: "${CLP_REDIS_LOGS_DIR_HOST:-../../var/log/redis}"
         target: "/var/log/redis"
     command: [
       "redis-server",
@@ -168,14 +168,14 @@ services:
         target: 27017
     volumes:
       - type: "bind"
-        source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}"
+        source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-../../etc/mongo/mongod.conf}"
         target: "/etc/mongo/mongod.conf"
         read_only: true
       - type: "bind"
-        source: "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}"
+        source: "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-../../var/data/results_cache}"
         target: "/data/db"
       - type: "bind"
-        source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}"
+        source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-../../var/log/results_cache}"
         target: "/var/log/mongodb"
     command: [
       "--config", "/etc/mongo/mongod.conf",
@@ -259,7 +259,7 @@ services:
       - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}"
       - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives"
       - type: "bind"
-        source: "${CLP_TMP_DIR_HOST:-./var/tmp}"
+        source: "${CLP_TMP_DIR_HOST:-../../var/tmp}"
         target: "/var/tmp"
     command: [
       "python3",
@@ -295,11 +295,11 @@ services:
       - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/opt/clp/.aws:ro"
       - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams"
       - type: "bind"
-        source: "./var/www/webui/client/settings.json"
+        source: "../../var/www/webui/client/settings.json"
         target: "/opt/clp/var/www/webui/client/settings.json"
         read_only: true
       - type: "bind"
-        source: "./var/www/webui/server/dist/settings.json"
+        source: "../../var/www/webui/server/dist/settings.json"
         target: "/opt/clp/var/www/webui/server/dist/settings.json"
         read_only: true
     depends_on:
@@ -452,7 +452,9 @@ services:
   mcp-server:
     <<: *service_defaults
     hostname: "mcp_server"
-    profiles: ["mcp"]
+    deploy:
+      # Value must be either 0 or 1. Set to 0 to disable the MCP server.
+      replicas: "${CLP_MCP_ENABLED:-0}"
     environment:
       CLP_LOGGING_LEVEL: "${CLP_MCP_LOGGING_LEVEL:-INFO}"
       CLP_LOGS_DIR: "/var/log/mcp_server"
diff --git a/tools/deployment/package/etc/docker-compose/docker-compose-base.yaml b/tools/deployment/package/etc/docker-compose/docker-compose-base.yaml
diff --git a/tools/deployment/package/etc/docker-compose/docker-compose.runtime.yaml b/tools/deployment/package/etc/docker-compose/docker-compose.runtime.yaml
diff --git a/tools/deployment/package/etc/docker-compose/docker-compose.utils.yaml b/tools/deployment/package/etc/docker-compose/docker-compose.utils.yaml
