diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23d6dead8a..f2e2df294b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: - name: Install poetry uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: - version: 1.8.5 + version: 2.1.3 virtualenvs-create: true virtualenvs-in-project: true diff --git a/Dockerfile b/Dockerfile index 6dd0635d66..118ccce197 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ ENV \ PYTHONDONTWRITEBYTECODE=1 \ PIP_DISABLE_PIP_VERSION_CHECK=on \ POETRY_NO_INTERACTION=1 \ - POETRY_VERSION=1.8.5 \ + POETRY_VERSION=2.1.3 \ POETRY_VIRTUALENVS_CREATE=true \ POETRY_CACHE_DIR='/tmp/cache/poetry' \ POETRY_HOME='/home/mitodl/.local' \ diff --git a/main/settings.py b/main/settings.py index f9f6c2d04b..22e79be0cf 100644 --- a/main/settings.py +++ b/main/settings.py @@ -130,7 +130,46 @@ "data_fixtures", "vector_search", "mitol.scim.apps.ScimApp", -) + "health_check", + "health_check.cache", + "health_check.contrib.migrations", + "health_check.contrib.celery_ping", + "health_check.contrib.redis", + "health_check.contrib.db_heartbeat", +) + +HEALTH_CHECK = { + "SUBSETS": { + # The 'startup' subset includes checks that must pass before the application can + # start. + "startup": [ + "MigrationsHealthCheck", # Ensures database migrations are applied. + "CacheBackend", # Verifies the cache backend is operational. + "RedisHealthCheck", # Confirms Redis is reachable and functional. + "DatabaseHeartBeatCheck", # Checks the database connection is alive. + ], + # The 'liveness' subset includes checks to determine if the application is + # running. + "liveness": ["DatabaseHeartBeatCheck"], # Minimal check to ensure the app is + # alive. + # The 'readiness' subset includes checks to determine if the application is + # ready to serve requests. + "readiness": [ + "CacheBackend", # Ensures the cache is ready for use. + "RedisHealthCheck", # Confirms Redis is ready for use. + "DatabaseHeartBeatCheck", # Verifies the database is ready for queries. + ], + # The 'full' subset includes all available health checks for a comprehensive + # status report. + "full": [ + "MigrationsHealthCheck", # Ensures database migrations are applied. + "CacheBackend", # Verifies the cache backend is operational. + "RedisHealthCheck", # Confirms Redis is reachable and functional. + "DatabaseHeartBeatCheck", # Checks the database connection is alive. + "CeleryPingHealthCheck", # Verifies Celery workers are responsive. + ], + } +} if not get_bool("RUN_DATA_MIGRATIONS", default=False): MIGRATION_MODULES = {"data_fixtures": None} @@ -453,7 +492,6 @@ } STATUS_TOKEN = get_string("STATUS_TOKEN", "") -HEALTH_CHECK = ["CELERY", "REDIS", "POSTGRES", "OPEN_SEARCH"] GA_TRACKING_ID = get_string("GA_TRACKING_ID", "") GA_G_TRACKING_ID = get_string("GA_G_TRACKING_ID", "") diff --git a/main/settings_celery.py b/main/settings_celery.py index d854be8741..39b883242c 100644 --- a/main/settings_celery.py +++ b/main/settings_celery.py @@ -17,13 +17,11 @@ DEV_ENV = get_bool("DEV_ENV", False) # noqa: FBT003 USE_CELERY = True -CELERY_BROKER_URL = get_string("CELERY_BROKER_URL", get_string("REDISCLOUD_URL", None)) +REDIS_URL = get_string("REDIS_URL", get_string("REDISCLOUD_URL", None)) +CELERY_BROKER_URL = get_string("CELERY_BROKER_URL", REDIS_URL) +CELERY_RESULT_BACKEND = get_string("CELERY_RESULT_BACKEND", REDIS_URL) CELERY_BEAT_SCHEDULER = RedBeatScheduler -CELERY_REDBEAT_REDIS_URL = CELERY_BROKER_URL redbeat_redis_url = CELERY_BROKER_URL -CELERY_RESULT_BACKEND = get_string( - "CELERY_RESULT_BACKEND", get_string("REDISCLOUD_URL", None) -) CELERY_TASK_ALWAYS_EAGER = get_bool("CELERY_TASK_ALWAYS_EAGER", False) # noqa: FBT003 CELERY_TASK_EAGER_PROPAGATES = get_bool( "CELERY_TASK_EAGER_PROPAGATES", diff --git a/main/urls.py b/main/urls.py index a657f3e9bb..b1e67fd8d9 100644 --- a/main/urls.py +++ b/main/urls.py @@ -61,6 +61,7 @@ re_path(r"^app", RedirectView.as_view(url=settings.APP_BASE_URL)), # Hijack re_path(r"^hijack/", include("hijack.urls", namespace="hijack")), + re_path(r"^health/", include("health_check.urls")), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/poetry.lock b/poetry.lock index 3caa47f74c..5a19136531 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1566,6 +1566,29 @@ files = [ [package.dependencies] Django = ">=2.2" +[[package]] +name = "django-health-check" +version = "3.18.4.dev18+g9cfe2ea" +description = "Run checks on services like databases, queue servers, celery processes, etc." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [] +develop = false + +[package.dependencies] +django = ">=2.2" + +[package.extras] +docs = ["sphinx"] +test = ["boto3", "celery", "django-storages", "pytest", "pytest-cov", "pytest-django", "redis"] + +[package.source] +type = "git" +url = "https://github.com/revsys/django-health-check" +reference = "9cfe2eaec5a15219513a36210b34875c03c64fe4" +resolved_reference = "9cfe2eaec5a15219513a36210b34875c03c64fe4" + [[package]] name = "django-hijack" version = "3.7.2" @@ -3977,11 +4000,8 @@ files = [ {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, @@ -5954,7 +5974,6 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -9094,4 +9113,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "~3.12" -content-hash = "f58890d176ad80a2ac17e95b8c043c0a9762aef8156b59a7a4b1021ec23ef98f" +content-hash = "a7d83acce10fef1dbafda5c0c576b40296f6df4c8e9b222255f0d3d32c1169d2" diff --git a/pyproject.toml b/pyproject.toml index 4d0400898e..3942a38cce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,108 +8,110 @@ license = "BSD-3" readme = "README.md" packages = [] authors = ["MIT ODL"] +requires-poetry = ">2.1,<3" [tool.poetry.dependencies] python = "~3.12" +Django = "4.2.21" attrs = "^25.0.0" base36 = "^0.1.1" beautifulsoup4 = "^4.8.2" boto3 = "^1.26.155" cairosvg = "2.7.1" celery = "^5.3.1" +celery-redbeat = "^2.3.2" cffi = "^1.15.1" cryptography = "^44.0.0" +dateparser = "^1.2.0" +deepmerge = "^2.0" dj-database-url = "^2.0.0" dj-static = "^0.0.6" -Django = "4.2.21" django-anymail = {extras = ["mailgun"], version = "^12.0"} django-bitfield = "^2.2.0" django-cache-memoize = "^0.2.0" django-cors-headers = "^4.0.0" django-filter = "^2.4.0" django-guardian = "^2.4.0" +django-health-check = { git = "https://github.com/revsys/django-health-check", rev="9cfe2eaec5a15219513a36210b34875c03c64fe4" } # pragma: allowlist secret django-hijack = "^3.4.1" django-imagekit = "^5.0.0" django-ipware = "^7.0.0" django-json-widget = "^2.0.0" +django-oauth-toolkit = "^2.3.0" django-redis = "^5.2.0" +django-scim2 = "^0.19.1" django-server-status = "^0.7.0" django-storages = "^1.13.2" djangorestframework = "^3.14.0" drf-jwt = "^1.19.2" +drf-nested-routers = "^0.94.0" drf-spectacular = "^0.28.0" feedparser = "^6.0.10" google-api-python-client = "^2.89.0" +html2text = "^2025.0.0" html5lib = "^1.1" ipython = "^9.0.0" +isodate = "^0.7.2" jedi = "^0.19.0" +langchain = "^0.3.11" +langchain-experimental = "^0.3.4" +langchain-openai = "^0.3.2" +litellm = "1.66.1" +llama-index = "^0.12.6" +llama-index-agent-openai = "^0.4.1" +llama-index-llms-openai = "^0.3.12" lxml = "^5.0.0" +markdown = "^3.7" markdown2 = "^2.4.8" +mitol-django-scim = "^2025.3.31" +named-enum = "^1.4.0" nested-lookup = "^0.2.25" +nh3 = "^0.2.14" ocw-data-parser = "^0.35.1" +onnxruntime = "1.21.0" +openai = "^1.55.3" opensearch-dsl = "^2.0.0" opensearch-py = "^2.0.0" +opentelemetry-api = ">=1.31.0" +opentelemetry-exporter-otlp = ">=1.31.0" +opentelemetry-instrumentation-celery = ">=0.52b0" +opentelemetry-instrumentation-django = ">=0.52b0" +opentelemetry-instrumentation-psycopg = ">=0.52b0" +opentelemetry-instrumentation-redis = ">=0.52b0" +opentelemetry-instrumentation-requests = ">=0.52b0" +opentelemetry-sdk = ">=1.31.0" +pluggy = "^1.3.0" +posthog = "^3.5.0" +psycopg = "^3.2.4" psycopg2 = "^2.9.6" +pycountry = "^24.6.1" pygithub = "^2.0.0" +pyparsing = "^3.2.1" +pytest-lazy-fixtures = "^1.1.1" python-dateutil = "^2.8.2" python-rapidjson = "^1.8" pyyaml = "^6.0.0" +qdrant-client = {extras = ["fastembed"], version = "^1.12.0"} redis = "^5.0.0" requests = "^2.31.0" -sentry-sdk = "2.25.1" +retry2 = "^0.9.5" +ruff = "0.11.5" +selenium = "^4.30.0" +sentry-sdk = "^2.25.1" social-auth-app-django = "^5.2.0" +social-auth-core = {extras = ["openidconnect"], version = "^4.4.2"} static3 = "^0.7.0" tika = "^2.6.0" +tiktoken = "^0.9.0" tldextract = "^5.0.0" toolz = "^1.0.0" ulid-py = "^0.2.0" urllib3 = "^2.0.0" -uwsgi = "^2.0.21" +uwsgi = "^2.0.29" +uwsgitop = "^0.12" wrapt = "^1.14.1" -social-auth-core = {extras = ["openidconnect"], version = "^4.4.2"} -nh3 = "^0.2.14" -retry2 = "^0.9.5" -pluggy = "^1.3.0" -named-enum = "^1.4.0" -drf-nested-routers = "^0.94.0" -django-scim2 = "^0.19.1" -django-oauth-toolkit = "^2.3.0" youtube-transcript-api = "^1.0.0" -posthog = "^3.5.0" -ruff = "0.11.5" -dateparser = "^1.2.0" -uwsgitop = "^0.12" -pytest-lazy-fixtures = "^1.1.1" -pycountry = "^24.6.1" -qdrant-client = {extras = ["fastembed"], version = "^1.12.0"} -onnxruntime = "1.21.0" -openai = "^1.55.3" -litellm = "1.66.1" -langchain = "^0.3.11" -tiktoken = "^0.9.0" -llama-index = "^0.12.6" -llama-index-llms-openai = "^0.3.12" -llama-index-agent-openai = "^0.4.1" -langchain-experimental = "^0.3.4" -langchain-openai = "^0.3.2" -deepmerge = "^2.0" -pyparsing = "^3.2.1" -html2text = "^2025.0.0" -markdown = "^3.7" -isodate = "^0.7.2" -selenium = "^4.30.0" -mitol-django-scim = "^2025.3.31" -opentelemetry-api = ">=1.31.0" -opentelemetry-sdk = ">=1.31.0" -opentelemetry-instrumentation-django = ">=0.52b0" -opentelemetry-instrumentation-psycopg = ">=0.52b0" -opentelemetry-instrumentation-redis = ">=0.52b0" -opentelemetry-instrumentation-celery = ">=0.52b0" -opentelemetry-instrumentation-requests = ">=0.52b0" -opentelemetry-exporter-otlp = ">=1.31.0" -psycopg = "^3.2.4" -celery-redbeat = "^2.3.2" [tool.poetry.group.dev.dependencies] bpython = "^0.25"