From ae908dae074d68d423f0d4ef4a1cee517608a5f4 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 13 May 2025 16:15:15 -0700 Subject: [PATCH] fix: For the openedx-events broker, use kafka _everywhere_ instead of redis Devstack is currently split between services pre-configured to use redis, and others pre-configured to use kafka. This results in the exceedingly dumb result of events produced via one broker, but never consumed because the consuming apps are hooked up to the other broker. --- docker-compose-host.yml | 8 +++++++- docker-compose.yml | 20 ++++++++++++++++++-- py_configuration_files/cms.py | 11 ++++++----- py_configuration_files/course_discovery.py | 9 ++++++--- py_configuration_files/enterprise_access.py | 6 ++---- py_configuration_files/enterprise_subsidy.py | 5 +---- py_configuration_files/lms.py | 10 ++++++---- py_private_requirements/cms.txt | 1 + py_private_requirements/lms.txt | 1 + 9 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 py_private_requirements/cms.txt create mode 100644 py_private_requirements/lms.txt diff --git a/docker-compose-host.yml b/docker-compose-host.yml index e63812a5..4b193c06 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -41,6 +41,7 @@ services: - ${DEVSTACK_WORKSPACE}/src:/edx/src - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py + - ${PWD}/py_private_requirements/lms.txt:/edx/private_requirements.txt lms-worker: volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform @@ -71,6 +72,7 @@ services: - ${DEVSTACK_WORKSPACE}/src:/edx/src - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py + - ${PWD}/py_private_requirements/cms.txt:/edx/private_requirements.txt cms-worker: volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform @@ -95,7 +97,11 @@ services: - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy - ${DEVSTACK_WORKSPACE}/src:/edx/src - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py - + enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy + - ${DEVSTACK_WORKSPACE}/src:/edx/src + - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py enterprise-catalog: volumes: - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog diff --git a/docker-compose.yml b/docker-compose.yml index 639f35b0..40b82e16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -449,7 +449,7 @@ services: lms: # Switch to `--settings devstack_with_worker` if you want to use lms-worker - command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done' + command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: @@ -638,7 +638,7 @@ services: cms: # Switch to `--settings devstack_with_worker` if you want to use cms-worker - command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done' + command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms" hostname: cms.devstack.edx depends_on: @@ -905,6 +905,22 @@ services: volumes: - /edx/var/enterprise-subsidy + enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle: + image: edxops/enterprise-subsidy-dev + container_name: edx.devstack.enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle + depends_on: + - mysql80 + - memcached + command: >- + bash -c 'while true; do python /edx/app/enterprise-subsidy/manage.py consume_events -t learner-credit-course-enrollment-lifecycle -g enterprise_subsidy_dev; sleep 2; done' + tty: true + environment: + DJANGO_SETTINGS_MODULE: enterprise_subsidy.settings.devstack + working_dir: /edx/app/enterprise-subsidy + ports: + - "18281:18281" + stdin_open: true + # ========================================================================== # edX Microfrontends # diff --git a/py_configuration_files/cms.py b/py_configuration_files/cms.py index d2ef80c4..4afa105b 100644 --- a/py_configuration_files/cms.py +++ b/py_configuration_files/cms.py @@ -90,7 +90,7 @@ ################################ DEBUG TOOLBAR ################################ -INSTALLED_APPS += ['debug_toolbar'] +INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware') INTERNAL_IPS = ('127.0.0.1',) @@ -296,11 +296,12 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing ] #################### Event bus backend ######################## - -EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer' -EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:password@edx.devstack.redis:6379/' +INSTALLED_APPS += ('edx_event_bus_kafka',) +EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081' +EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092' +EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer' +EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer' EVENT_BUS_TOPIC_PREFIX = 'dev' -EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer' course_catalog_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.course.catalog_info.changed.v1'] course_catalog_event_setting['course-catalog-info-changed']['enabled'] = True diff --git a/py_configuration_files/course_discovery.py b/py_configuration_files/course_discovery.py index 889850b9..9fa60390 100644 --- a/py_configuration_files/course_discovery.py +++ b/py_configuration_files/course_discovery.py @@ -86,9 +86,12 @@ CELERY_TASK_ALWAYS_EAGER = False -EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer' -EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer' -EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:password@edx.devstack.redis:6379/' +#################### Event bus backend ######################## +INSTALLED_APPS += ('edx_event_bus_kafka',) +EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081' +EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092' +EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer' +EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer' EVENT_BUS_TOPIC_PREFIX = 'dev' ##################################################################### diff --git a/py_configuration_files/enterprise_access.py b/py_configuration_files/enterprise_access.py index c2541527..77e081a6 100644 --- a/py_configuration_files/enterprise_access.py +++ b/py_configuration_files/enterprise_access.py @@ -61,10 +61,7 @@ # Install django-extensions for improved dev experiences # https://github.com/django-extensions/django-extensions#using-it -INSTALLED_APPS += ( - 'django_extensions', - 'edx_event_bus_kafka', -) +INSTALLED_APPS += ('django_extensions',) # BEGIN CELERY CELERY_WORKER_HIJACK_ROOT_LOGGER = True @@ -114,6 +111,7 @@ ################### Kafka Related Settings ############################## # "Standard" Kafka settings as defined in https://github.com/openedx/event-bus-kafka/tree/main +INSTALLED_APPS += ('edx_event_bus_kafka',) EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081' EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092' EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer' diff --git a/py_configuration_files/enterprise_subsidy.py b/py_configuration_files/enterprise_subsidy.py index 9a4fb364..7873adfc 100644 --- a/py_configuration_files/enterprise_subsidy.py +++ b/py_configuration_files/enterprise_subsidy.py @@ -33,10 +33,6 @@ } -INSTALLED_APPS += ( - 'edx_event_bus_kafka', -) - # Generic OAuth2 variables irrespective of SSO/backend service key types. OAUTH2_PROVIDER_URL = 'http://edx.devstack.lms:18000/oauth2' @@ -83,6 +79,7 @@ # Kafka Settings # "Standard" Kafka settings as defined in https://github.com/openedx/event-bus-kafka/tree/main +INSTALLED_APPS += ('edx_event_bus_kafka',) EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081' EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092' EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer' diff --git a/py_configuration_files/lms.py b/py_configuration_files/lms.py index 27d1625b..db6686a1 100644 --- a/py_configuration_files/lms.py +++ b/py_configuration_files/lms.py @@ -80,7 +80,7 @@ ################################ DEBUG TOOLBAR ################################ -INSTALLED_APPS += ['debug_toolbar'] +INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE += [ 'lms.djangoapps.discussion.django_comment_client.utils.QueryCountDebugMiddleware', 'debug_toolbar.middleware.DebugToolbarMiddleware', @@ -504,10 +504,12 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing CLOSEST_CLIENT_IP_FROM_HEADERS = [] #################### Event bus backend ######################## -EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer' -EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:password@edx.devstack.redis:6379/' +INSTALLED_APPS += ('edx_event_bus_kafka',) +EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081' +EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092' +EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer' +EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer' EVENT_BUS_TOPIC_PREFIX = 'dev' -EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer' certificate_revoked_event_config = EVENT_BUS_PRODUCER_CONFIG['org.openedx.learning.certificate.revoked.v1'] certificate_revoked_event_config['learning-certificate-lifecycle']['enabled'] = True diff --git a/py_private_requirements/cms.txt b/py_private_requirements/cms.txt new file mode 100644 index 00000000..feb3295f --- /dev/null +++ b/py_private_requirements/cms.txt @@ -0,0 +1 @@ +confluent-kafka[avro,schema-registry]==2.10.0 diff --git a/py_private_requirements/lms.txt b/py_private_requirements/lms.txt new file mode 100644 index 00000000..feb3295f --- /dev/null +++ b/py_private_requirements/lms.txt @@ -0,0 +1 @@ +confluent-kafka[avro,schema-registry]==2.10.0