Install qcluster. Switch to redis.

Author: sdarwin

defaults/main.yml

@@ -358,11 +358,12 @@ mailman3_distribute_maps_virtualenv_dir: "{{ mailman3_core_var_dir }}/distribute
 # Base set of Python packages
 __mailman3_pip_packages:
   # - django-allauth==0.54.0
+  - django-mailman3==1.3.13
   - importlib-resources==5.13.0
   - mailman
   - mailman-web==0.0.9
-  - HyperKitty==1.3.9
-  - django-allauth==0.61.1
+  - HyperKitty==1.3.10
+  - django-allauth==0.63.3
   - mailman-hyperkitty  # ??
   - elasticsearch==7.17.9
   - environs
@@ -390,8 +391,9 @@ __mailman3_system_dependency_packages:
     - python3
     - python3-venv
     - python3-setuptools
-    - memcached
-    - libmemcached-tools
+    # - memcached
+    # - libmemcached-tools
+    - redis
   redhat-7:
     - rh-python38   # requires EL7+ and CentOS SCL
   redhat-8:

handlers/main.yml

@@ -29,6 +29,13 @@
     - mailman3_process_manager == "systemd"
     - ansible_virtualization_type != "docker"
 
+- name: Restart qcluster service  # noqa command-instead-of-module
+  ansible.builtin.command: systemctl try-reload-or-restart qcluster.service
+  changed_when: false
+  when:
+    - mailman3_process_manager == "systemd"
+    - ansible_virtualization_type != "docker"
+
 - name: Restart nginx
   ansible.builtin.service:
     name: nginx

tasks/django.yml

@@ -109,7 +109,7 @@
     user: "{{ __mailman3_web_user_name }}"
     minute: "{{ item.minute | default(omit) }}"
     special_time: "{{ item.special_time | default(omit) }}"
-    job: "MAILMAN_WEB_CONFIG={{ mailman3_django_project_dir }}/settings.py {{ mailman3_install_dir }}/bin/mailman-web runjobs {{ item.name | default(item.special_time) }}"
+    job: "MAILMAN_WEB_CONFIG={{ mailman3_django_project_dir }}/settings.py {{ mailman3_install_dir }}/bin/mailman-web runjobs {{ item.name | default(item.special_time) }} > {{ item.name | default(item.special_time) }}.txt 2>&1"
   loop:
     - special_time: hourly
     - special_time: daily

tasks/install_pip.yml

@@ -89,6 +89,8 @@
       dest: "{{ mailman3_core_service_name }}.service"
     - src: mailman3-web.service
       dest: "{{ mailman3_web_service_name }}{{ '@' if mailman3_domains is defined else '' }}.service"
+    - src: qcluster.service
+      dest: qcluster.service
   loop_control:
     label: "{{ item.dest }}"
   when: mailman3_process_manager == 'systemd'
@@ -97,6 +99,7 @@
     # TODO: split tasks to only restart the affected service
     - Restart mailman3-core service
     - Restart mailman3-web service
+    - Restart qcluster service
 
 - name: Install systemd socket unit file
   ansible.builtin.template:

tasks/main.yml

@@ -67,3 +67,10 @@
     state: started
   loop: "{{ mailman3_domains | default(['']) }}"
   when: mailman3_process_manager != "supervisor"
+
+- name: Ensure qcluster is enabled and running
+  ansible.builtin.service:
+    name: qcluster
+    enabled: true
+    state: started
+  when: mailman3_process_manager != "supervisor"

templates/qcluster.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=HyperKitty async tasks runner
+After=network.target remote-fs.target
+ConditionPathExists={{ mailman3_django_project_dir }}/settings.py
+
+[Service]
+Environment="MAILMAN_WEB_CONFIG={{ mailman3_django_project_dir }}/settings.py"
+ExecStart={{ mailman3_install_dir }}/bin/mailman-web qcluster
+User={{ __mailman3_web_user_name }}
+Group={{ __mailman3_web_group_name }}
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

templates/settings.py.j2

@@ -26,10 +26,17 @@ DATABASES = {
     }
 }
 
+# CACHES = {
+#     'default': {
+#         "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+#         'LOCATION': '127.0.0.1:11211',
+#     }
+# }
+
 CACHES = {
-    'default': {
-        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
-        'LOCATION': '127.0.0.1:11211',
+    "default": {
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": "redis://127.0.0.1:6379",
     }
 }