Celery -> RQ, fix task overflowing, docker prod

Fix check for whether new parsing task should be created
Infra update, add prod containers and compose

Author: dhvcc

.dockerignore

@@ -107,7 +107,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.envs/.env
+.env
 .venv
 env/
 venv/
@@ -150,3 +150,5 @@ LICENSE
 README.md
 redis-data
 redis.conf
+staticfiles/
+docker/

.idea/backend.iml

@@ -1,13 +1,6 @@
 from django.apps import AppConfig
-from django.core.checks import register
-
-from apps.core.checks import check_redis
 
 
 class CoreConfig(AppConfig):
     name = "apps.core"
     verbose_name = "Core"
-
-    def ready(self) -> None:
-        register(check_redis)
-        return super().ready()

apps/core/apps.py

@@ -1,12 +1,5 @@
 import re
 
-import redis
-from django.conf import settings
-
-
-def init_redis_client() -> redis.Redis:
-    return redis.StrictRedis.from_url(settings.REDIS_URL, decode_responses=True)
-
 
 def url_prefix(url: str) -> str:
     return re.match(r"(^https?://(.*))/.*$", url).group(1)

apps/core/checks.py

@@ -33,9 +33,9 @@ def get_manga(request, manga_id: int):
     cache_entry = cache.get(manga.source_url)
     if cache_entry and cache_entry != ParsingStatus.parsing.value:
         return cache_entry
-    else:
+    elif not cache_entry:
         cache.set(manga.source_url, ParsingStatus.parsing.value)
-        run_spider_task(DETAIL_PARSER, url=manga.source_url)
+        run_spider_task.delay(DETAIL_PARSER, url=manga.source_url)
 
     return MangaOut(
         status=ParsingStatus.parsing.value,
@@ -53,9 +53,9 @@ def get_chapters(request, manga_id: int):
     cache_entry = cache.get(manga.rss_url)
     if cache_entry and cache_entry != ParsingStatus.parsing.value:
         return cache_entry
-    else:
+    elif not cache_entry:
         cache.set(manga.rss_url, ParsingStatus.parsing.value)
-        run_spider_task(CHAPTER_PARSER, url=manga.rss_url)
+        run_spider_task.delay(CHAPTER_PARSER, url=manga.rss_url)
 
     return ChapterListOut(
         status=ParsingStatus.parsing.value,

apps/core/utils.py

@@ -1,14 +1,14 @@
-from celery import shared_task
+from django_rq import job
 from scrapy.utils.project import get_project_settings
 from scrapyscript import Job, Processor
 
 from apps.parse.parser import PARSERS
 
 
-@shared_task
+@job
 def run_spider_task(parser_type: str, catalogue_name: str = "readmanga", url: str = None):
     parser_map = PARSERS[catalogue_name]
     spider, _ = parser_map[parser_type]
-    job = Job(spider, url=url)
+    j = Job(spider, url=url)
     p = Processor(settings=get_project_settings())
-    p.run(job)
+    p.run(j)

apps/manga/api/api.py

@@ -4,7 +4,6 @@
 from orjson import loads
 from scrapy.http import HtmlResponse
 
-from apps.core.utils import init_redis_client
 from apps.parse.items import ImagesItem
 from apps.parse.spider import InjectUrlMixin
 
@@ -18,7 +17,7 @@ class ReadmangaImageSpider(InjectUrlMixin, scrapy.Spider):
     "Chapter url, like https://readmanga.live/podniatie_urovnia_v_odinochku__A5664/vol1/0"
 
     def __init__(self, *args, url: str, **kwargs):
-        super().__init__(*args, **kwargs, start_urls=[url], redis_client=init_redis_client())
+        super().__init__(*args, **kwargs, start_urls=[url])
 
     def parse(self, response: HtmlResponse, **kwargs):
         images = re.search(r"rm_h.initReader\(.*(\[{2}.*]{2}).*\)", response.text)

apps/parse/tasks.py

@@ -1,63 +1,50 @@
-# Compose for development using volumes to ensure hot reload
-#version: "3"
-
 services:
   web:
     profiles:
       - app
     build:
       context: .
-      dockerfile: docker/web/Dockerfile.dev
+      dockerfile: docker/web/Dockerfile
+    restart: unless-stopped
     expose:
       - 8000
     environment:
       - DEBUG
       - ALLOWED_HOSTS
+      - SECRET_KEY
       - DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-sora}
-      - SECRET_KEY=${SECRET_KEY:-DevServer}
       # Static
       - HOST=0.0.0.0
       - PORT=8000
       - REDIS_URL=redis://redis:6379
       - TYPESENSE_HOST=typesense
     ports:
-      - "${PORT:-8888}:8000"
-    volumes:
-      - ./:/app
-      - sora_poetry:/home/sora/.cache/pypoetry
+      - "${PORT:-80}:8000"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:8000/api/health" ]
       interval: 5s
       timeout: 5s
 
-  celery:
+  rq:
     profiles:
       - app
-      - base
     image: backend_web
-    command: poetry run celery -A manga_reader worker -l INFO
+    restart: unless-stopped
+    command: poetry run python manage.py rqworker default
     environment:
       - DEBUG
       - PROXY
+      - SECRET_KEY
       - DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-sora}
-      - SECRET_KEY=${SECRET_KEY:-DevServer}
       - REDIS_URL=redis://redis:6379
       - TYPESENSE_HOST=typesense
-    volumes:
-      - ./:/app
-      - sora_poetry:/home/sora/.cache/pypoetry
     depends_on:
       - web
-    healthcheck:
-      test: poetry run celery -A manga_reader status
-      interval: 5s
-      timeout: 5s
 
   db:
     profiles:
       - data
       - app
-      - base
     restart: unless-stopped
     image: postgres:12
     ports:
@@ -78,7 +65,6 @@ services:
     profiles:
       - data
       - app
-      - base
     image: redis
     command: redis-server
     restart: unless-stopped
@@ -100,7 +86,6 @@ services:
     profiles:
       - data
       - app
-      - base
     image: typesense/typesense:0.23.1
     restart: unless-stopped
     command: --data-dir /data --api-key=$TYPESENSE_API_KEY --enable-cors
@@ -111,6 +96,5 @@ services:
 
 volumes:
   tsdata:
-  sora_poetry:
   sora_redis:
   sora_postgres:

apps/readmanga/images.py

@@ -14,9 +14,6 @@ if $(is_debug); then
     info "Debug mode" "Installing dependencies"
     sudo chown sora:sora ~/.cache/pypoetry/
     poetry install
-else
-    info "Production mode" "Collecting static files to /app/staticfiles"
-    poetry run ./manage.py collectstatic --no-input --clear
 fi
 
 info "Waiting for postgres"
@@ -34,9 +31,6 @@ until curl "http://$TYPESENSE_HOST:8108/health"; do
     sleep 1
 done
 
-#info "Rebuilding index"
-#poetry run ./manage.py rebuild_index
-
 # Get core count
 core_count=$(grep 'cpu[0-9]+' /proc/stat | wc -l)
 # Calculate worker count, max 12