Skip to content

Use database to keep track of session cleaning #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Aug 5, 2023
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ Using the following categories, list your changes in this order:
- If using `settings.py:REACTPY_DATABASE`, `reactpy_django.database.Router` must now be registered in `settings.py:DATABASE_ROUTERS`.
- By default, ReactPy will now use a backhaul thread to increase performance.
- Minimum Python version required is now `3.9`
- A thread-safe cache is no longer required.

## [3.2.1] - 2023-06-29

Expand Down
2 changes: 1 addition & 1 deletion docs/python/settings.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Cache used to store ReactPy web modules.
# ReactPy requires a multiprocessing-safe and thread-safe cache.
# ReactPy benefits from a fast, well indexed cache.
# We recommend redis or python-diskcache.
REACTPY_CACHE = "default"

Expand Down
2 changes: 1 addition & 1 deletion docs/src/contribute/running-tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,5 @@ Alternatively, if you want to only run Django related tests, you can use the fol

```bash linenums="0"
cd tests
python mange.py test
python manage.py test
```
72 changes: 53 additions & 19 deletions src/reactpy_django/checks.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import sys

from django.contrib.staticfiles.finders import find
from django.core.checks import Error, Tags, Warning, register


Expand All @@ -6,6 +9,8 @@ def reactpy_warnings(app_configs, **kwargs):
from django.conf import settings
from django.urls import reverse

from reactpy_django.config import REACTPY_FAILED_COMPONENTS

warnings = []

# REACTPY_DATABASE is not an in-memory database.
Expand All @@ -25,23 +30,6 @@ def reactpy_warnings(app_configs, **kwargs):
)
)

# REACTPY_CACHE is not an in-memory cache.
if getattr(settings, "CACHES", {}).get(
getattr(settings, "REACTPY_CACHE", "default"), {}
).get("BACKEND", None) in {
"django.core.cache.backends.dummy.DummyCache",
"django.core.cache.backends.locmem.LocMemCache",
}:
warnings.append(
Warning(
"Using ReactPy with an in-memory cache can cause unexpected "
"behaviors.",
hint="Configure settings.py:CACHES[REACTPY_CACHE], to use a "
"multiprocessing and thread safe cache.",
id="reactpy_django.W002",
)
)

# ReactPy URLs exist
try:
reverse("reactpy:web_modules", kwargs={"file": "example"})
Expand All @@ -52,10 +40,47 @@ def reactpy_warnings(app_configs, **kwargs):
"ReactPy URLs have not been registered.",
hint="""Add 'path("reactpy/", include("reactpy_django.http.urls"))' """
"to your application's urlpatterns.",
id="reactpy_django.W002",
)
)

# Warn if REACTPY_BACKHAUL_THREAD is set to True on Linux with Daphne
if (
sys.argv
and sys.argv[0].endswith("daphne")
and getattr(settings, "REACTPY_BACKHAUL_THREAD", True)
and sys.platform == "linux"
):
warnings.append(
Warning(
"REACTPY_BACKHAUL_THREAD is enabled but you running with Daphne on Linux. "
"This configuration is known to be unstable.",
hint="Set settings.py:REACTPY_BACKHAUL_THREAD to False or use a different webserver.",
id="reactpy_django.W003",
)
)

# Check if reactpy_django/client.js is available
if not find("reactpy_django/client.js"):
warnings.append(
Warning(
"ReactPy client.js could not be found within Django static files!",
hint="Check your Django static file configuration.",
id="reactpy_django.W004",
)
)

# Check if any components failed to be registered
if REACTPY_FAILED_COMPONENTS:
warnings.append(
Warning(
"ReactPy failed to register the following components:\n\t+ "
+ "\n\t+ ".join(REACTPY_FAILED_COMPONENTS),
hint="Check if these paths are valid, or if an exception is being raised during import.",
id="reactpy_django.W005",
)
)

return warnings


Expand All @@ -69,8 +94,7 @@ def reactpy_errors(app_configs, **kwargs):
if not getattr(settings, "ASGI_APPLICATION", None):
errors.append(
Error(
"ASGI_APPLICATION is not defined."
" ReactPy requires ASGI to be enabled.",
"ASGI_APPLICATION is not defined, but ReactPy requires ASGI.",
hint="Add ASGI_APPLICATION to settings.py.",
id="reactpy_django.E001",
)
Expand Down Expand Up @@ -150,4 +174,14 @@ def reactpy_errors(app_configs, **kwargs):
)
)

# Check for dependencies
if "channels" not in settings.INSTALLED_APPS:
errors.append(
Error(
"Django Channels is not installed.",
hint="Add 'channels' to settings.py:INSTALLED_APPS.",
id="reactpy_django.E009",
)
)

return errors
1 change: 0 additions & 1 deletion src/reactpy_django/components.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,6 @@ def _cached_static_contents(static_path: str):
)

# Fetch the file from cache, if available
# Cache is preferrable to `use_memo` due to multiprocessing capabilities
last_modified_time = os.stat(abs_path).st_mtime
cache_key = f"reactpy_django:static_contents:{static_path}"
file_contents = caches[REACTPY_CACHE].get(
Expand Down
20 changes: 1 addition & 19 deletions src/reactpy_django/config.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
from __future__ import annotations

import logging
import sys

from django.conf import settings
from django.core.cache import DEFAULT_CACHE_ALIAS
from django.db import DEFAULT_DB_ALIAS
Expand All @@ -16,12 +13,10 @@
)
from reactpy_django.utils import import_dotted_path

_logger = logging.getLogger(__name__)


# Non-configurable values
REACTPY_DEBUG_MODE.set_current(getattr(settings, "DEBUG"))
REACTPY_REGISTERED_COMPONENTS: dict[str, ComponentConstructor] = {}
REACTPY_FAILED_COMPONENTS: set[str] = set()
REACTPY_VIEW_COMPONENT_IFRAMES: dict[str, ViewComponentIframe] = {}


Expand Down Expand Up @@ -68,16 +63,3 @@
"REACTPY_BACKHAUL_THREAD",
True,
)

# Settings checks (separate from Django checks)
if (
sys.platform == "linux"
and sys.argv
and sys.argv[0].endswith("daphne")
and REACTPY_BACKHAUL_THREAD
):
_logger.warning(
"ReactPy is running on Linux with Daphne, but REACTPY_BACKHAUL_THREAD is set "
"to True. This configuration is known to be unstable. Either set "
"REACTPY_BACKHAUL_THREAD to False, or run ReactPy with a different ASGI server."
)
14 changes: 7 additions & 7 deletions src/reactpy_django/http/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
from reactpy_django.config import REACTPY_CACHE

web_modules_dir = REACTPY_WEB_MODULES_DIR.current
path = os.path.abspath(web_modules_dir.joinpath(*file.split("/")))
path = os.path.abspath(web_modules_dir.joinpath(file))

# Prevent attempts to walk outside of the web modules dir
if str(web_modules_dir) != os.path.commonpath((path, web_modules_dir)):
Expand All @@ -25,18 +25,18 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:

# Fetch the file from cache, if available
last_modified_time = os.stat(path).st_mtime
cache_key = create_cache_key("web_module", str(path).lstrip(str(web_modules_dir)))
response = await caches[REACTPY_CACHE].aget(
cache_key = create_cache_key("web_modules", path)
file_contents = await caches[REACTPY_CACHE].aget(
cache_key, version=int(last_modified_time)
)
if response is None:
if file_contents is None:
async with async_open(path, "r") as fp:
response = HttpResponse(await fp.read(), content_type="text/javascript")
file_contents = await fp.read()
await caches[REACTPY_CACHE].adelete(cache_key)
await caches[REACTPY_CACHE].aset(
cache_key, response, timeout=None, version=int(last_modified_time)
cache_key, file_contents, timeout=604800, version=int(last_modified_time)
)
return response
return HttpResponse(file_contents, content_type="text/javascript")


async def view_to_component_iframe(
Expand Down
27 changes: 27 additions & 0 deletions src/reactpy_django/migrations/0004_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Generated by Django 4.2.3 on 2023-08-04 05:49

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("reactpy_django", "0003_componentsession_delete_componentparams"),
]

operations = [
migrations.CreateModel(
name="Config",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("cleaned_at", models.DateTimeField(auto_now_add=True)),
],
),
]
16 changes: 16 additions & 0 deletions src/reactpy_django/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,19 @@ class ComponentSession(models.Model):
uuid = models.UUIDField(primary_key=True, editable=False, unique=True) # type: ignore
params = models.BinaryField(editable=False) # type: ignore
last_accessed = models.DateTimeField(auto_now_add=True) # type: ignore


class Config(models.Model):
"""A singleton model for storing ReactPy configuration."""

cleaned_at = models.DateTimeField(auto_now_add=True) # type: ignore

def save(self, *args, **kwargs):
"""Singleton save method."""
self.pk = 1
super().save(*args, **kwargs)

@classmethod
def load(cls):
obj, created = cls.objects.get_or_create(pk=1)
return obj
48 changes: 19 additions & 29 deletions src/reactpy_django/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@
import logging
import os
import re
from datetime import datetime, timedelta
from datetime import timedelta
from fnmatch import fnmatch
from importlib import import_module
from inspect import iscoroutinefunction
from typing import Any, Callable, Sequence

from channels.db import database_sync_to_async
from django.core.cache import caches
from django.db.models import ManyToManyField, ManyToOneRel, prefetch_related_objects
from django.db.models.base import Model
from django.db.models.query import QuerySet
Expand All @@ -24,7 +23,6 @@

from reactpy_django.exceptions import ComponentDoesNotExistError, ComponentParamError


_logger = logging.getLogger(__name__)
_component_tag = r"(?P<tag>component)"
_component_path = r"(?P<path>\"[^\"'\s]+\"|'[^\"'\s]+')"
Expand Down Expand Up @@ -88,14 +86,18 @@ def _register_component(dotted_path: str) -> Callable:
"""Adds a component to the mapping of registered components.
This should only be called on startup to maintain synchronization during mulitprocessing.
"""
from reactpy_django.config import REACTPY_REGISTERED_COMPONENTS
from reactpy_django.config import (
REACTPY_FAILED_COMPONENTS,
REACTPY_REGISTERED_COMPONENTS,
)

if dotted_path in REACTPY_REGISTERED_COMPONENTS:
return REACTPY_REGISTERED_COMPONENTS[dotted_path]

try:
REACTPY_REGISTERED_COMPONENTS[dotted_path] = import_dotted_path(dotted_path)
except AttributeError as e:
REACTPY_FAILED_COMPONENTS.add(dotted_path)
raise ComponentDoesNotExistError(
f"Error while fetching '{dotted_path}'. {(str(e).capitalize())}."
) from e
Expand Down Expand Up @@ -266,7 +268,7 @@ def django_query_postprocessor(
# Force the query to execute
getattr(data, field.name, None)

if many_to_one and type(field) == ManyToOneRel:
if many_to_one and type(field) == ManyToOneRel: # noqa: #E721
prefetch_fields.append(field.related_name or f"{field.name}_set")

elif many_to_many and isinstance(field, ManyToManyField):
Expand Down Expand Up @@ -332,35 +334,23 @@ def create_cache_key(*args):
def db_cleanup(immediate: bool = False):
"""Deletes expired component sessions from the database.
This function may be expanded in the future to include additional cleanup tasks."""
from .config import REACTPY_CACHE, REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX
from .models import ComponentSession

clean_started_at = datetime.now()
cache_key: str = create_cache_key("last_cleaned")
now_str: str = datetime.strftime(timezone.now(), DATE_FORMAT)
cleaned_at_str: str = caches[REACTPY_CACHE].get(cache_key)
cleaned_at: datetime = timezone.make_aware(
datetime.strptime(cleaned_at_str or now_str, DATE_FORMAT)
)
clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX)
expires_by: datetime = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX)
from .config import REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX
from .models import ComponentSession, Config

# Component params exist in the DB, but we don't know when they were last cleaned
if not cleaned_at_str and ComponentSession.objects.all():
_logger.warning(
"ReactPy has detected component sessions in the database, "
"but no timestamp was found in cache. This may indicate that "
"the cache has been cleared."
)
config = Config.load()
start_time = timezone.now()
cleaned_at = config.cleaned_at
clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX)

# Delete expired component parameters
# Use timestamps in cache (`cleaned_at_str`) as a no-dependency rate limiter
if immediate or not cleaned_at_str or timezone.now() >= clean_needed_by:
ComponentSession.objects.filter(last_accessed__lte=expires_by).delete()
caches[REACTPY_CACHE].set(cache_key, now_str, timeout=None)
if immediate or timezone.now() >= clean_needed_by:
expiration_date = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX)
ComponentSession.objects.filter(last_accessed__lte=expiration_date).delete()
config.cleaned_at = timezone.now()
config.save()

# Check if cleaning took abnormally long
clean_duration = datetime.now() - clean_started_at
clean_duration = timezone.now() - start_time
if REACTPY_DEBUG_MODE and clean_duration.total_seconds() > 1:
_logger.warning(
"ReactPy has taken %s seconds to clean up expired component sessions. "
Expand Down
11 changes: 8 additions & 3 deletions tests/test_app/admin.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from django.contrib import admin
from reactpy_django.models import ComponentSession, Config

from test_app.models import (
AsyncForiegnChild,
AsyncRelationalChild,
Expand All @@ -10,8 +12,6 @@
TodoItem,
)

from reactpy_django.models import ComponentSession


@admin.register(TodoItem)
class TodoItemAdmin(admin.ModelAdmin):
Expand Down Expand Up @@ -55,4 +55,9 @@ class AsyncForiegnChildAdmin(admin.ModelAdmin):

@admin.register(ComponentSession)
class ComponentSessionAdmin(admin.ModelAdmin):
list_display = ("uuid", "last_accessed")
list_display = ["uuid", "last_accessed"]


@admin.register(Config)
class ConfigAdmin(admin.ModelAdmin):
list_display = ["pk", "cleaned_at"]
Loading