diff --git a/CHANGELOG.md b/CHANGELOG.md index c27809b0..dd5d8ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Using the following categories, list your changes in this order: - If using `settings.py:REACTPY_DATABASE`, `reactpy_django.database.Router` must now be registered in `settings.py:DATABASE_ROUTERS`. - By default, ReactPy will now use a backhaul thread to increase performance. - Minimum Python version required is now `3.9` +- A thread-safe cache is no longer required. ## [3.2.1] - 2023-06-29 diff --git a/docs/python/settings.py b/docs/python/settings.py index 56fc037d..7454818d 100644 --- a/docs/python/settings.py +++ b/docs/python/settings.py @@ -1,5 +1,5 @@ # Cache used to store ReactPy web modules. -# ReactPy requires a multiprocessing-safe and thread-safe cache. +# ReactPy benefits from a fast, well indexed cache. # We recommend redis or python-diskcache. REACTPY_CACHE = "default" diff --git a/docs/src/contribute/running-tests.md b/docs/src/contribute/running-tests.md index c9145c2d..79d3b114 100644 --- a/docs/src/contribute/running-tests.md +++ b/docs/src/contribute/running-tests.md @@ -41,5 +41,5 @@ Alternatively, if you want to only run Django related tests, you can use the fol ```bash linenums="0" cd tests -python mange.py test +python manage.py test ``` diff --git a/src/reactpy_django/checks.py b/src/reactpy_django/checks.py index 3feb4f18..03955950 100644 --- a/src/reactpy_django/checks.py +++ b/src/reactpy_django/checks.py @@ -1,3 +1,6 @@ +import sys + +from django.contrib.staticfiles.finders import find from django.core.checks import Error, Tags, Warning, register @@ -6,6 +9,8 @@ def reactpy_warnings(app_configs, **kwargs): from django.conf import settings from django.urls import reverse + from reactpy_django.config import REACTPY_FAILED_COMPONENTS + warnings = [] # REACTPY_DATABASE is not an in-memory database. @@ -25,23 +30,6 @@ def reactpy_warnings(app_configs, **kwargs): ) ) - # REACTPY_CACHE is not an in-memory cache. - if getattr(settings, "CACHES", {}).get( - getattr(settings, "REACTPY_CACHE", "default"), {} - ).get("BACKEND", None) in { - "django.core.cache.backends.dummy.DummyCache", - "django.core.cache.backends.locmem.LocMemCache", - }: - warnings.append( - Warning( - "Using ReactPy with an in-memory cache can cause unexpected " - "behaviors.", - hint="Configure settings.py:CACHES[REACTPY_CACHE], to use a " - "multiprocessing and thread safe cache.", - id="reactpy_django.W002", - ) - ) - # ReactPy URLs exist try: reverse("reactpy:web_modules", kwargs={"file": "example"}) @@ -52,10 +40,47 @@ def reactpy_warnings(app_configs, **kwargs): "ReactPy URLs have not been registered.", hint="""Add 'path("reactpy/", include("reactpy_django.http.urls"))' """ "to your application's urlpatterns.", + id="reactpy_django.W002", + ) + ) + + # Warn if REACTPY_BACKHAUL_THREAD is set to True on Linux with Daphne + if ( + sys.argv + and sys.argv[0].endswith("daphne") + and getattr(settings, "REACTPY_BACKHAUL_THREAD", True) + and sys.platform == "linux" + ): + warnings.append( + Warning( + "REACTPY_BACKHAUL_THREAD is enabled but you running with Daphne on Linux. " + "This configuration is known to be unstable.", + hint="Set settings.py:REACTPY_BACKHAUL_THREAD to False or use a different webserver.", id="reactpy_django.W003", ) ) + # Check if reactpy_django/client.js is available + if not find("reactpy_django/client.js"): + warnings.append( + Warning( + "ReactPy client.js could not be found within Django static files!", + hint="Check your Django static file configuration.", + id="reactpy_django.W004", + ) + ) + + # Check if any components failed to be registered + if REACTPY_FAILED_COMPONENTS: + warnings.append( + Warning( + "ReactPy failed to register the following components:\n\t+ " + + "\n\t+ ".join(REACTPY_FAILED_COMPONENTS), + hint="Check if these paths are valid, or if an exception is being raised during import.", + id="reactpy_django.W005", + ) + ) + return warnings @@ -69,8 +94,7 @@ def reactpy_errors(app_configs, **kwargs): if not getattr(settings, "ASGI_APPLICATION", None): errors.append( Error( - "ASGI_APPLICATION is not defined." - " ReactPy requires ASGI to be enabled.", + "ASGI_APPLICATION is not defined, but ReactPy requires ASGI.", hint="Add ASGI_APPLICATION to settings.py.", id="reactpy_django.E001", ) @@ -150,4 +174,14 @@ def reactpy_errors(app_configs, **kwargs): ) ) + # Check for dependencies + if "channels" not in settings.INSTALLED_APPS: + errors.append( + Error( + "Django Channels is not installed.", + hint="Add 'channels' to settings.py:INSTALLED_APPS.", + id="reactpy_django.E009", + ) + ) + return errors diff --git a/src/reactpy_django/components.py b/src/reactpy_django/components.py index a4e31840..9d6e6a98 100644 --- a/src/reactpy_django/components.py +++ b/src/reactpy_django/components.py @@ -210,7 +210,6 @@ def _cached_static_contents(static_path: str): ) # Fetch the file from cache, if available - # Cache is preferrable to `use_memo` due to multiprocessing capabilities last_modified_time = os.stat(abs_path).st_mtime cache_key = f"reactpy_django:static_contents:{static_path}" file_contents = caches[REACTPY_CACHE].get( diff --git a/src/reactpy_django/config.py b/src/reactpy_django/config.py index 6d5b1130..ab9bebfb 100644 --- a/src/reactpy_django/config.py +++ b/src/reactpy_django/config.py @@ -1,8 +1,5 @@ from __future__ import annotations -import logging -import sys - from django.conf import settings from django.core.cache import DEFAULT_CACHE_ALIAS from django.db import DEFAULT_DB_ALIAS @@ -16,12 +13,10 @@ ) from reactpy_django.utils import import_dotted_path -_logger = logging.getLogger(__name__) - - # Non-configurable values REACTPY_DEBUG_MODE.set_current(getattr(settings, "DEBUG")) REACTPY_REGISTERED_COMPONENTS: dict[str, ComponentConstructor] = {} +REACTPY_FAILED_COMPONENTS: set[str] = set() REACTPY_VIEW_COMPONENT_IFRAMES: dict[str, ViewComponentIframe] = {} @@ -68,16 +63,3 @@ "REACTPY_BACKHAUL_THREAD", True, ) - -# Settings checks (separate from Django checks) -if ( - sys.platform == "linux" - and sys.argv - and sys.argv[0].endswith("daphne") - and REACTPY_BACKHAUL_THREAD -): - _logger.warning( - "ReactPy is running on Linux with Daphne, but REACTPY_BACKHAUL_THREAD is set " - "to True. This configuration is known to be unstable. Either set " - "REACTPY_BACKHAUL_THREAD to False, or run ReactPy with a different ASGI server." - ) diff --git a/src/reactpy_django/http/views.py b/src/reactpy_django/http/views.py index 752e263b..12129791 100644 --- a/src/reactpy_django/http/views.py +++ b/src/reactpy_django/http/views.py @@ -15,7 +15,7 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse: from reactpy_django.config import REACTPY_CACHE web_modules_dir = REACTPY_WEB_MODULES_DIR.current - path = os.path.abspath(web_modules_dir.joinpath(*file.split("/"))) + path = os.path.abspath(web_modules_dir.joinpath(file)) # Prevent attempts to walk outside of the web modules dir if str(web_modules_dir) != os.path.commonpath((path, web_modules_dir)): @@ -25,18 +25,18 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse: # Fetch the file from cache, if available last_modified_time = os.stat(path).st_mtime - cache_key = create_cache_key("web_module", str(path).lstrip(str(web_modules_dir))) - response = await caches[REACTPY_CACHE].aget( + cache_key = create_cache_key("web_modules", path) + file_contents = await caches[REACTPY_CACHE].aget( cache_key, version=int(last_modified_time) ) - if response is None: + if file_contents is None: async with async_open(path, "r") as fp: - response = HttpResponse(await fp.read(), content_type="text/javascript") + file_contents = await fp.read() await caches[REACTPY_CACHE].adelete(cache_key) await caches[REACTPY_CACHE].aset( - cache_key, response, timeout=None, version=int(last_modified_time) + cache_key, file_contents, timeout=604800, version=int(last_modified_time) ) - return response + return HttpResponse(file_contents, content_type="text/javascript") async def view_to_component_iframe( diff --git a/src/reactpy_django/migrations/0004_config.py b/src/reactpy_django/migrations/0004_config.py new file mode 100644 index 00000000..61b093d9 --- /dev/null +++ b/src/reactpy_django/migrations/0004_config.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.3 on 2023-08-04 05:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("reactpy_django", "0003_componentsession_delete_componentparams"), + ] + + operations = [ + migrations.CreateModel( + name="Config", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("cleaned_at", models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/src/reactpy_django/models.py b/src/reactpy_django/models.py index 79d22072..65152126 100644 --- a/src/reactpy_django/models.py +++ b/src/reactpy_django/models.py @@ -9,3 +9,19 @@ class ComponentSession(models.Model): uuid = models.UUIDField(primary_key=True, editable=False, unique=True) # type: ignore params = models.BinaryField(editable=False) # type: ignore last_accessed = models.DateTimeField(auto_now_add=True) # type: ignore + + +class Config(models.Model): + """A singleton model for storing ReactPy configuration.""" + + cleaned_at = models.DateTimeField(auto_now_add=True) # type: ignore + + def save(self, *args, **kwargs): + """Singleton save method.""" + self.pk = 1 + super().save(*args, **kwargs) + + @classmethod + def load(cls): + obj, created = cls.objects.get_or_create(pk=1) + return obj diff --git a/src/reactpy_django/utils.py b/src/reactpy_django/utils.py index f41c09db..f4e0f8e6 100644 --- a/src/reactpy_django/utils.py +++ b/src/reactpy_django/utils.py @@ -5,14 +5,13 @@ import logging import os import re -from datetime import datetime, timedelta +from datetime import timedelta from fnmatch import fnmatch from importlib import import_module from inspect import iscoroutinefunction from typing import Any, Callable, Sequence from channels.db import database_sync_to_async -from django.core.cache import caches from django.db.models import ManyToManyField, ManyToOneRel, prefetch_related_objects from django.db.models.base import Model from django.db.models.query import QuerySet @@ -24,7 +23,6 @@ from reactpy_django.exceptions import ComponentDoesNotExistError, ComponentParamError - _logger = logging.getLogger(__name__) _component_tag = r"(?Pcomponent)" _component_path = r"(?P\"[^\"'\s]+\"|'[^\"'\s]+')" @@ -88,7 +86,10 @@ def _register_component(dotted_path: str) -> Callable: """Adds a component to the mapping of registered components. This should only be called on startup to maintain synchronization during mulitprocessing. """ - from reactpy_django.config import REACTPY_REGISTERED_COMPONENTS + from reactpy_django.config import ( + REACTPY_FAILED_COMPONENTS, + REACTPY_REGISTERED_COMPONENTS, + ) if dotted_path in REACTPY_REGISTERED_COMPONENTS: return REACTPY_REGISTERED_COMPONENTS[dotted_path] @@ -96,6 +97,7 @@ def _register_component(dotted_path: str) -> Callable: try: REACTPY_REGISTERED_COMPONENTS[dotted_path] = import_dotted_path(dotted_path) except AttributeError as e: + REACTPY_FAILED_COMPONENTS.add(dotted_path) raise ComponentDoesNotExistError( f"Error while fetching '{dotted_path}'. {(str(e).capitalize())}." ) from e @@ -266,7 +268,7 @@ def django_query_postprocessor( # Force the query to execute getattr(data, field.name, None) - if many_to_one and type(field) == ManyToOneRel: + if many_to_one and type(field) == ManyToOneRel: # noqa: #E721 prefetch_fields.append(field.related_name or f"{field.name}_set") elif many_to_many and isinstance(field, ManyToManyField): @@ -332,35 +334,23 @@ def create_cache_key(*args): def db_cleanup(immediate: bool = False): """Deletes expired component sessions from the database. This function may be expanded in the future to include additional cleanup tasks.""" - from .config import REACTPY_CACHE, REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX - from .models import ComponentSession - - clean_started_at = datetime.now() - cache_key: str = create_cache_key("last_cleaned") - now_str: str = datetime.strftime(timezone.now(), DATE_FORMAT) - cleaned_at_str: str = caches[REACTPY_CACHE].get(cache_key) - cleaned_at: datetime = timezone.make_aware( - datetime.strptime(cleaned_at_str or now_str, DATE_FORMAT) - ) - clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX) - expires_by: datetime = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX) + from .config import REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX + from .models import ComponentSession, Config - # Component params exist in the DB, but we don't know when they were last cleaned - if not cleaned_at_str and ComponentSession.objects.all(): - _logger.warning( - "ReactPy has detected component sessions in the database, " - "but no timestamp was found in cache. This may indicate that " - "the cache has been cleared." - ) + config = Config.load() + start_time = timezone.now() + cleaned_at = config.cleaned_at + clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX) # Delete expired component parameters - # Use timestamps in cache (`cleaned_at_str`) as a no-dependency rate limiter - if immediate or not cleaned_at_str or timezone.now() >= clean_needed_by: - ComponentSession.objects.filter(last_accessed__lte=expires_by).delete() - caches[REACTPY_CACHE].set(cache_key, now_str, timeout=None) + if immediate or timezone.now() >= clean_needed_by: + expiration_date = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX) + ComponentSession.objects.filter(last_accessed__lte=expiration_date).delete() + config.cleaned_at = timezone.now() + config.save() # Check if cleaning took abnormally long - clean_duration = datetime.now() - clean_started_at + clean_duration = timezone.now() - start_time if REACTPY_DEBUG_MODE and clean_duration.total_seconds() > 1: _logger.warning( "ReactPy has taken %s seconds to clean up expired component sessions. " diff --git a/tests/test_app/admin.py b/tests/test_app/admin.py index 99f40e46..77e43d4c 100644 --- a/tests/test_app/admin.py +++ b/tests/test_app/admin.py @@ -1,4 +1,6 @@ from django.contrib import admin +from reactpy_django.models import ComponentSession, Config + from test_app.models import ( AsyncForiegnChild, AsyncRelationalChild, @@ -10,8 +12,6 @@ TodoItem, ) -from reactpy_django.models import ComponentSession - @admin.register(TodoItem) class TodoItemAdmin(admin.ModelAdmin): @@ -55,4 +55,9 @@ class AsyncForiegnChildAdmin(admin.ModelAdmin): @admin.register(ComponentSession) class ComponentSessionAdmin(admin.ModelAdmin): - list_display = ("uuid", "last_accessed") + list_display = ["uuid", "last_accessed"] + + +@admin.register(Config) +class ConfigAdmin(admin.ModelAdmin): + list_display = ["pk", "cleaned_at"] diff --git a/tests/test_app/settings.py b/tests/test_app/settings.py index f1b0a69e..10cd8f6b 100644 --- a/tests/test_app/settings.py +++ b/tests/test_app/settings.py @@ -24,7 +24,7 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = "django-insecure-n!bd1#+7ufw5#9ipayu9k(lyu@za$c2ajbro7es(v8_7w1$=&c" -# Run with debug off whenever the server is not run with `runserver` +# Run in production mode when using a real webserver DEBUG = all( not sys.argv[0].endswith(substring) for substring in {"hypercorn", "uvicorn", "daphne"} diff --git a/tests/test_app/tests/test_database.py b/tests/test_app/tests/test_database.py index d6bd9059..3bd23527 100644 --- a/tests/test_app/tests/test_database.py +++ b/tests/test_app/tests/test_database.py @@ -4,7 +4,6 @@ import dill as pickle from django.test import TransactionTestCase - from reactpy_django import utils from reactpy_django.models import ComponentSession from reactpy_django.types import ComponentParamData @@ -13,6 +12,11 @@ class RoutedDatabaseTests(TransactionTestCase): databases = {"reactpy"} + @classmethod + def setUpClass(cls): + super().setUpClass() + utils.db_cleanup(immediate=True) + def test_component_params(self): # Make sure the ComponentParams table is empty self.assertEqual(ComponentSession.objects.count(), 0) @@ -20,7 +24,9 @@ def test_component_params(self): # Check if a component params are in the database self.assertEqual(ComponentSession.objects.count(), 1) - self.assertEqual(pickle.loads(ComponentSession.objects.first().params), params_1) # type: ignore + self.assertEqual( + pickle.loads(ComponentSession.objects.first().params), params_1 # type: ignore + ) # Force `params_1` to expire from reactpy_django import config @@ -37,7 +43,9 @@ def test_component_params(self): # Make sure `params_1` has expired self.assertEqual(ComponentSession.objects.count(), 1) - self.assertEqual(pickle.loads(ComponentSession.objects.first().params), params_2) # type: ignore + self.assertEqual( + pickle.loads(ComponentSession.objects.first().params), params_2 # type: ignore + ) def _save_params_to_db(self, value: Any) -> ComponentParamData: db = list(self.databases)[0]