netbox-community · jeremystretch · Jul 8, 2021 · Jul 7, 2021 · Jul 7, 2021 · Jul 8, 2021
diff --git a/base_requirements.txt b/base_requirements.txt
@@ -2,10 +2,6 @@
 # https://github.com/django/django
 Django
 
-# Django caching using Redis
-# https://github.com/Suor/django-cacheops
-django-cacheops
-
 # Django middleware which permits cross-domain API requests
 # https://github.com/OttoYiu/django-cors-headers
 django-cors-headers
@@ -34,6 +30,10 @@ django-pglocks
 # https://github.com/korfuri/django-prometheus
 django-prometheus
 
+# Django chaching backend using Redis
+# https://github.com/jazzband/django-redis
+django-redis
+
 # Django integration for RQ (Reqis queuing)
 # https://github.com/rq/django-rq
 django-rq

diff --git a/docs/additional-features/caching.md b/docs/additional-features/caching.md
diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md
@@ -52,14 +52,6 @@ BASE_PATH = 'netbox/'
 
 ---
 
-## CACHE_TIMEOUT
-
-Default: 900
-
-The number of seconds that cache entries will be retained before expiring.
-
----
-
 ## CHANGELOG_RETENTION
 
 Default: 90

diff --git a/docs/plugins/development.md b/docs/plugins/development.md
@@ -113,7 +113,6 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
 | `min_version` | Minimum version of NetBox with which the plugin is compatible |
 | `max_version` | Maximum version of NetBox with which the plugin is compatible |
 | `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
-| `caching_config` | Plugin-specific cache configuration
 | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
 | `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
 
@@ -384,32 +383,4 @@ class SiteAnimalCount(PluginTemplateExtension):
         })
 
 template_extensions = [SiteAnimalCount]
-```
-
-## Caching Configuration
-
-By default, all query operations within a plugin are cached. To change this, define a caching configuration under the PluginConfig class' `caching_config` attribute. All configuration keys will be applied within the context of the plugin; there is no need to include the plugin name. An example configuration is below:
-
-```python
-class MyPluginConfig(PluginConfig):
-    ...
-    caching_config = {
-        'foo': {
-            'ops': 'get',
-            'timeout': 60 * 15,
-        },
-        '*': {
-            'ops': 'all',
-        }
-    }
-```
-
-To disable caching for your plugin entirely, set:
-
-```python
-caching_config = {
-    '*': None
-}
-```
-
-See the [django-cacheops](https://github.com/Suor/django-cacheops) documentation for more detail on configuring caching.
+```
diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md
@@ -5,6 +5,9 @@
 ### Breaking Changes
 
 * The default CSV export format for all objects now includes all available data. Additionally, the CSV headers now use human-friendly titles rather than the raw field names.
+* Support for queryset caching configuration (`caching_config`) has been removed from the plugins API (see [#6639](https://github.com/netbox-community/netbox/issues/6639)).
+* The `cacheops_*` metrics have been removed from the Prometheus exporter (see [#6639](https://github.com/netbox-community/netbox/issues/6639)).
+* The `invalidate` management command has been removed.
 
 ### New Features
 
@@ -64,6 +67,11 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi
 * [#5994](https://github.com/netbox-community/netbox/issues/5994) - Drop support for `display_field` argument on ObjectVar
 * [#6068](https://github.com/netbox-community/netbox/issues/6068) - Drop support for legacy static CSV export
 * [#6338](https://github.com/netbox-community/netbox/issues/6338) - Decimal fields are no longer coerced to strings in REST API
+* [#6639](https://github.com/netbox-community/netbox/issues/6639) - Drop support for queryset caching (django-cacheops)
+
+### Configuration Changes
+
+* The `CACHE_TIMEOUT` configuration parameter has been removed.
 
 ### REST API Changes
 

diff --git a/netbox/dcim/signals.py b/netbox/dcim/signals.py
@@ -1,6 +1,5 @@
 import logging
 
-from cacheops import invalidate_obj
 from django.contrib.contenttypes.models import ContentType
 from django.db.models.signals import post_save, post_delete, pre_delete
 from django.db import transaction
@@ -33,7 +32,6 @@ def rebuild_paths(obj):
         for cp in cable_paths:
             cp.delete()
             if cp.origin:
-                invalidate_obj(cp.origin)
                 create_cablepath(cp.origin)
 
 

diff --git a/netbox/extras/management/commands/renaturalize.py b/netbox/extras/management/commands/renaturalize.py
@@ -1,4 +1,3 @@
-from cacheops import invalidate_model
 from django.apps import apps
 from django.core.management.base import BaseCommand, CommandError
 
@@ -108,8 +107,5 @@ def handle(self, *args, **options):
                 elif options['verbosity']:
                     self.stdout.write(self.style.SUCCESS(str(count)))
 
-            # Invalidate cached queries
-            invalidate_model(model)
-
         if options['verbosity']:
             self.stdout.write(self.style.SUCCESS("Done."))
diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py
@@ -47,11 +47,6 @@ class PluginConfig(AppConfig):
     # Middleware classes provided by the plugin
     middleware = []
 
-    # Cacheops configuration. Cache all operations by default.
-    caching_config = {
-        '*': {'ops': 'all'},
-    }
-
     # Default integration paths. Plugin authors can override these to customize the paths to
     # integrated components.
     template_extensions = 'template_content.template_extensions'

diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py
@@ -1,4 +1,3 @@
-from cacheops.signals import cache_invalidated, cache_read
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.db.models.signals import m2m_changed, post_save, pre_delete
@@ -138,27 +137,3 @@ def run_custom_validators(sender, instance, **kwargs):
     validators = settings.CUSTOM_VALIDATORS.get(model_name, [])
     for validator in validators:
         validator(instance)
-
-
-#
-# Caching
-#
-
-cacheops_cache_hit = Counter('cacheops_cache_hit', 'Number of cache hits')
-cacheops_cache_miss = Counter('cacheops_cache_miss', 'Number of cache misses')
-cacheops_cache_invalidated = Counter('cacheops_cache_invalidated', 'Number of cache invalidations')
-
-
-def cache_read_collector(sender, func, hit, **kwargs):
-    if hit:
-        cacheops_cache_hit.inc()
-    else:
-        cacheops_cache_miss.inc()
-
-
-def cache_invalidated_collector(sender, obj_dict, **kwargs):
-    cacheops_cache_invalidated.inc()
-
-
-cache_read.connect(cache_read_collector)
-cache_invalidated.connect(cache_invalidated_collector)
diff --git a/netbox/extras/tests/test_plugins.py b/netbox/extras/tests/test_plugins.py
@@ -80,12 +80,6 @@ def test_middleware(self):
         """
         self.assertIn('extras.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
 
-    def test_caching_config(self):
-        """
-        Check that plugin caching configuration is registered.
-        """
-        self.assertIn('extras.tests.dummy_plugin.*', settings.CACHEOPS)
-
     def test_min_version(self):
         """
         Check enforcement of minimum NetBox version.

diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py
@@ -89,9 +89,6 @@
 # BASE_PATH = 'netbox/'
 BASE_PATH = ''
 
-# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
-CACHE_TIMEOUT = 900
-
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
 CHANGELOG_RETENTION = 90
 

diff --git a/netbox/netbox/releases.py b/netbox/netbox/releases.py
@@ -1,7 +1,7 @@
 import logging
 
-from cacheops import CacheMiss, cache
 from django.conf import settings
+from django.core.cache import cache
 from django_rq import get_queue
 
 from utilities.background_tasks import get_releases
@@ -12,12 +12,11 @@
 def get_latest_release(pre_releases=False):
     if settings.RELEASE_CHECK_URL:
         logger.debug("Checking for most recent release")
-        try:
-            latest_release = cache.get('latest_release')
-            if latest_release:
-                logger.debug("Found cached release: {}".format(latest_release))
-                return latest_release
-        except CacheMiss:
+        latest_release = cache.get('latest_release')
+        if latest_release:
+            logger.debug(f"Found cached release: {latest_release}")
+            return latest_release
+        else:
             # Check for an existing job. This can happen if the RQ worker process is not running.
             queue = get_queue('check_releases')
             if queue.jobs:

diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py
@@ -45,6 +45,10 @@
         )
     raise
 
+# Warn on removed config parameters
+if hasattr(configuration, 'CACHE_TIMEOUT'):
+    warnings.warn("The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect.")
+
 # Enforce required configuration parameters
 for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']:
     if not hasattr(configuration, parameter):
@@ -69,7 +73,6 @@
 BASE_PATH = getattr(configuration, 'BASE_PATH', '')
 if BASE_PATH:
     BASE_PATH = BASE_PATH.strip('/') + '/'  # Enforce trailing slash only
-CACHE_TIMEOUT = getattr(configuration, 'CACHE_TIMEOUT', 900)
 CHANGELOG_RETENTION = getattr(configuration, 'CHANGELOG_RETENTION', 90)
 CORS_ORIGIN_ALLOW_ALL = getattr(configuration, 'CORS_ORIGIN_ALLOW_ALL', False)
 CORS_ORIGIN_REGEX_WHITELIST = getattr(configuration, 'CORS_ORIGIN_REGEX_WHITELIST', [])
@@ -225,19 +228,31 @@ def _setting(name, default=None):
     raise ImproperlyConfigured(
         "REDIS section in configuration.py is missing caching subsection."
     )
-CACHING_REDIS = REDIS['caching']
-CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost')
-CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379)
-CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', [])
-CACHING_REDIS_USING_SENTINEL = all([
-    isinstance(CACHING_REDIS_SENTINELS, (list, tuple)),
-    len(CACHING_REDIS_SENTINELS) > 0
-])
-CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default')
-CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '')
-CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0)
-CACHING_REDIS_SSL = CACHING_REDIS.get('SSL', False)
-CACHING_REDIS_SKIP_TLS_VERIFY = CACHING_REDIS.get('INSECURE_SKIP_TLS_VERIFY', False)
+CACHING_REDIS_HOST = REDIS['caching'].get('HOST', 'localhost')
+CACHING_REDIS_PORT = REDIS['caching'].get('PORT', 6379)
+CACHING_REDIS_DATABASE = REDIS['caching'].get('DATABASE', 0)
+CACHING_REDIS_PASSWORD = REDIS['caching'].get('PASSWORD', '')
+CACHING_REDIS_SENTINELS = REDIS['caching'].get('SENTINELS', [])
+CACHING_REDIS_SENTINEL_SERVICE = REDIS['caching'].get('SENTINEL_SERVICE', 'default')
+CACHING_REDIS_PROTO = 'rediss' if REDIS['caching'].get('SSL', False) else 'redis'
+CACHING_REDIS_SKIP_TLS_VERIFY = REDIS['caching'].get('INSECURE_SKIP_TLS_VERIFY', False)
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': f'{CACHING_REDIS_PROTO}://{CACHING_REDIS_HOST}:{CACHING_REDIS_PORT}/{CACHING_REDIS_DATABASE}',
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+            'PASSWORD': CACHING_REDIS_PASSWORD,
+        }
+    }
+}
+if CACHING_REDIS_SENTINELS:
+    CACHES['default']['LOCATION'] = f'{CACHING_REDIS_PROTO}://{CACHING_REDIS_SENTINEL_SERVICE}/{CACHING_REDIS_DATABASE}'
+    CACHES['default']['OPTIONS']['CLIENT_CLASS'] = 'django_redis.client.SentinelClient'
+    CACHES['default']['OPTIONS']['SENTINELS'] = CACHING_REDIS_SENTINELS
+if CACHING_REDIS_SKIP_TLS_VERIFY:
+    CACHES['default']['OPTIONS']['CONNECTION_POOL_KWARGS']['ssl_cert_reqs'] = False
 
 
 #
@@ -280,7 +295,6 @@ def _setting(name, default=None):
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
-    'cacheops',
     'corsheaders',
     'debug_toolbar',
     'graphiql_debug_toolbar',
@@ -396,53 +410,6 @@ def _setting(name, default=None):
     ('users', 'objectpermission'),
 )
 
-#
-# Caching
-#
-if CACHING_REDIS_USING_SENTINEL:
-    CACHEOPS_SENTINEL = {
-        'locations': CACHING_REDIS_SENTINELS,
-        'service_name': CACHING_REDIS_SENTINEL_SERVICE,
-        'db': CACHING_REDIS_DATABASE,
-        'password': CACHING_REDIS_PASSWORD,
-    }
-else:
-    CACHEOPS_REDIS = {
-        'host': CACHING_REDIS_HOST,
-        'port': CACHING_REDIS_PORT,
-        'db': CACHING_REDIS_DATABASE,
-        'password': CACHING_REDIS_PASSWORD,
-        'ssl': CACHING_REDIS_SSL,
-        'ssl_cert_reqs': None if CACHING_REDIS_SKIP_TLS_VERIFY else 'required',
-    }
-
-if not CACHE_TIMEOUT:
-    CACHEOPS_ENABLED = False
-else:
-    CACHEOPS_ENABLED = True
-
-
-CACHEOPS_DEFAULTS = {
-    'timeout': CACHE_TIMEOUT
-}
-CACHEOPS = {
-    'auth.user': {'ops': 'get', 'timeout': 60 * 15},
-    'auth.*': {'ops': ('fetch', 'get')},
-    'auth.permission': {'ops': 'all'},
-    'circuits.*': {'ops': 'all'},
-    'dcim.inventoryitem': None,  # MPTT models are exempt due to raw SQL
-    'dcim.region': None,  # MPTT models are exempt due to raw SQL
-    'dcim.location': None,  # MPTT models are exempt due to raw SQL
-    'dcim.*': {'ops': 'all'},
-    'ipam.*': {'ops': 'all'},
-    'extras.*': {'ops': 'all'},
-    'users.*': {'ops': 'all'},
-    'tenancy.tenantgroup': None,  # MPTT models are exempt due to raw SQL
-    'tenancy.*': {'ops': 'all'},
-    'virtualization.*': {'ops': 'all'},
-}
-CACHEOPS_DEGRADE_ON_FAILURE = True
-
 
 #
 # Django Prometheus
@@ -632,12 +599,3 @@ def _setting(name, default=None):
     plugin_middleware = plugin_config.middleware
     if plugin_middleware and type(plugin_middleware) in (list, tuple):
         MIDDLEWARE.extend(plugin_middleware)
-
-    # Apply cacheops config
-    if type(plugin_config.caching_config) is not dict:
-        raise ImproperlyConfigured(
-            "Plugin {} caching_config must be a dictionary.".format(plugin_name)
-        )
-    CACHEOPS.update({
-        "{}.{}".format(plugin_name, key): value for key, value in plugin_config.caching_config.items()
-    })