AnalyticsRollupCommentsReferrerDaily (#2802)

Co-authored-by: Your Name <[email protected]>

Author: peterbe

peterbecom/api/analytics.py

@@ -72,6 +72,13 @@ def query(request):
                 q,
             )
 
+        elif table == "analyticsrollupcommentsreferrerdaily":
+            q = re.sub(
+                r"\banalyticsrollupcommentsreferrerdaily\b",
+                "base_analyticsrollupcommentsreferrerdaily",
+                q,
+            )
+
         elif table == "requestlog":
             q = re.sub(r"\brequestlog\b", "base_requestlog", q)
 

peterbecom/base/migrations/0027_analyticsrollupcommentsreferrerdaily.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.2.7 on 2025-10-26 00:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0026_requestlogrollupsbotagentstatuscodedaily'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AnalyticsRollupCommentsReferrerDaily',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('day', models.DateTimeField(db_index=True)),
+                ('count', models.IntegerField()),
+                ('referrer', models.URLField()),
+                ('pathname', models.CharField(max_length=300)),
+                ('is_bot', models.BooleanField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                'verbose_name': 'Analytics Rollup Comments by Referrer and Pathname daily',
+            },
+        ),
+    ]

peterbecom/base/models.py

@@ -291,6 +291,65 @@ def rollup(cls, day=None):
             cls.objects.bulk_create(bulk)
 
 
+class AnalyticsRollupCommentsReferrerDaily(models.Model):
+    day = models.DateTimeField(db_index=True)
+    count = models.IntegerField()
+    referrer = models.URLField()
+    pathname = models.CharField(max_length=300)
+    is_bot = models.BooleanField()
+    created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        verbose_name = "Analytics Rollup Comments by Referrer and Pathname daily"
+
+    @classmethod
+    def rollup(cls, day=None):
+        if not day:
+            # Use yesterday
+            day = timezone.now() - datetime.timedelta(days=1)
+
+        start_of_day = day.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_of_day = start_of_day + datetime.timedelta(days=1)
+
+        day = start_of_day
+        print(f" ROLLUP BY PATHNAME DAY: {day.isoformat()} ".center(80, "-"))
+        with transaction.atomic():
+            cls.objects.filter(day=day).delete()
+
+            agg_query = (
+                AnalyticsEvent.objects.filter(
+                    created__gte=start_of_day,
+                    created__lt=end_of_day,
+                    type="pageview",
+                    data__is_comment=True,
+                    # data__referrer__isnull=False,
+                    data__pathname__isnull=False,
+                    data__is_bot__isnull=False,
+                )
+                .values("data__referrer", "data__pathname", "data__is_bot")
+                .annotate(count=Count("id"))
+                .order_by("-count")
+            )
+            bulk = []
+            for agg in agg_query:
+                print(
+                    f"{agg['count']:>5} {agg['data__referrer'] or '':<12} {agg['data__pathname']:<12} is_bot={agg['data__is_bot']}"
+                )
+                bulk.append(
+                    cls(
+                        day=day,
+                        count=agg["count"],
+                        referrer=agg["data__referrer"] or "",
+                        is_bot=agg["data__is_bot"],
+                        pathname=agg["data__pathname"],
+                    )
+                )
+                if len(bulk) > 100:
+                    cls.objects.bulk_create(bulk)
+                    bulk = []
+            cls.objects.bulk_create(bulk)
+
+
 class AnalyticsGeoEvent(models.Model):
     event = models.OneToOneField(AnalyticsEvent, on_delete=models.CASCADE)
     ip_address = models.GenericIPAddressField()

peterbecom/base/tasks.py

@@ -19,6 +19,7 @@
 from peterbecom.base.cdn import purge_cdn_urls
 from peterbecom.base.models import (
     AnalyticsEvent,
+    AnalyticsRollupCommentsReferrerDaily,
     AnalyticsRollupsDaily,
     AnalyticsRollupsPathnameDaily,
     CDNPurgeURL,
@@ -221,24 +222,30 @@ def delete_old_analyticsevents():
     AnalyticsEvent.objects.filter(created__lt=old).delete()
 
     # The publicapi-pageview ones are much more numerous and less
-    # useful and use up a lot of space
+    # useful and use up a lot of spacecd
     old = timezone.now() - datetime.timedelta(days=60)
     AnalyticsEvent.objects.filter(created__lt=old, type="publicapi-pageview").delete()
 
 
-@periodic_task(crontab(hour="*/15") if settings.DEBUG else crontab(hour=0, minute=1))
+@periodic_task(crontab(minute="*/15") if settings.DEBUG else crontab(hour=0, minute=1))
 @log_task_run
 def analytics_rollups_daily():
     AnalyticsRollupsDaily.rollup()
 
 
-@periodic_task(crontab(hour="*/15") if settings.DEBUG else crontab(hour=0, minute=0))
+@periodic_task(crontab(minute="*/15") if settings.DEBUG else crontab(hour=0, minute=0))
 @log_task_run
 def analytics_rollups_pathname_daily():
     AnalyticsRollupsPathnameDaily.rollup()
 
 
-@periodic_task(crontab(hour="*/15") if settings.DEBUG else crontab(hour=10))
+@periodic_task(crontab(minute="*/15") if settings.DEBUG else crontab(hour="*/12"))
+@log_task_run
+def analytics_rollup_comments_referrer_pathname_daily():
+    AnalyticsRollupCommentsReferrerDaily.rollup()
+
+
+@periodic_task(crontab(minute="*/15") if settings.DEBUG else crontab(hour=10))
 @log_task_run
 def requestlog_rollups_bot_agent_status_code_daily():
     RequestLogRollupsBotAgentStatusCodeDaily.rollup()