adding filters to pull in test_mode resources

Author: shanbady

learning_resources/etl/edx_shared.py

@@ -6,6 +6,8 @@
 from tarfile import ReadError
 from tempfile import TemporaryDirectory
 
+from django.db.models import Q
+
 from learning_resources.etl.constants import ETLSource
 from learning_resources.etl.loaders import load_content_files
 from learning_resources.etl.utils import (
@@ -112,9 +114,11 @@ def sync_edx_course_files(
         runs = LearningResourceRun.objects.filter(
             learning_resource__etl_source=etl_source,
             learning_resource_id__in=ids,
-            learning_resource__published=True,
             published=True,
+        ).filter(
+            Q(learning_resource__published=True) | Q(learning_resource__test_mode=True)
         )
+
         if etl_source == ETLSource.mit_edx.name:
             # Additional processing of run ids and tarfile names,
             # because edx data is structured differently

learning_resources/management/commands/backpopulate_mit_edx_files.py

@@ -3,16 +3,18 @@
 from django.conf import settings
 from django.core.management import BaseCommand
 
+from learning_resources.management.commands.mixins import BaseCommandMixin
 from learning_resources.tasks import import_all_mit_edx_files
 from main.utils import now_in_utc
 
 
-class Command(BaseCommand):
+class Command(BaseCommandMixin, BaseCommand):
     """Populate MIT edX course run files"""
 
     help = "Populate MIT edX course run files"
 
     def add_arguments(self, parser):
+        super().add_arguments(parser)
         parser.add_argument(
             "-c",
             "--chunk-size",
@@ -43,6 +45,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             if options["learning_resource_ids"]
             else None
         )
+        self.configure_test_resources(options)
         task = import_all_mit_edx_files.delay(
             chunk_size=chunk_size,
             overwrite=options["force_overwrite"],

learning_resources/management/commands/backpopulate_mitxonline_files.py

@@ -2,17 +2,19 @@
 
 from django.core.management import BaseCommand
 
+from learning_resources.management.commands.mixins import BaseCommandMixin
 from learning_resources.tasks import import_all_mitxonline_files
 from main import settings
 from main.utils import now_in_utc
 
 
-class Command(BaseCommand):
+class Command(BaseCommandMixin, BaseCommand):
     """Populate mitxonline course run files"""
 
     help = "Populate mitxonline course run files"
 
     def add_arguments(self, parser):
+        super().add_arguments(parser)
         parser.add_argument(
             "-c",
             "--chunk-size",
@@ -43,6 +45,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             if options["learning_resource_ids"]
             else None
         )
+        self.configure_test_resources(options)
         task = import_all_mitxonline_files.delay(
             chunk_size=chunk_size,
             overwrite=options["force_overwrite"],

learning_resources/management/commands/backpopulate_oll_files.py

@@ -2,12 +2,13 @@
 
 from django.core.management import BaseCommand
 
+from learning_resources.management.commands.mixins import BaseCommandMixin
 from learning_resources.tasks import import_all_oll_files
 from main import settings
 from main.utils import now_in_utc
 
 
-class Command(BaseCommand):
+class Command(BaseCommandMixin, BaseCommand):
     """Populate OLL course run files"""
 
     help = "Populate OLL course run files"
@@ -46,6 +47,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             if options["learning_resource_ids"]
             else None
         )
+        self.configure_test_resources(options)
         task = import_all_oll_files.delay(
             chunk_size=chunk_size,
             overwrite=options["force_overwrite"],

learning_resources/management/commands/backpopulate_xpro_files.py

@@ -3,16 +3,18 @@
 from django.conf import settings
 from django.core.management import BaseCommand
 
+from learning_resources.management.commands.mixins import BaseCommandMixin
 from learning_resources.tasks import import_all_xpro_files
 from main.utils import now_in_utc
 
 
-class Command(BaseCommand):
+class Command(BaseCommandMixin, BaseCommand):
     """Populate xpro course run files"""
 
     help = "Populate xpro course run files"
 
     def add_arguments(self, parser):
+        super().add_arguments(parser)
         parser.add_argument(
             "-c",
             "--chunk-size",
@@ -42,6 +44,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             if options["learning_resource_ids"]
             else None
         )
+        self.configure_test_resources(options)
         task = import_all_xpro_files.delay(
             chunk_size=chunk_size,
             overwrite=options["force_overwrite"],

learning_resources/management/commands/mixins.py

@@ -0,0 +1,17 @@
+from learning_resources.models import LearningResource
+
+
+class BaseCommandMixin:
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--test-ids",
+            dest="test_ids",
+            help="List of readable IDs to use for testing",
+        )
+
+    def configure_test_resources(self, options):
+        if options["test_ids"]:
+            test_ids = options["test_ids"].split(",")
+            LearningResource.objects.filter(id__in=test_ids).update(
+                test_mode=True, published=False
+            )

learning_resources/tasks.py

@@ -9,6 +9,7 @@
 import boto3
 import celery
 from django.conf import settings
+from django.db.models import Q
 from django.utils import timezone
 
 from learning_resources.content_summarizer import ContentSummarizer
@@ -52,7 +53,8 @@ def get_micromasters_data():
 
 @app.task
 def get_mit_edx_data(
-    api_course_datafile: str | None = None, api_program_datafile: str | None = None
+    api_course_datafile: str | None = None,
+    api_program_datafile: str | None = None,
 ) -> int:
     """Task to sync MIT edX data with the database
 
@@ -169,9 +171,8 @@ def get_content_tasks(  # noqa: PLR0913
         ).values_list("id", flat=True)
     else:
         learning_resources = (
-            LearningResource.objects.filter(
-                published=True, course__isnull=False, etl_source=etl_source
-            )
+            LearningResource.objects.filter(Q(published=True) | Q(test_mode=True))
+            .filter(course__isnull=False, etl_source=etl_source)
             .exclude(readable_id__in=blocklisted_ids)
             .order_by("-id")
             .values_list("id", flat=True)