ingestion of test/sandbox courses (#2224)

* adding test mode field

* adding filters to pull in test_mode resources

* fixes

* update specs

* adding test

* adding test

* adding test for etl loading of test courses

* making sure test mode content files are embedded

* fixing tests

* fixing tests

* ensure test mode content files are indexed

* fixing test

* test fix

* mocking deindex_items

* making sure all etl commands process test_mode appropriately

* making run filters look for published | test_mode=True

* fixing migration

* fetching all courses for mitxonline and moving published logic to transform_course

* fixing tests

* fixing flag for published programs and fixing tests

* fixing cases where fields dont exist

* fixing filter for fetch courses by id

* fixing published conditional

* fixing test

Author: shanbady

frontends/api/src/generated/v0/api.ts

@@ -6,6 +6,8 @@
 from tarfile import ReadError
 from tempfile import TemporaryDirectory
 
+from django.db.models import Q
+
 from learning_resources.etl.constants import ETLSource
 from learning_resources.etl.loaders import load_content_files
 from learning_resources.etl.utils import (
@@ -109,12 +111,18 @@ def sync_edx_course_files(
         matches = re.search(rf"{s3_prefix}/(.+)\.tar\.gz$", key)
         run_id = matches.group(1).split("/")[-1]
         log.info("Run is is %s", run_id)
-        runs = LearningResourceRun.objects.filter(
-            learning_resource__etl_source=etl_source,
-            learning_resource_id__in=ids,
-            learning_resource__published=True,
-            published=True,
+        runs = (
+            LearningResourceRun.objects.filter(
+                learning_resource__etl_source=etl_source,
+                learning_resource_id__in=ids,
+            )
+            .filter(Q(published=True) | Q(learning_resource__test_mode=True))
+            .filter(
+                Q(learning_resource__published=True)
+                | Q(learning_resource__test_mode=True)
+            )
         )
+
         if etl_source == ETLSource.mit_edx.name:
             # Additional processing of run ids and tarfile names,
             # because edx data is structured differently
@@ -142,7 +150,11 @@ def sync_edx_course_files(
         resource = run.learning_resource
         if run != (
             resource.next_run
-            or resource.runs.filter(published=True).order_by("-start_date").first()
+            or resource.runs.filter(
+                Q(published=True) | Q(learning_resource__test_mode=True)
+            )
+            .order_by("-start_date")
+            .first()
         ):
             log.info("Skipping %s, not the next / most recent run", run.run_id)
             continue