Better channel restoration for local dev with content proxying

.run/devserver.run.xml

@@ -13,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="false" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="contentcuration/manage.py" />
-    <option name="PARAMETERS" value="runserver --settings=contentcuration.dev_settings 0.0.0.0:8080" />
+    <option name="PARAMETERS" value="runserver --settings=contentcuration.dev_settings 0.0.0.0:8081" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />

Makefile

@@ -1,9 +1,13 @@
-# standalone install method
-DOCKER_COMPOSE = docker-compose
+SHELL := /bin/bash
 
-# support new plugin installation for docker-compose
-ifeq (, $(shell which docker-compose))
+# new plugin installation method for docker compose
 DOCKER_COMPOSE = docker compose
+
+WEBPACK_CMD = $(if $(filter $(WEBPACK_MODE),hot),pnpm run build:dev:hot, pnpm run build:dev)
+
+# support fallback to old docker-compose
+ifeq (, $(shell $DOCKER_COMPOSE version 2>/dev/null))
+	DOCKER_COMPOSE = docker-compose
 endif
 
 ###############################################################
@@ -133,9 +137,6 @@ dummyusers:
 	cd contentcuration/ && python manage.py loaddata contentcuration/fixtures/admin_user.json
 	cd contentcuration/ && python manage.py loaddata contentcuration/fixtures/admin_user_token.json
 
-hascaptions:
-	python contentcuration/manage.py set_orm_based_has_captions
-
 BRANCH_NAME := $(shell git rev-parse --abbrev-ref HEAD | sed 's/[^a-zA-Z0-9_-]/-/g')
 
 export COMPOSE_PROJECT_NAME=studio_$(BRANCH_NAME)
@@ -149,9 +150,31 @@ destroy-and-recreate-database: purge-postgres setup
 devceleryworkers:
 	$(MAKE) -e DJANGO_SETTINGS_MODULE=contentcuration.dev_settings prodceleryworkers
 
-run-services:
+devrun-django:
+	python contentcuration/manage.py runserver --settings=contentcuration.dev_settings 0.0.0.0:8081
+
+devrun-server:
+	set -ex; \
+	function _on_interrupt() { $(DOCKER_COMPOSE) stop studio-nginx; }; \
+	trap _on_interrupt SIGINT SIGTERM SIGKILL ERR; \
+	$(DOCKER_COMPOSE) up -d studio-nginx; \
+	$(MAKE) -j 2 devrun-django devrun-webpack
+
+devrun-server-hot:
+	$(MAKE) -e devrun-server WEBPACK_MODE=hot
+
+devrun-services:
 	$(MAKE) -j 2 dcservicesup devceleryworkers
 
+devrun-setup:
+	python contentcuration/manage.py setup --settings=contentcuration.dev_settings
+
+devrun-shell:
+	python contentcuration/manage.py shell --settings=contentcuration.dev_settings
+
+devrun-webpack:
+	$(WEBPACK_CMD)
+
 .docker/minio:
 	mkdir -p $@
 
@@ -172,7 +195,7 @@ dcbuild:
 
 dcup: .docker/minio .docker/postgres
 	# run all services except for cloudprober
-	$(DOCKER_COMPOSE) up studio-app celery-worker
+	$(DOCKER_COMPOSE) up studio-nginx studio-app
 
 dcup-cloudprober: .docker/minio .docker/postgres
 	# run all services including cloudprober
@@ -200,8 +223,8 @@ dctest: .docker/minio .docker/postgres
 
 dcservicesup: .docker/minio .docker/postgres
 	# launch all studio's dependent services using docker-compose
-	$(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.alt.yml up minio postgres redis
+	$(DOCKER_COMPOSE) up minio postgres redis
 
 dcservicesdown:
 	# stop services that were started using dcservicesup
-	$(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.alt.yml down
+	$(DOCKER_COMPOSE) down

contentcuration/contentcuration/management/commands/restore_channel.py

@@ -2,28 +2,65 @@
 
 from django.core.management.base import BaseCommand
 
-from contentcuration.utils.import_tools import import_channel
+from contentcuration.utils.import_tools import ImportManager
 
-logger = logging.getLogger('command')
+logger = logging.getLogger("command")
 
 
 class Command(BaseCommand):
+    """
+    This command is used to restore a channel from another Studio instance. This is for
+    development purposes only and should not be used in production.
+    """
 
     def add_arguments(self, parser):
         # ID of channel to read data from
-        parser.add_argument('source_id', type=str)
+        parser.add_argument("source_id", type=str)
 
         # ID of channel to write data to (can be same as source channel)
-        parser.add_argument('--target', help='restore channel db to TARGET CHANNEL ID')
-        parser.add_argument('--download-url', help='where to download db from')
-        parser.add_argument('--editor', help='add user as editor to channel')
+        parser.add_argument(
+            "--target",
+            help="A different channel ID for which to restore the channel. If not provided, the source channel ID will be used.",
+        )
+        parser.add_argument(
+            "--source-url",
+            default="http://localhost:8080",
+            help="Studio instance from which to download the channel DB or content files",
+        )
+        parser.add_argument("--token", help="API token for the Studio instance")
+        parser.add_argument(
+            "--editor",
+            default="[email protected]",
+            help="Add user as editor to channel with provided email address",
+        )
+        parser.add_argument(
+            "--download-content",
+            action="store_true",
+            default=False,
+            help="Whether to download content files",
+        )
+        parser.add_argument(
+            "--public",
+            action="store_true",
+            default=False,
+            help="Whether to make the channel public",
+        )
+        parser.add_argument(
+            "--publish",
+            action="store_true",
+            default=False,
+            help="Whether to publish the channel after restoration",
+        )
 
     def handle(self, *args, **options):
-        # Set up variables for restoration process
-        logger.info("\n\n********** STARTING CHANNEL RESTORATION **********")
-        source_id = options['source_id']
-        target_id = options.get('target') or source_id
-        download_url = options.get('download_url')
-        editor = options.get('editor')
-
-        import_channel(source_id, target_id, download_url, editor, logger=logger)
+        manager = ImportManager(
+            options["source_url"],
+            options["source_id"],
+            target_id=options.get("target"),
+            editor=options.get("editor"),
+            public=options.get("public"),
+            publish=options.get("publish"),
+            token=options.get("token"),
+            download_content=options.get("download_content"),
+        )
+        manager.run()

contentcuration/contentcuration/management/commands/set_content_mimetypes.py

@@ -14,7 +14,7 @@
 from django.core.files.storage import default_storage
 from django.core.management.base import BaseCommand
 
-from contentcuration.utils.storage_common import determine_content_type
+from contentcuration.utils.storage.common import determine_content_type
 
 
 class Command(BaseCommand):

contentcuration/contentcuration/models.py

@@ -2,7 +2,6 @@
 import json
 import logging
 import os
-import urllib.parse
 import uuid
 from datetime import datetime
 
@@ -581,44 +580,9 @@ def generate_storage_url(filename, request=None, *args):
 
     path = generate_object_storage_name(os.path.splitext(filename)[0], filename)
 
-    # There are three scenarios where Studio might be run as:
-    #
-    # 1. In normal kubernetes, nginx will proxy for us. We'll know we're in kubernetes when the
-    # environment variable RUN_MODE=k8s
-    #
-    # 2. In Docker Compose and bare metal runserver, we'll be running in runserver, and minio
-    # will be exposed in port 9000 in the host's localhost network.
-
-    # Note (aron): returning the true storage URL (e.g. https://storage.googleapis.com/storage/a.mp4)
-    # isn't too important, because we have CDN in front of our servers, so it should be cached.
-    # But change the logic here in case there is a potential for bandwidth and latency improvement.
-
-    # Detect our current state first
-    run_mode = os.getenv("RUN_MODE")
-
-    # if we're running inside k8s, then just serve the normal /content/{storage,databases} URL,
-    # and let nginx handle proper proxying.
-    if run_mode == "k8s":
-        url = "/content/{path}".format(
-            path=path,
-        )
-
-    # if we're in docker-compose or in baremetal, just return the object storage URL as localhost:9000
-    elif run_mode == "docker-compose" or run_mode is None:
-        # generate the minio storage URL, so we can get the GET parameters that give everyone
-        # access even if they don't need to log in
-        params = urllib.parse.urlparse(default_storage.url(path)).query
-        host = "localhost"
-        port = 9000  # hardcoded to the default minio IP address
-        url = "http://{host}:{port}/{bucket}/{path}?{params}".format(
-            host=host,
-            port=port,
-            bucket=settings.AWS_S3_BUCKET_NAME,
-            path=path,
-            params=params,
-        )
-
-    return url
+    return "/content/{path}".format(
+        path=path,
+    )
 
 
 class FileOnDiskStorage(FileSystemStorage):

contentcuration/contentcuration/production_settings.py

@@ -10,7 +10,7 @@
 
 MEDIA_ROOT = base_settings.STORAGE_ROOT
 
-DEFAULT_FILE_STORAGE = 'contentcuration.utils.gcs_storage.CompositeGCS'
+DEFAULT_FILE_STORAGE = 'contentcuration.utils.storage.gcs.CompositeGCS'
 SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
 # email settings

contentcuration/contentcuration/sandbox_settings.py

@@ -3,7 +3,7 @@
 
 DEBUG = True
 
-DEFAULT_FILE_STORAGE = "contentcuration.utils.gcs_storage.CompositeGCS"
+DEFAULT_FILE_STORAGE = "contentcuration.utils.storage.gcs.CompositeGCS"
 
 LANGUAGES += (("ar", gettext("Arabic")),)  # noqa
 

contentcuration/contentcuration/settings.py

@@ -361,7 +361,7 @@ def gettext(s):
 ORPHAN_DATE_CLEAN_UP_THRESHOLD = TWO_WEEKS_AGO
 
 # CLOUD STORAGE SETTINGS
-DEFAULT_FILE_STORAGE = 'django_s3_storage.storage.S3Storage'
+DEFAULT_FILE_STORAGE = 'contentcuration.utils.storage.dev.CompositeStorage'
 AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') or 'development'
 AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') or 'development'
 AWS_S3_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME') or 'content'