embedding content updates and removals (#2217)

* adding method to remove qdrant points by filter params

* adding safety check

* adding method to remove points by db id

* ensuring ocw contentfiles have a checksum

* adding checksum to contentfile serializer

* ensuring we only generate embeddings if embedding context has changed

* enforce checksum on contentfile save

* adding checksum to param map

* reverting changes to pipeline

* adding separate archive checksum field

* upserting points and contentfile diffing

* adding missing migration

* update specs

* test fixes

* fixing test

* fixing tests

* addition of updating payloads

* adding more tests and task to remove embeddings from index

* adding plugin hooks for embeddings

* reverting additions to plugins

* adding plugin hooks

* fixing tests

* adding setting to toggle embedding tasks in plugin methods

* fix test

* add docstring

* setting archve_checksum from checksum value

* reorganizing call order

* removing archive_checksum from serializer

* fix tests

* separate methods for updating payloads

* regenerate spec

* fixing test to adjust for separate methods

* separate _embedding_context methods for learning resource and content file

* splitting methods for embedding contentfiles and learning resources

Author: shanbady

frontends/api/src/generated/v0/api.ts

@@ -124,7 +124,7 @@ def test_sync_edx_course_files_matching_checksum(
         "learning_resources_search.plugins.tasks.index_run_content_files"
     )
     mock_deindex = mocker.patch(
-        "learning_resources_search.plugins.tasks.deindex_run_content_files"
+        "learning_resources_search.plugins.tasks.deindex_run_content_files.si"
     )
     mock_log = mocker.patch("learning_resources.etl.edx_shared.log.info")
     mock_load = mocker.patch("learning_resources.etl.edx_shared.load_content_files")
@@ -142,7 +142,7 @@ def test_sync_edx_course_files_matching_checksum(
     )
     sync_edx_course_files("mitxonline", [run.learning_resource.id], [key])
     mock_log.assert_any_call("Checksums match for %s, skipping load", key)
-    mock_deindex.assert_called_once_with(other_run.id, False)  # noqa: FBT003
+    mock_deindex.assert_called_once_with(other_run.id, unpublished_only=False)
     mock_load.assert_not_called()
     mock_index.assert_not_called()
 

frontends/api/src/generated/v1/api.ts

@@ -132,8 +132,11 @@ def mock_upsert_tasks(mocker):
         deindex_learning_resource=mocker.patch(
             "learning_resources_search.tasks.deindex_document"
         ),
+        deindex_learning_resource_immutable_signature=mocker.patch(
+            "learning_resources_search.tasks.deindex_document.si"
+        ),
         batch_deindex_resources=mocker.patch(
-            "learning_resources_search.tasks.bulk_deindex_learning_resources"
+            "learning_resources_search.tasks.bulk_deindex_learning_resources.si"
         ),
     )
 
@@ -260,13 +263,15 @@ def test_load_program(  # noqa: PLR0913
         assert relationship.child.readable_id == data.learning_resource.readable_id
 
     if program_exists and not is_published:
-        mock_upsert_tasks.deindex_learning_resource.assert_called_with(
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_called_with(
             result.id, result.resource_type
         )
     elif is_published:
-        mock_upsert_tasks.upsert_learning_resource.assert_called_with(result.id)
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
+            result.id
+        )
     else:
-        mock_upsert_tasks.upsert_learning_resource.assert_not_called()
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_not_called()
 
 
 def test_load_program_bad_platform(mocker):
@@ -414,14 +419,16 @@ def test_load_course(  # noqa: PLR0913,PLR0912,PLR0915
     )
 
     if course_exists and ((not is_published or not is_run_published) or blocklisted):
-        mock_upsert_tasks.deindex_learning_resource.assert_called_with(
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_called_with(
             result.id, result.resource_type
         )
     elif is_published and is_run_published and not blocklisted:
-        mock_upsert_tasks.upsert_learning_resource.assert_called_with(result.id)
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
+            result.id
+        )
     else:
-        mock_upsert_tasks.deindex_learning_resource.assert_not_called()
-        mock_upsert_tasks.upsert_learning_resource.assert_not_called()
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_not_called()
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_not_called()
 
     if course_exists and is_published and not blocklisted:
         course.refresh_from_db()
@@ -545,11 +552,11 @@ def test_load_duplicate_course(
     result = load_course(props, [], duplicates)
 
     if course_id_is_duplicate and duplicate_course_exists:
-        mock_upsert_tasks.deindex_learning_resource.assert_called()
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_called()
     else:
-        mock_upsert_tasks.deindex_learning_resource.assert_not_called()
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_not_called()
     if course.learning_resource.id:
-        mock_upsert_tasks.upsert_learning_resource.assert_called_with(
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
             course.learning_resource.id
         )
 
@@ -925,10 +932,10 @@ def test_load_content_files(mocker, is_published, extra_run, calc_score):
         autospec=True,
     )
     mock_bulk_index = mocker.patch(
-        "learning_resources_search.plugins.tasks.index_run_content_files",
+        "learning_resources_search.plugins.tasks.index_run_content_files.si",
     )
     mock_bulk_delete = mocker.patch(
-        "learning_resources_search.plugins.tasks.deindex_run_content_files",
+        "learning_resources_search.plugins.tasks.deindex_run_content_files.si",
         autospec=True,
     )
     mock_calc_score = mocker.patch(
@@ -1108,13 +1115,15 @@ def test_load_podcast_episode(
         assert getattr(result, key) == value, f"Property {key} should equal {value}"
 
     if podcast_episode_exists and not is_published:
-        mock_upsert_tasks.deindex_learning_resource.assert_called_with(
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_called_with(
             result.id, result.resource_type
         )
     elif is_published:
-        mock_upsert_tasks.upsert_learning_resource.assert_called_with(result.id)
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
+            result.id
+        )
     else:
-        mock_upsert_tasks.upsert_learning_resource.assert_not_called()
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_not_called()
         mock_upsert_tasks.deindex_learning_resource.assert_not_called()
 
 
@@ -1183,14 +1192,16 @@ def test_load_podcast(
         assert new_podcast.resources.count() == 0
 
     if podcast_exists and not is_published:
-        mock_upsert_tasks.deindex_learning_resource.assert_called_with(
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_called_with(
             result.id, result.resource_type
         )
     elif is_published:
-        mock_upsert_tasks.upsert_learning_resource.assert_called_with(result.id)
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
+            result.id
+        )
     else:
-        mock_upsert_tasks.deindex_learning_resource.assert_not_called()
-        mock_upsert_tasks.upsert_learning_resource.assert_not_called()
+        mock_upsert_tasks.deindex_learning_resource_immutable_signature.assert_not_called()
+        mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_not_called()
 
 
 @pytest.mark.parametrize("video_exists", [True, False])
@@ -1322,7 +1333,7 @@ def test_load_playlist(mocker):
 
 def test_load_playlists_unpublish(mocker):
     """Test load_playlists when a video/playlist gets unpublished"""
-    mocker.patch("learning_resources_search.tasks.bulk_deindex_learning_resources")
+    mocker.patch("learning_resources_search.tasks.bulk_deindex_learning_resources.si")
     channel = VideoChannelFactory.create()
 
     playlists = sorted(
@@ -1498,7 +1509,9 @@ def test_load_course_percolation(
 
     blocklist = [learning_resource.readable_id] if blocklisted else []
     result = load_course(props, blocklist, [], config=CourseLoaderConfig(prune=True))
-    mock_upsert_tasks.upsert_learning_resource.assert_called_with(result.id)
+    mock_upsert_tasks.upsert_learning_resource_immutable_signature.assert_called_with(
+        result.id
+    )
 
 
 @pytest.mark.parametrize("certification", [True, False])

learning_resources/etl/edx_shared_test.py

@@ -331,14 +331,14 @@ def documents_from_olx(
                     filebytes = f.read()
 
                 mimetype = mimetypes.types_map.get(extension_lower)
-                checksum = md5(filebytes).hexdigest()  # noqa: S324
+                archive_checksum = md5(filebytes).hexdigest()  # noqa: S324
 
                 yield (
                     filebytes,
                     {
                         "content_type": CONTENT_TYPE_FILE,
                         "mime_type": mimetype,
-                        "checksum": checksum,
+                        "archive_checksum": archive_checksum,
                         "file_extension": extension_lower,
                         "source_path": f"{path}/{filename}",
                     },
@@ -434,7 +434,7 @@ def transform_content_files(
             existing_content = ContentFile.objects.filter(key=key, run=run).first()
             if (
                 not existing_content
-                or existing_content.checksum != metadata.get("checksum")
+                or existing_content.archive_checksum != metadata.get("archive_checksum")
             ) or overwrite:
                 if settings.SKIP_TIKA and settings.ENVIRONMENT != "production":
                     content_dict = {
@@ -478,7 +478,7 @@ def transform_content_files(
                     "key": key,
                     "published": True,
                     "content_type": content_type,
-                    "checksum": metadata.get("checksum"),
+                    "archive_checksum": metadata.get("archive_checksum"),
                     "file_extension": file_extension,
                     "source_path": source_path,
                     "edx_module_id": edx_module_id,

learning_resources/etl/loaders_test.py

@@ -182,7 +182,7 @@ def test_transform_content_files(  # noqa: PLR0913
     document = "some text in the document"
     file_extension = ".pdf" if folder == "static" else ".html"
     content_type = "course"
-    checksum = "7s35721d1647f962d59b8120a52210a7"
+    archive_checksum = "7s35721d1647f962d59b8120a52210a7"
     metadata = {"title": "the title of the course"} if has_metadata else None
     tika_output = {"content": tika_content, "metadata": metadata}
 
@@ -202,7 +202,7 @@ def test_transform_content_files(  # noqa: PLR0913
             content_type=content_type,
             published=True,
             run=run,
-            checksum=checksum,
+            archive_checksum=archive_checksum,
             key=edx_module_id,
         )
 
@@ -213,7 +213,7 @@ def test_transform_content_files(  # noqa: PLR0913
                 document,
                 {
                     "content_type": content_type,
-                    "checksum": checksum,
+                    "archive_checksum": archive_checksum,
                     "file_extension": file_extension,
                     "source_path": f"root/{folder}/uuid{file_extension}",
                 },
@@ -244,7 +244,7 @@ def test_transform_content_files(  # noqa: PLR0913
                 "published": True,
                 "content_title": metadata["title"] if has_metadata else "",
                 "content_type": content_type,
-                "checksum": checksum,
+                "archive_checksum": archive_checksum,
                 "file_extension": file_extension,
                 "source_path": f"root/{folder}/uuid{file_extension}",
                 "edx_module_id": edx_module_id,

learning_resources/etl/utils.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.20 on 2025-04-25 19:34
+
+from django.db import migrations, models
+
+
+def populate_archive_checksum(apps, schema_editor):
+    for content_file in apps.get_model(
+        "learning_resources", "ContentFile"
+    ).objects.all():
+        if content_file.archive_checksum is None:
+            content_file.archive_checksum = content_file.checksum
+            content_file.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("learning_resources", "0088_add_content_summarizer_config"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="contentfile",
+            name="archive_checksum",
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.RunPython(populate_archive_checksum, migrations.RunPython.noop),
+    ]

learning_resources/etl/utils_test.py

@@ -3,6 +3,7 @@
 import uuid
 from abc import abstractmethod
 from functools import cached_property
+from hashlib import md5
 from typing import TYPE_CHECKING, Optional
 
 from django.conf import settings
@@ -904,13 +905,25 @@ class ContentFile(TimestampedModel):
     )
     content_tags = models.ManyToManyField(LearningResourceContentTag)
     published = models.BooleanField(default=True)
+    archive_checksum = models.CharField(max_length=32, null=True, blank=True)  # noqa: DJ001
     checksum = models.CharField(max_length=32, null=True, blank=True)  # noqa: DJ001
     source_path = models.CharField(max_length=1024, null=True, blank=True)  # noqa: DJ001
     file_extension = models.CharField(max_length=32, null=True, blank=True)  # noqa: DJ001
     edx_module_id = models.CharField(max_length=1024, null=True, blank=True)  # noqa: DJ001
     summary = models.TextField(blank=True, default="")
     flashcards = models.JSONField(blank=True, default=list)
 
+    def content_checksum(self):
+        hasher = md5()  # noqa: S324
+        if self.content:
+            hasher.update(self.content.encode("utf-8"))
+            return hasher.hexdigest()
+        return None
+
+    def save(self, **kwargs):
+        self.checksum = self.content_checksum()
+        super().save(**kwargs)
+
     class Meta:
         unique_together = (("key", "run"),)
         verbose_name = "contentfile"

learning_resources/migrations/0089_contentfile_archive_checksum.py

@@ -1179,6 +1179,7 @@ class ContentFileSerializer(serializers.ModelSerializer):
     run_title = serializers.CharField(source="run.title", required=False)
     run_slug = serializers.CharField(source="run.slug", required=False)
     semester = serializers.CharField(source="run.semester", required=False)
+    checksum = serializers.CharField(required=False)
     year = serializers.IntegerField(source="run.year", required=False)
     topics = serializers.SerializerMethodField()
     resource_id = serializers.SerializerMethodField()
@@ -1293,6 +1294,7 @@ class Meta:
             "content_title",
             "content_author",
             "content_language",
+            "checksum",
             "image_src",
             "resource_id",
             "resource_readable_id",

learning_resources/models.py

@@ -570,6 +570,7 @@ def test_content_file_serializer(settings, expected_types, has_channels):
             "description": content_file.description,
             "file_type": content_file.file_type,
             "content_type": content_file.content_type,
+            "checksum": content_file.checksum,
             "url": content_file.url,
             "content": content_kwargs["content"],
             "content_title": content_kwargs["content_title"],