Content File Vector Search Endpoint (#1907)

* updating deps and adding method for getting token count

* working contentfile chunk embeds

* storing chunks and update initial resource record with embeddings from contentfile chunk

* adding management command flag to generate embeds by id

* fixing test

* ensuring we stay under token size

* removing full content from points

* moving splitter to separate function

* adding test for text splitter

* adding more tests

* fixing test

* changing chunk key name

* fix test setting:

* fixing test

* adding initial serializers

* adding initial working view

* pinning onnxruntime to older version

* add setting for qdrant chunk size and some refactoring

* removing _id field

* test fix

* fix docstring

* adding more filters

* regenerate oai spec

* add view tests

* regenerate spec

* setting some indexes

* fixes

* fixing tests

* docstring fix

* fixes to filter out empty chunks and adding file extension as property

* fix test

* fix method name

* making sure we only process unique content for each run

* filktering for published

* removing next run method

* removing published field

* removed unique content id function and updating spec

* fixing issue in generate by id

* pinning litellm version

Author: shanbady

frontends/api/src/generated/v0/api.ts

@@ -373,6 +373,12 @@ export interface ContentFile {
    * @memberof ContentFile
    */
   file_type?: string | null
+  /**
+   *
+   * @type {string}
+   * @memberof ContentFile
+   */
+  file_extension?: string | null
   /**
    *
    * @type {LearningResourceOfferor}

frontends/api/src/generated/v1/api.ts

@@ -815,6 +815,7 @@ class Meta:
             "resource_readable_id",
             "course_number",
             "file_type",
+            "file_extension",
             "offered_by",
             "platform",
             "run_readable_id",

learning_resources/serializers.py

@@ -496,6 +496,7 @@ def test_content_file_serializer(settings, expected_types, has_channels):
                 "name": PlatformType[platform].value,
                 "code": platform,
             },
+            "file_extension": content_file.file_extension,
             "offered_by": {
                 "name": content_file.run.learning_resource.offered_by.name,
                 "code": content_file.run.learning_resource.offered_by.code,

learning_resources/serializers_test.py

@@ -35,6 +35,7 @@
     adjust_search_for_percolator,
     document_percolated_actions,
 )
+from vector_search.constants import RESOURCES_COLLECTION_NAME
 
 log = logging.getLogger(__name__)
 
@@ -929,7 +930,7 @@ def _qdrant_similar_results(doc, num_resources):
     return [
         hit.payload
         for hit in client.query_points(
-            collection_name=f"{settings.QDRANT_BASE_COLLECTION_NAME}.resources",
+            collection_name=RESOURCES_COLLECTION_NAME,
             query=vector_point_id(doc["readable_id"]),
             limit=num_resources,
             using=encoder.model_short_name(),

learning_resources_search/api.py

@@ -804,6 +804,11 @@ def get_all_config_keys():
     name="QDRANT_SPARSE_MODEL", default="prithivida/Splade_PP_en_v1"
 )
 
+QDRANT_CHUNK_SIZE = get_int(
+    name="QDRANT_CHUNK_SIZE",
+    default=100,
+)
+
 QDRANT_ENCODER = get_string(
     name="QDRANT_ENCODER", default="vector_search.encoders.fastembed.FastEmbedEncoder"
 )

main/settings.py

@@ -314,6 +314,121 @@ paths:
               schema:
                 $ref: '#/components/schemas/CKEditorSettings'
           description: ''
+  /api/v0/content_files_vector_search/:
+    get:
+      operationId: content_files_vector_search_retrieve
+      description: Vector Search for content
+      summary: Content File Vector Search
+      parameters:
+      - in: query
+        name: content_feature_type
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: The feature type of the content file. Possible options are at
+          api/v1/course_features/
+      - in: query
+        name: course_number
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: Course number of the content file
+      - in: query
+        name: file_extension
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: 'The extension of the content file. '
+      - in: query
+        name: key
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: The filename of the content file
+      - in: query
+        name: limit
+        schema:
+          type: integer
+        description: Number of results to return per page
+      - in: query
+        name: offered_by
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: Offeror of the content file
+      - in: query
+        name: offset
+        schema:
+          type: integer
+        description: The initial index from which to return the results
+      - in: query
+        name: platform
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: platform(s) of the content file
+      - in: query
+        name: q
+        schema:
+          type: string
+          minLength: 1
+        description: The search text
+      - in: query
+        name: resource_readable_id
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: The readable_id value of the parent learning resource for the
+          content file
+      - in: query
+        name: run_readable_id
+        schema:
+          type: array
+          items:
+            type: string
+            minLength: 1
+        description: The readable_id value of the run that the content file belongs
+          to
+      - in: query
+        name: sortby
+        schema:
+          enum:
+          - id
+          - -id
+          - resource_readable_id
+          - -resource_readable_id
+          type: string
+          minLength: 1
+        description: |-
+          if the parameter starts with '-' the sort is in descending order
+
+          * `id` - id
+          * `-id` - -id
+          * `resource_readable_id` - resource_readable_id
+          * `-resource_readable_id` - -resource_readable_id
+      tags:
+      - content_files_vector_search
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ContentFileVectorSearchResponse'
+          description: ''
   /api/v0/learning_resources_search_admin_params/:
     get:
       operationId: learning_resources_search_admin_params_retrieve
@@ -1618,6 +1733,184 @@ components:
           default: false
       required:
       - message
+    ContentFile:
+      type: object
+      description: Serializer class for course run ContentFiles
+      properties:
+        id:
+          type: integer
+          readOnly: true
+        run_id:
+          type: integer
+        run_title:
+          type: string
+        run_slug:
+          type: string
+        departments:
+          type: array
+          items:
+            $ref: '#/components/schemas/LearningResourceDepartment'
+        semester:
+          type: string
+        year:
+          type: integer
+        topics:
+          type: array
+          items:
+            $ref: '#/components/schemas/LearningResourceTopic'
+        key:
+          type: string
+          nullable: true
+          maxLength: 1024
+        uid:
+          type: string
+          nullable: true
+          maxLength: 36
+        title:
+          type: string
+          nullable: true
+          maxLength: 1024
+        description:
+          type: string
+          nullable: true
+        url:
+          type: string
+          nullable: true
+        content_feature_type:
+          type: array
+          items:
+            type: string
+        content_type:
+          $ref: '#/components/schemas/ContentTypeEnum'
+        content:
+          type: string
+          nullable: true
+        content_title:
+          type: string
+          nullable: true
+          maxLength: 1024
+        content_author:
+          type: string
+          nullable: true
+          maxLength: 1024
+        content_language:
+          type: string
+          nullable: true
+          maxLength: 24
+        image_src:
+          type: string
+          format: uri
+          nullable: true
+          maxLength: 200
+        resource_id:
+          type: string
+        resource_readable_id:
+          type: string
+        course_number:
+          type: array
+          items:
+            type: string
+          description: Extract the course number(s) from the associated course
+          readOnly: true
+        file_type:
+          type: string
+          nullable: true
+          maxLength: 128
+        file_extension:
+          type: string
+          nullable: true
+          maxLength: 32
+        offered_by:
+          $ref: '#/components/schemas/LearningResourceOfferor'
+        platform:
+          $ref: '#/components/schemas/LearningResourcePlatform'
+        run_readable_id:
+          type: string
+      required:
+      - content_feature_type
+      - course_number
+      - departments
+      - id
+      - offered_by
+      - platform
+      - resource_id
+      - resource_readable_id
+      - run_id
+      - run_readable_id
+      - run_slug
+      - run_title
+      - semester
+      - topics
+      - year
+    ContentFileVectorSearchResponse:
+      type: object
+      description: SearchResponseSerializer with OpenAPI annotations for Content Files
+        search
+      properties:
+        count:
+          type: integer
+          readOnly: true
+        next:
+          type: string
+          nullable: true
+          readOnly: true
+        previous:
+          type: string
+          nullable: true
+          readOnly: true
+        results:
+          type: array
+          items:
+            $ref: '#/components/schemas/ContentFile'
+          readOnly: true
+        metadata:
+          type: object
+          properties:
+            aggregations:
+              type: object
+              additionalProperties:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    key:
+                      type: string
+                    doc_count:
+                      type: integer
+                  required:
+                  - doc_count
+                  - key
+            suggestions:
+              type: array
+              items:
+                type: string
+          required:
+          - aggregations
+          - suggestions
+          readOnly: true
+      required:
+      - count
+      - metadata
+      - next
+      - previous
+      - results
+    ContentTypeEnum:
+      enum:
+      - page
+      - file
+      - video
+      - pdf
+      type: string
+      description: |-
+        * `page` - page
+        * `file` - file
+        * `video` - video
+        * `pdf` - pdf
+      x-enum-descriptions:
+      - page
+      - file
+      - video
+      - pdf
     Counts:
       type: object
       properties:

openapi/specs/v0.yaml

@@ -8599,6 +8599,10 @@ components:
           type: string
           nullable: true
           maxLength: 128
+        file_extension:
+          type: string
+          nullable: true
+          maxLength: 32
         offered_by:
           $ref: '#/components/schemas/LearningResourceOfferor'
         platform: