add option to not embed field names for vector db destination

Author: Frankie Bromage

airbyte_cdk/destinations/vector_db_based/config.py

@@ -108,6 +108,12 @@ class ProcessingConfigModel(BaseModel):
         always_show=True,
         examples=["text", "user.name", "users.*.name"],
     )
+    omit_field_names_from_embeddings: bool = Field(
+        default=False,
+        title="Omit field names from embeddings",
+        description="Do not include the field names in the text that gets embedded. By default field names are embedded e.g 'user: name, user.email: [email protected]'. If set to true, only the values are embedded e.g. 'name, [email protected]'.",
+        always_show=True,
+    )
     metadata_fields: Optional[List[str]] = Field(
         default=[],
         title="Fields to store as metadata",

airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -125,6 +125,7 @@ def __init__(self, config: ProcessingConfigModel, catalog: ConfiguredAirbyteCata
         self.text_fields = config.text_fields
         self.metadata_fields = config.metadata_fields
         self.field_name_mappings = config.field_name_mappings
+        self.omit_field_names_from_embeddings = config.omit_field_names_from_embeddings
         self.logger = logging.getLogger("airbyte.document_processor")
 
     def process(self, record: AirbyteRecordMessage) -> Tuple[List[Chunk], Optional[str]]:
@@ -162,7 +163,12 @@ def _generate_document(self, record: AirbyteRecordMessage) -> Optional[Document]
         relevant_fields = self._extract_relevant_fields(record, self.text_fields)
         if len(relevant_fields) == 0:
             return None
-        text = stringify_dict(relevant_fields)
+        if self.omit_field_names_from_embeddings == False:
+            text = stringify_dict(relevant_fields)
+        else:
+            text = ""
+            for key, value in relevant_fields.items():
+                text += f"{value}\n"
         metadata = self._extract_metadata(record)
         return Document(page_content=text, metadata=metadata)