Add AWS S3 Document Storage (WIP) (#596)

## Motivation and Context (Why the change? What's the scenario?)

Allow KM to store documents/files in AWS S3 and compatible services.

## High level description (Approach, Design)

Added new extension for AWS S3 storage, configuration settings, and nuget.

---------

Co-authored-by: Konstantine Kalbazov <[email protected]>
Co-authored-by: Devis Lucato <[email protected]>
Co-authored-by: Devis Lucato <[email protected]>

Author: 3 people

Directory.Packages.props

@@ -4,6 +4,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.0-preview.7.24251.11" />
+    <PackageVersion Include="AWSSDK.S3" Version="3.7.308.9" />
     <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0" />
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.1" />
     <PackageVersion Include="Azure.Identity" Version="1.12.0" />

KernelMemory.sln

@@ -116,6 +116,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{CA49F1A1
 		tools\run-redis.sh = tools\run-redis.sh
 		tools\search.sh = tools\search.sh
 		tools\upload-file.sh = tools\upload-file.sh
+		tools\run-s3ninja.sh = tools\run-s3ninja.sh
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abstractions", "service\Abstractions\Abstractions.csproj", "{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}"
@@ -290,6 +291,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "209-dotnet-using-context-ov
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "210-KM-without-builder", "examples\210-KM-without-builder\210-KM-without-builder.csproj", "{00A3DDF3-2230-4AEC-8B5B-B75F958D194B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "S3", "extensions\AWS\S3\S3.csproj", "{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -548,6 +551,10 @@ Global
 		{00A3DDF3-2230-4AEC-8B5B-B75F958D194B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{00A3DDF3-2230-4AEC-8B5B-B75F958D194B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{00A3DDF3-2230-4AEC-8B5B-B75F958D194B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -637,6 +644,7 @@ Global
 		{C746CE00-8BAE-4B46-A757-FE85D68747CE} = {DBEA0A6B-474A-4E8C-BCC8-D5D43C063A54}
 		{06A507C7-46B9-4D36-B88B-B4E4A0E8C0AC} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{00A3DDF3-2230-4AEC-8B5B-B75F958D194B} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
+		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E} = {155DA079-E267-49AF-973A-D1D44681970F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

KernelMemory.sln.DotSettings

@@ -219,6 +219,8 @@ public void It$SOMENAME$()
 </s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=abcdefghijklmnopqrstuvwxyz/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=AOAI/@EntryIndexedValue">True</s:Boolean>
+    <s:Boolean x:Key="/Default/UserDictionary/Words/=AWSS3/@EntryIndexedValue">True</s:Boolean>
+    <s:Boolean x:Key="/Default/UserDictionary/Words/=AWS/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=AZAI/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=AZDOCINTEL/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=AZSEARCH/@EntryIndexedValue">True</s:Boolean>
@@ -299,4 +301,4 @@ public void It$SOMENAME$()
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=wellknown/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Wordprocessing/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=xact/@EntryIndexedValue">True</s:Boolean>
-</wpf:ResourceDictionary>
+</wpf:ResourceDictionary>

README.md

@@ -31,6 +31,7 @@ public static async Task Main()
         var azureAISearchConfig = new AzureAISearchConfig();
         var postgresConfig = new PostgresConfig();
         var azureBlobConfig = new AzureBlobsConfig();
+        var awsS3Config = new AWSS3Config();
 
         new ConfigurationBuilder()
             .AddJsonFile("appsettings.json")
@@ -44,6 +45,7 @@ public static async Task Main()
             .BindSection("KernelMemory:Services:AzureAIDocIntel", azDocIntelConfig)
             .BindSection("KernelMemory:Services:AzureAISearch", azureAISearchConfig)
             .BindSection("KernelMemory:Services:AzureBlobs", azureBlobConfig)
+            .BindSection("KernelMemory:Services:AWSS3", awsS3Config)
             .BindSection("KernelMemory:Services:Postgres", postgresConfig)
             .BindSection("KernelMemory:Retrieval:SearchClient", searchClientConfig);
 
@@ -60,9 +62,10 @@ public static async Task Main()
             // .WithAzureAISearchMemoryDb(azureAISearchConfig)              // Store memories in Azure AI Search
             // .WithPostgresMemoryDb(postgresConfig)                        // Store memories in Postgres
             // .WithQdrantMemoryDb("http://127.0.0.1:6333")                 // Store memories in Qdrant
-            // .WithAzureBlobsDocumentStorage(azureBlobConfig)              // Store files in Azure Blobs
             // .WithSimpleVectorDb(SimpleVectorDbConfig.Persistent)         // Store memories on disk
+            // .WithAzureBlobsDocumentStorage(azureBlobConfig)              // Store files in Azure Blobs
             // .WithSimpleFileStorage(SimpleFileStorageConfig.Persistent)   // Store files on disk
+            // .WithAWSS3DocumentStorage(awsS3Config)                       // Store files on AWS S3
             .WithAzureOpenAITextGeneration(azureOpenAITextConfig, new DefaultGPTTokenizer())
             .WithAzureOpenAITextEmbeddingGeneration(azureOpenAIEmbeddingConfig, new DefaultGPTTokenizer());
 

docs/index.md

@@ -102,6 +102,23 @@
         // See https://platform.openai.com/docs/api-reference/embeddings/create
         "MaxEmbeddingBatchSize": 100
       },
+      "AWSS3": {
+        "Auth": "AccessKey",
+        // AccessKey ID, required when using AccessKey auth
+        // Note: you can use an env var 'KernelMemory__Services__AWSS3__AccessKey' to set this
+        "AccessKey": "",
+        // SecretAccessKey, required when using AccessKey auth
+        // Note: you can use an env var 'KernelMemory__Services__AWSS3__SecretAccessKey' to set this
+        "SecretAccessKey": "",
+        // Required bucket name where to create directories and upload files.
+        // Note: you can use an env var 'KernelMemory__Services__AWSS3__BucketName' to set this
+        "BucketName": ""
+        // Allows to specify a custom AWS or a compatible endpoint
+        // Examples: "https://s3.amazonaws.com", "https://s3.us-west-2.amazonaws.com", "http://127.0.0.1:9444"
+        // Note: you can use an env var 'KernelMemory__Services__AWSS3__Endpoint' to set this
+        // Note: you can test locally using S3 Ninja https://s3ninja.net
+        // "Endpoint": "https://s3.amazonaws.com"
+      },
       "LlamaSharp": {
         // path to file, e.g. "llama-2-7b-chat.Q6_K.gguf"
         "ModelPath": "",

examples/002-dotnet-Serverless/Program.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+#pragma warning disable IDE0130 // reduce number of "using" statements
+// ReSharper disable once CheckNamespace - reduce number of "using" statements
+namespace Microsoft.KernelMemory;
+
+public class AWSS3Config
+{
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public enum AuthTypes
+    {
+        Unknown = -1,
+        AccessKey,
+    }
+
+    public AuthTypes Auth { get; set; } = AuthTypes.Unknown;
+
+    /// <summary>
+    /// AWS IAM Access Key (aka Key Name)
+    /// </summary>
+    public string AccessKey { get; set; } = string.Empty;
+
+    /// <summary>
+    /// AWS IAM Secret Access Key (aka Password)
+    /// </summary>
+    public string SecretAccessKey { get; set; } = string.Empty;
+
+    /// <summary>
+    /// AWS S3 endpoint, e.g. https://s3.us-west-2.amazonaws.com
+    /// You can use S3 compatible services and dev tools like S3 Ninja.
+    /// </summary>
+    public string Endpoint { get; set; } = "https://s3.amazonaws.com";
+
+    /// <summary>
+    /// S3 bucket name
+    /// </summary>
+    public string BucketName { get; set; } = string.Empty;
+
+    public void Validate()
+    {
+        if (this.Auth == AuthTypes.Unknown)
+        {
+            throw new ConfigurationException($"Authentication type '{this.Auth}' undefined or not supported");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.AccessKey))
+        {
+            throw new ConfigurationException("S3 Access Key is undefined");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.SecretAccessKey))
+        {
+            throw new ConfigurationException("S3 Secret Key Access undefined");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.BucketName))
+        {
+            throw new ConfigurationException("S3 bucket name undefined");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.Endpoint))
+        {
+            throw new ConfigurationException("S3 endpoint name undefined");
+        }
+    }
+}