Hook up an index option

Author: thecoop

server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java

@@ -86,7 +86,8 @@ private void indexVectors() {
                           "index": true,
                           "similarity": "l2_norm",
                           "index_options": {
-                            "type": "%type%"
+                            "type": "%type%",
+                            "direct_raw_vector_reads": true
                           }
                         }
                       }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -388,7 +388,8 @@ private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boo
                 return new BBQHnswIndexOptions(
                     Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
                     Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-                    new RescoreVector(DEFAULT_OVERSAMPLE)
+                    new RescoreVector(DEFAULT_OVERSAMPLE),
+                    false
                 );
             } else if (defaultInt8Hnsw) {
                 return new Int8HnswIndexOptions(
@@ -1455,15 +1456,20 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?
                 }
                 int m = XContentMapValues.nodeIntegerValue(mNode);
                 int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode);
+
                 RescoreVector rescoreVector = null;
                 if (hasRescoreIndexVersion(indexVersion)) {
                     rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
                     if (rescoreVector == null && defaultOversampleForBBQ(indexVersion)) {
                         rescoreVector = new RescoreVector(DEFAULT_OVERSAMPLE);
                     }
                 }
+
+                Object directRawVectorReadsNode = indexOptionsMap.remove("direct_raw_vector_reads");
+                boolean directRawVectorReads = XContentMapValues.nodeBooleanValue(directRawVectorReadsNode, false);
+
                 MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
-                return new BBQHnswIndexOptions(m, efConstruction, rescoreVector);
+                return new BBQHnswIndexOptions(m, efConstruction, rescoreVector, directRawVectorReads);
             }
 
             @Override
@@ -1518,10 +1524,12 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?
                         );
                     }
                 }
+
                 RescoreVector rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
                 if (rescoreVector == null) {
                     rescoreVector = new RescoreVector(DEFAULT_OVERSAMPLE);
                 }
+
                 Object visitPercentageNode = indexOptionsMap.remove("default_visit_percentage");
                 double visitPercentage = 0d;
                 if (visitPercentageNode != null) {
@@ -1536,8 +1544,12 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?
                         );
                     }
                 }
+
+                Object directRawVectorReadsNode = indexOptionsMap.remove("direct_raw_vector_reads");
+                boolean directRawVectorReads = XContentMapValues.nodeBooleanValue(directRawVectorReadsNode, false);
+
                 MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
-                return new BBQIVFIndexOptions(clusterSize, visitPercentage, rescoreVector);
+                return new BBQIVFIndexOptions(clusterSize, visitPercentage, rescoreVector, directRawVectorReads);
             }
 
             @Override
@@ -2009,17 +2021,23 @@ public String toString() {
     public static class BBQHnswIndexOptions extends QuantizedIndexOptions {
         private final int m;
         private final int efConstruction;
+        private final boolean directRawVectorReads;
 
-        public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) {
+        public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector, boolean directRawVectorReads) {
             super(VectorIndexType.BBQ_HNSW, rescoreVector);
             this.m = m;
             this.efConstruction = efConstruction;
+            this.directRawVectorReads = directRawVectorReads;
         }
 
         @Override
         KnnVectorsFormat getVectorsFormat(ElementType elementType) {
             assert elementType == ElementType.FLOAT;
-            return new ES93HnswBinaryQuantizedVectorsFormat(m, efConstruction);
+            var format = new ES93HnswBinaryQuantizedVectorsFormat(m, efConstruction);
+            if (directRawVectorReads) {
+                format.useDirectIO();
+            }
+            return format;
         }
 
         @Override
@@ -2031,12 +2049,13 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
         @Override
         boolean doEquals(DenseVectorIndexOptions other) {
             BBQHnswIndexOptions that = (BBQHnswIndexOptions) other;
-            return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector);
+            return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector)
+                && directRawVectorReads == that.directRawVectorReads;
         }
 
         @Override
         int doHashCode() {
-            return Objects.hash(m, efConstruction, rescoreVector);
+            return Objects.hash(m, efConstruction, rescoreVector, directRawVectorReads);
         }
 
         @Override
@@ -2053,6 +2072,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (rescoreVector != null) {
                 rescoreVector.toXContent(builder, params);
             }
+            if (directRawVectorReads) {
+                builder.field("direct_raw_vector_reads", true);
+            }
             builder.endObject();
             return builder;
         }
@@ -2131,17 +2153,23 @@ public boolean validateDimension(int dim, boolean throwOnError) {
     static class BBQIVFIndexOptions extends QuantizedIndexOptions {
         final int clusterSize;
         final double defaultVisitPercentage;
+        final boolean directRawVectorReads;
 
-        BBQIVFIndexOptions(int clusterSize, double defaultVisitPercentage, RescoreVector rescoreVector) {
+        BBQIVFIndexOptions(int clusterSize, double defaultVisitPercentage, RescoreVector rescoreVector, boolean directRawVectorReads) {
             super(VectorIndexType.BBQ_DISK, rescoreVector);
             this.clusterSize = clusterSize;
             this.defaultVisitPercentage = defaultVisitPercentage;
+            this.directRawVectorReads = directRawVectorReads;
         }
 
         @Override
         KnnVectorsFormat getVectorsFormat(ElementType elementType) {
             assert elementType == ElementType.FLOAT;
-            return new ES920DiskBBQVectorsFormat(clusterSize, ES920DiskBBQVectorsFormat.DEFAULT_CENTROIDS_PER_PARENT_CLUSTER);
+            var format = new ES920DiskBBQVectorsFormat(clusterSize, ES920DiskBBQVectorsFormat.DEFAULT_CENTROIDS_PER_PARENT_CLUSTER);
+            if (directRawVectorReads) {
+                format.useDirectIO();
+            }
+            return format;
         }
 
         @Override
@@ -2154,12 +2182,13 @@ boolean doEquals(DenseVectorIndexOptions other) {
             BBQIVFIndexOptions that = (BBQIVFIndexOptions) other;
             return clusterSize == that.clusterSize
                 && defaultVisitPercentage == that.defaultVisitPercentage
-                && Objects.equals(rescoreVector, that.rescoreVector);
+                && Objects.equals(rescoreVector, that.rescoreVector)
+                && directRawVectorReads == that.directRawVectorReads;
         }
 
         @Override
         int doHashCode() {
-            return Objects.hash(clusterSize, defaultVisitPercentage, rescoreVector);
+            return Objects.hash(clusterSize, defaultVisitPercentage, rescoreVector, directRawVectorReads);
         }
 
         @Override
@@ -2176,6 +2205,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (rescoreVector != null) {
                 rescoreVector.toXContent(builder, params);
             }
+            if (directRawVectorReads) {
+                builder.field("direct_raw_vector_reads", true);
+            }
             builder.endObject();
             return builder;
         }

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java

@@ -100,7 +100,8 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA
                 new DenseVectorFieldMapper.BBQHnswIndexOptions(
                     randomIntBetween(1, 100),
                     randomIntBetween(1, 10_000),
-                    randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
+                    randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()),
+                    randomBoolean()
                 ),
                 new DenseVectorFieldMapper.BBQFlatIndexOptions(
                     randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
@@ -113,7 +114,8 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA
                 new DenseVectorFieldMapper.BBQIVFIndexOptions(
                     randomIntBetween(MIN_VECTORS_PER_CLUSTER, MAX_VECTORS_PER_CLUSTER),
                     randomFloatBetween(0.0f, 100.0f, true),
-                    randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
+                    randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()),
+                    randomBoolean()
                 )
             );
         }
@@ -141,7 +143,7 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQua
                 randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)),
                 rescoreVector
             ),
-            new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector)
+            new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector, randomBoolean())
         );
     }