support NOHL and SKIPINITIALSCAN tags on FT.CREATE (#125)

* support NOHL and SKIPINITIALSCAN tags on FT.CREATE

* add SKIP-INITIAL-SCAN test

* add NO-HIGHLIGHT test

* [fix] Running SKIPINITIALSCAN related tests on RediSearch >= v2.x

* [fix] Using CreateIndex on TestSchema_SummarizationDisabled for RediSearch v1.6 compatible testing

* [fix] Fix flakiness of TestSchema_SkipInitialScan by waiting for all docs to be indexed before querying

Co-authored-by: filipecosta90 <[email protected]>

Author: AvitalFineRedis

redisearch/schema.go

@@ -28,7 +28,7 @@ type Options struct {
 	// This is an option that is applied and index level.
 	NoFrequencies bool
 
-	// If set, , we avoid saving the term offsets for documents.
+	// If set, we avoid saving the term offsets for documents.
 	// This saves memory but does not allow exact searches or highlighting. Implies NOHL
 	// This is an option that is applied and index level.
 	NoOffsetVectors bool
@@ -49,6 +49,13 @@ type Options struct {
 	// If set to true This option forces RediSearch to encode indexes as if there were more than 32 text fields,
 	// which allows you to add additional fields (beyond 32).
 	MaxTextFieldsFlag bool
+
+	// If set to true, conserves storage space and memory by disabling highlighting support.
+	// Also implied by NoOffsetVectors
+	NoHighlights bool
+
+	// If set to true, we do not scan and index.
+	SkipInitialScan bool
 }
 
 func NewOptions() *Options {
@@ -89,6 +96,18 @@ func (options *Options) SetMaxTextFieldsFlag(flag bool) *Options {
 	return options
 }
 
+// SetNoHighlight conserves storage space and memory by disabling highlighting support.
+func (options *Options) SetNoHighlight(flag bool) *Options {
+	options.NoHighlights = flag
+	return options
+}
+
+// SetSkipInitialScan determines if scan the index on creation
+func (options *Options) SetSkipInitialScan(flag bool) *Options {
+	options.SkipInitialScan = flag
+	return options
+}
+
 // DefaultOptions represents the default options
 var DefaultOptions = Options{
 	NoSave:            false,
@@ -99,6 +118,8 @@ var DefaultOptions = Options{
 	Temporary:         false,
 	TemporaryPeriod:   0,
 	MaxTextFieldsFlag: false,
+	NoHighlights:      false,
+	SkipInitialScan:   false,
 }
 
 // Field Types
@@ -282,12 +303,18 @@ func SerializeSchema(s *Schema, args redis.Args) (argsOut redis.Args, err error)
 	if s.Options.Temporary {
 		argsOut = append(argsOut, "TEMPORARY", s.Options.TemporaryPeriod)
 	}
+	if s.Options.NoHighlights {
+		argsOut = append(argsOut, "NOHL")
+	}
 	if s.Options.NoFieldFlags {
 		argsOut = append(argsOut, "NOFIELDS")
 	}
 	if s.Options.NoFrequencies {
 		argsOut = append(argsOut, "NOFREQS")
 	}
+	if s.Options.SkipInitialScan {
+		argsOut = append(argsOut, "SKIPINITIALSCAN")
+	}
 
 	if s.Options.Stopwords != nil {
 		argsOut = argsOut.Add("STOPWORDS", len(s.Options.Stopwords))

redisearch/schema_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"reflect"
 	"testing"
+	"time"
 )
 
 func TestNewSchema(t *testing.T) {
@@ -22,6 +23,10 @@ func TestNewSchema(t *testing.T) {
 			Options: Options{Stopwords: []string{"custom"}}}},
 		{"no-frequencies", args{Options{NoFrequencies: true}}, &Schema{Fields: []Field{},
 			Options: Options{NoFrequencies: true}}},
+		{"no-highlights", args{Options{NoHighlights: true}}, &Schema{Fields: []Field{},
+			Options: Options{NoHighlights: true}}},
+		{"skip-initial-scan", args{Options{SkipInitialScan: true}}, &Schema{Fields: []Field{},
+			Options: Options{SkipInitialScan: true}}},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -50,6 +55,10 @@ func TestSerializeSchema(t *testing.T) {
 		{"default-args-with-different-constructor", args{NewSchema(*NewOptions()), redis.Args{}}, redis.Args{"SCHEMA"}, false},
 		{"temporary", args{NewSchema(*NewOptions().SetTemporaryPeriod(60)), redis.Args{}}, redis.Args{"TEMPORARY", 60, "SCHEMA"}, false},
 		{"no-frequencies", args{NewSchema(Options{NoFrequencies: true}), redis.Args{}}, redis.Args{"NOFREQS", "SCHEMA"}, false},
+		{"no-hithlights", args{NewSchema(Options{NoHighlights: true}), redis.Args{}}, redis.Args{"NOHL", "SCHEMA"}, false},
+		{"no-hithlights-with-different-consturctor", args{NewSchema(*NewOptions().SetNoHighlight(true)), redis.Args{}}, redis.Args{"NOHL", "SCHEMA"}, false},
+		{"skip-inital-scan", args{NewSchema(Options{SkipInitialScan: true}), redis.Args{}}, redis.Args{"SKIPINITIALSCAN", "SCHEMA"}, false},
+		{"skipinitalscan-with-different-consturctor", args{NewSchema(*NewOptions().SetSkipInitialScan(true)), redis.Args{}}, redis.Args{"SKIPINITIALSCAN", "SCHEMA"}, false},
 		{"no-fields", args{NewSchema(Options{NoFieldFlags: true}), redis.Args{}}, redis.Args{"NOFIELDS", "SCHEMA"}, false},
 		{"custom-stopwords", args{NewSchema(Options{Stopwords: []string{"custom"}}), redis.Args{}}, redis.Args{"STOPWORDS", 1, "custom", "SCHEMA"}, false},
 		{"custom-stopwords-with-different-constructor", args{NewSchema(*NewOptions().SetStopWords([]string{"custom"})), redis.Args{}}, redis.Args{"STOPWORDS", 1, "custom", "SCHEMA"}, false},
@@ -112,3 +121,69 @@ func TestSchema_AddField(t *testing.T) {
 		})
 	}
 }
+
+func TestSchema_SkipInitialScan(t *testing.T) {
+	c := createClient("skip-initial-scan-test")
+	flush(c)
+
+	// check RediSearch version
+	version, err := c.getRediSearchVersion()
+	assert.Nil(t, err)
+	// This feature is only available since RediSearch >= v2.0
+	if version <= 10699 {
+		return
+	}
+
+	vanillaConnection := c.pool.Get()
+	_, err = vanillaConnection.Do("HSET", "create-index-info:doc1", "name", "Jon", "age", 25)
+	assert.Nil(t, err)
+
+	q := NewQuery("@name:Jon")
+	schema1 := NewSchema(DefaultOptions).AddField(NewTextField("name"))
+	schema2 := NewSchema(Options{SkipInitialScan: true}).AddField(NewTextField("name"))
+	indexDefinition := NewIndexDefinition()
+
+	c = createClient("skip-initial-scan-test-scan")
+	c.CreateIndexWithIndexDefinition(schema1, indexDefinition)
+	assert.Nil(t, err)
+
+	// Wait for all documents to be indexed
+	info, err := c.Info()
+	assert.Nil(t, err)
+	for info.IsIndexing {
+		time.Sleep(time.Second)
+		info, _ = c.Info()
+	}
+
+	_, total, err := c.Search(q)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, total)
+
+	c = createClient("skip-initial-scan-test-skip-scan")
+	c.CreateIndexWithIndexDefinition(schema2, indexDefinition)
+	assert.Nil(t, err)
+	_, total, err = c.Search(q)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, total)
+}
+
+func TestSchema_SummarizationDisabled(t *testing.T) {
+	doc := NewDocument("TestSchema-doc1", 1.0).Set("body", "foo bar")
+
+	c := createClient("summarize-disable-no-term-offsets-test")
+	flush(c)
+	schema := NewSchema(Options{NoOffsetVectors: true}).AddField(NewTextField("body"))
+
+	c.CreateIndex(schema)
+	assert.Nil(t, c.IndexOptions(DefaultIndexingOptions, doc))
+	_, _, err := c.Search(NewQuery("body").Summarize())
+	assert.NotNil(t, err)
+
+	c = createClient("summarize-disable-no-highlights-test")
+	flush(c)
+	schema = NewSchema(Options{NoHighlights: true}).AddField(NewTextField("body"))
+	c.CreateIndex(schema)
+	assert.Nil(t, c.IndexOptions(DefaultIndexingOptions, doc))
+	_, _, err = c.Search(NewQuery("body").Summarize())
+	assert.NotNil(t, err)
+}