MB-65284: Add synonym query tracking and update stats map (#2141)

- Add a Scorch counter stat `TotSynonymSearches` to track the number of
synonym-enabled queries received by the index. This stat will be
incremented by 1 each time the FieldTermSynonymMap is set, indicating
that the query will use synonyms.

---------

Co-authored-by: Abhinav Dangeti <[email protected]>

index/scorch/scorch.go

@@ -635,6 +635,7 @@ func (s *Scorch) StatsMap() map[string]interface{} {
 	m["term_searchers_started"] = m["TotTermSearchersStarted"]
 	m["term_searchers_finished"] = m["TotTermSearchersFinished"]
 	m["knn_searches"] = m["TotKNNSearches"]
+	m["synonym_searches"] = m["TotSynonymSearches"]
 
 	m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"]
 	m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"]

index/scorch/segment_plugin.go

@@ -18,8 +18,8 @@ import (
 	"fmt"
 
 	"github.com/RoaringBitmap/roaring/v2"
-	index "github.com/blevesearch/bleve_index_api"
 	"github.com/blevesearch/bleve/v2/geo"
+	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 
 	zapv11 "github.com/blevesearch/zapx/v11"

index/scorch/snapshot_index.go

@@ -1134,3 +1134,7 @@ func (is *IndexSnapshot) ThesaurusKeysRegexp(name string,
 		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
 	})
 }
+
+func (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) {
+	atomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta)
+}

index/scorch/stats.go

@@ -51,7 +51,8 @@ type Stats struct {
 	TotTermSearchersStarted  uint64
 	TotTermSearchersFinished uint64
 
-	TotKNNSearches uint64
+	TotKNNSearches     uint64
+	TotSynonymSearches uint64
 
 	TotEventTriggerStarted   uint64
 	TotEventTriggerCompleted uint64

index_impl.go

@@ -636,6 +636,9 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	setKnnHitsInCollector(knnHits, req, coll)
 
 	if fts != nil {
+		if is, ok := indexReader.(*scorch.IndexSnapshot); ok {
+			is.UpdateSynonymSearchCount(1)
+		}
 		ctx = context.WithValue(ctx, search.FieldTermSynonymMapKey, fts)
 	}
 

mapping/mapping_vectors_test.go

@@ -310,18 +310,18 @@ func TestProcessVector(t *testing.T) {
 
 func TestNormalizeVector(t *testing.T) {
 	vectors := [][]float32{
-		[]float32{1,2,3,4,5},
-		[]float32{1,0,0,0,0},
-		[]float32{0.182574183,0.365148365,0.547722578,0.730296731},
-		[]float32{1,1,1,1,1,1,1,1},
+		[]float32{1, 2, 3, 4, 5},
+		[]float32{1, 0, 0, 0, 0},
+		[]float32{0.182574183, 0.365148365, 0.547722578, 0.730296731},
+		[]float32{1, 1, 1, 1, 1, 1, 1, 1},
 		[]float32{0},
 	}
 
 	expectedNormalizedVectors := [][]float32{
-		[]float32{0.13483998,0.26967996,0.40451995,0.5393599,0.67419994},
-		[]float32{1,0,0,0,0},
-		[]float32{0.18257418,0.36514837,0.5477226,0.73029673},
-		[]float32{0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338},
+		[]float32{0.13483998, 0.26967996, 0.40451995, 0.5393599, 0.67419994},
+		[]float32{1, 0, 0, 0, 0},
+		[]float32{0.18257418, 0.36514837, 0.5477226, 0.73029673},
+		[]float32{0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338},
 		[]float32{0},
 	}
 

search_test.go

@@ -1387,7 +1387,6 @@ func TestMatchQueryPartialMatch(t *testing.T) {
 		t.Errorf("Expected 1 result, but got: %v", res.Total)
 	}
 	hit := res.Hits[0]
-	fmt.Println(hit.Expl, hit.ID)
 	if hit.ID != "doc1" || hit.Expl.PartialMatch {
 		t.Errorf("Expected doc1 to be a full match")
 	}
@@ -4319,6 +4318,15 @@ func TestSynonymSearchQueries(t *testing.T) {
 		},
 	}
 
+	getTotalSynonymSearchStat := func(idx Index) int {
+		ir, err := idx.Advanced()
+		if err != nil {
+			t.Fatal(err)
+		}
+		stat := ir.StatsMap()["synonym_searches"].(uint64)
+		return int(stat)
+	}
+
 	runTestQueries := func(idx Index) error {
 		for _, dtq := range testQueries {
 			q, err := query.ParseQuery([]byte(dtq.query))
@@ -4352,6 +4360,12 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// now verify that the stat for number of synonym enabled queries is correct
+	totalSynonymSearchStat := getTotalSynonymSearchStat(idx)
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+
 	// test with index alias - with 1 batch per index
 	numIndexes := len(batches)
 	indexes := make([]Index, numIndexes)
@@ -4384,6 +4398,20 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// verify the synonym search stat for the alias
+	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+	for i := 1; i < numIndexes; i++ {
+		idxStat := getTotalSynonymSearchStat(indexes[i])
+		if idxStat != totalSynonymSearchStat {
+			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
+		}
+	}
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
 	// test with multi-level alias now with two index per alias
 	// and having any extra index being in the final alias
 	numAliases := numIndexes / 2
@@ -4405,6 +4433,17 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// verify the synonym searches stat for the alias
+	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
+	if totalSynonymSearchStat != 2*len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+	for i := 1; i < numIndexes; i++ {
+		idxStat := getTotalSynonymSearchStat(indexes[i])
+		if idxStat != totalSynonymSearchStat {
+			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
+		}
+	}
 }
 
 func TestGeoDistanceInSort(t *testing.T) {