fallback to dataset index query

Author: kolesnikovae

pkg/experiment/block/compaction.go

@@ -227,13 +227,11 @@ func (b *CompactionPlan) writeDatasetIndex(w *Writer) error {
 		metadata.LabelValueDatasetTSDBIndex,
 	)
 	b.meta.Datasets = append(b.meta.Datasets, &metastorev1.Dataset{
-		Tenant:  b.meta.Tenant,
-		Name:    0, // Anonymous.
-		MinTime: b.meta.MinTime,
-		MaxTime: b.meta.MaxTime,
-		// FIXME: We mimic the default layout: empty profiles, index, and empty symbols.
-		//  Instead, it should be handled at the query time: substitute the dataset layout.
-		TableOfContents: []uint64{off, off, w.Offset()},
+		Tenant:          b.meta.Tenant,
+		Name:            0, // Anonymous.
+		MinTime:         b.meta.MinTime,
+		MaxTime:         b.meta.MaxTime,
+		TableOfContents: []uint64{off},
 		Size:            uint64(n),
 		Labels:          labels,
 	})

pkg/experiment/query_backend/query.go

@@ -171,9 +171,10 @@ func newBlockContext(
 func (q *blockContext) needsDatasetLookup() bool {
 	md := q.obj.Metadata()
 	if len(md.Datasets) > 1 {
-		// The metadata explicitly lists datasets to be queried.
+		// The blocks metadata explicitly lists datasets to be queried.
 		return false
 	}
+
 	ds := md.Datasets[0]
 	t := metadata.OpenStringTable(md)
 	m := metadata.NewLabelMatcher(t, []*labels.Matcher{{
@@ -185,14 +186,21 @@ func (q *blockContext) needsDatasetLookup() bool {
 	if !matches {
 		return false
 	}
+
+	// The block contains a dataset TSDB index.
+	//
+	// If a query only requires TSDB data, we can serve it using
+	// the dataset index without accessing the full dataset.
 	qc := queryContext{req: q.req}
 	if s := qc.sections(); len(s) == 1 && s[0] == block.SectionTSDB {
-		// The block has a dataset tsdb index and the queries
-		// only need TSDB data. In this case, we can serve the
-		// query directly using the dataset index, without need
-		// to access datasets.
+		// Create a TOC with a TSDB entry pointing to the dataset index.
+		// This allows the dataset to be used like a regular one.
+		// Version-specific behavior.
+		offset := ds.TableOfContents[0]
+		ds.TableOfContents = []uint64{offset, offset, ds.Size}
 		return false
 	}
+
 	q.idx = ds
 	return true
 }

pkg/frontend/read_path/query_frontend/query_frontend.go

@@ -71,11 +71,11 @@ func (q *QueryFrontend) Query(
 	if len(blocks) == 0 {
 		return new(queryv1.QueryResponse), nil
 	}
-
 	// Randomize the order of blocks to avoid hotspots.
 	xrand.Shuffle(len(blocks), func(i, j int) {
 		blocks[i], blocks[j] = blocks[j], blocks[i]
 	})
+	// TODO(kolesnikovae): Should be dynamic.
 	p := queryplan.Build(blocks, 4, 20)
 
 	resp, err := q.querybackendClient.Invoke(ctx, &queryv1.InvokeRequest{
@@ -110,25 +110,34 @@ func (q *QueryFrontend) QueryMetadata(
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
+	query := &metastorev1.QueryMetadataRequest{
+		TenantId:  tenants,
+		StartTime: req.StartTime,
+		EndTime:   req.EndTime,
+	}
+
+	// Delete all labels but service_name. If no labels left, request the
+	// dataset index for query backend to lookup datasets to be accessed.
 	matchers = slices.DeleteFunc(matchers, func(m *labels.Matcher) bool {
 		return m.Name != phlaremodel.LabelNameServiceName
 	})
 	if len(matchers) == 0 {
+		// We preserve the __tenant_dataset__= label: this is needed for the
+		// query backend to identify that the dataset is the tenant-wide index,
+		// and a dataset lookup is needed.
+		query.Labels = []string{metadata.LabelNameTenantDataset}
 		matchers = []*labels.Matcher{{
 			Name:  metadata.LabelNameTenantDataset,
 			Value: metadata.LabelValueDatasetTSDBIndex,
 			Type:  labels.MatchEqual,
 		}}
 	}
-	query := matchersToLabelSelector(matchers)
-	md, err := q.metadataQueryClient.QueryMetadata(ctx, &metastorev1.QueryMetadataRequest{
-		TenantId:  tenants,
-		StartTime: req.StartTime,
-		EndTime:   req.EndTime,
-		Query:     query,
-	})
+
+	query.Query = matchersToLabelSelector(matchers)
+	md, err := q.metadataQueryClient.QueryMetadata(ctx, query)
 	if err != nil {
 		return nil, err
 	}
+
 	return md.Blocks, nil
 }