explicit format handling

Author: kolesnikovae

api/gen/proto/go/metastore/v1/types.pb.go

@@ -35,6 +35,8 @@ message BlockMeta {
 }
 
 message Dataset {
+  uint32 format = 9;
+
   int32 tenant = 1;
   int32 name = 2;
   int64 min_time = 3;
@@ -44,10 +46,15 @@ message Dataset {
   // service region. The offsets are absolute.
   //
   // The interpretation of the table of contents is specific
-  // to the metadata format version. By default, the sections are:
+  // to the format.
+  //
+  // By default (format 0), the sections are:
   //  - 0: profiles.parquet
   //  - 1: index.tsdb
   //  - 2: symbols.symdb
+  //
+  // Format 1 corresponds to the tenant-wide index:
+  //  - 0: index.tsdb (dataset index)
   repeated uint64 table_of_contents = 5;
 
   // Size of the dataset in bytes.

api/gen/proto/go/metastore/v1/types_vtproto.pb.go

@@ -842,6 +842,10 @@
     "v1Dataset": {
       "type": "object",
       "properties": {
+        "format": {
+          "type": "integer",
+          "format": "int64"
+        },
         "tenant": {
           "type": "integer",
           "format": "int32"
@@ -864,7 +868,7 @@
             "type": "string",
             "format": "uint64"
           },
-          "description": "Table of contents lists data sections within the tenant\nservice region. The offsets are absolute.\n\nThe interpretation of the table of contents is specific\nto the metadata format version. By default, the sections are:\n - 0: profiles.parquet\n - 1: index.tsdb\n - 2: symbols.symdb"
+          "description": "Table of contents lists data sections within the tenant\nservice region. The offsets are absolute.\n\nThe interpretation of the table of contents is specific\nto the format.\n\nBy default (format 0), the sections are:\n - 0: profiles.parquet\n - 1: index.tsdb\n - 2: symbols.symdb\n\nFormat 1 corresponds to the tenant-wide index:\n - 0: index.tsdb (dataset index)"
         },
         "size": {
           "type": "string",

api/metastore/v1/types.proto

@@ -222,11 +222,15 @@ func (b *CompactionPlan) writeDatasetIndex(w *Writer) error {
 	if err != nil {
 		return err
 	}
+	// We annotate the dataset with the
+	// __tenant_dataset__ = "dataset_tsdb_index" label,
+	// so the dataset index metadata can be queried.
 	labels := metadata.NewLabelBuilder(b.strings).BuildPairs(
 		metadata.LabelNameTenantDataset,
 		metadata.LabelValueDatasetTSDBIndex,
 	)
 	b.meta.Datasets = append(b.meta.Datasets, &metastorev1.Dataset{
+		Format:          1,
 		Tenant:          b.meta.Tenant,
 		Name:            0, // Anonymous.
 		MinTime:         b.meta.MinTime,
@@ -352,7 +356,11 @@ func (m *datasetCompaction) open(ctx context.Context, path string) (err error) {
 	for _, s := range m.datasets {
 		s := s
 		g.Go(util.RecoverPanic(func() error {
-			if openErr := s.Open(ctx, allSections...); openErr != nil {
+			if openErr := s.Open(ctx,
+				SectionProfiles,
+				SectionTSDB,
+				SectionSymbols,
+			); openErr != nil {
 				return fmt.Errorf("opening tenant dataset (block %s): %w", s.obj.path, openErr)
 			}
 			return nil