tool: add db analyze-metadata command

internal/ascii/table/table.go

@@ -153,7 +153,7 @@ func (d *Layout[T]) Render(start ascii.Cursor, opts RenderOptions, tuples ...T)
 		// If one of the values exceeds the column width, widen the column as
 		// necessary.
 		for i := range vals {
-			width = max(width, len(vals[i]))
+			width = max(width, len([]rune(vals[i])))
 		}
 		header := f.header()
 		align := f.align()
@@ -162,7 +162,7 @@ func (d *Layout[T]) Render(start ascii.Cursor, opts RenderOptions, tuples ...T)
 		for rowIdx := range vals {
 			if opts.HorizontalDividers.Contains(rowIdx, len(vals)) {
 				rowCur = rowCur.Down(1)
-				rowCur.RepeatByte(width, '-')
+				rowCur.Repeat(width, '-')
 			}
 			rowCur = rowCur.Down(1)
 			pad(rowCur, width, align, vals[rowIdx])

internal/ascii/whiteboard.go

@@ -7,29 +7,37 @@ package ascii
 import (
 	"bytes"
 	"fmt"
+	"slices"
 	"strings"
 )
 
 // Board is a simple ASCII-based board for rendering ASCII text diagrams.
 type Board struct {
-	buf   []byte
+	buf   []rune
 	width int
 }
 
 // Make returns a new Board with the given initial width and height.
 func Make(width, height int) Board {
-	buf := make([]byte, 0, width*height)
+	buf := make([]rune, 0, width*height)
 	return Board{buf: buf, width: width}
 }
 
 // At returns a position at the given coordinates.
 func (b *Board) At(r, c int) Cursor {
 	if r >= b.lines() {
-		b.buf = append(b.buf, bytes.Repeat([]byte{' '}, (r-b.lines()+1)*b.width)...)
+		b.growBuf((r - b.lines() + 1) * b.width)
 	}
 	return Cursor{b: b, r: r, c: c}
 }
 
+func (b *Board) growBuf(n int) {
+	b.buf = slices.Grow(b.buf, n)
+	for range n {
+		b.buf = append(b.buf, ' ')
+	}
+}
+
 // NewLine appends a new line to the board and returns a position at the
 // beginning of the line.
 func (b *Board) NewLine() Cursor {
@@ -50,7 +58,7 @@ func (b *Board) Render(indent string) string {
 			buf.WriteByte('\n')
 		}
 		buf.WriteString(indent)
-		buf.Write(bytes.TrimRight(b.row(r), " "))
+		buf.WriteString(strings.TrimRight(string(b.row(r)), " "))
 	}
 	return buf.String()
 }
@@ -62,16 +70,18 @@ func (b *Board) Reset(w int) {
 }
 
 func (b *Board) write(r, c int, s string) {
-	if c+len(s) > b.width {
-		b.growWidth(c + len(s))
+	if n := len([]rune(s)); c+n > b.width {
+		b.growWidth(c + n)
 	}
 	row := b.row(r)
-	for i := 0; i < len(s); i++ {
-		row[c+i] = s[i]
+	i := 0
+	for _, ch := range s {
+		row[c+i] = ch
+		i++
 	}
 }
 
-func (b *Board) repeat(r, c int, n int, ch byte) {
+func (b *Board) repeat(r, c int, n int, ch rune) {
 	if c+n > b.width {
 		b.growWidth(c + n)
 	}
@@ -82,8 +92,11 @@ func (b *Board) repeat(r, c int, n int, ch byte) {
 }
 
 func (b *Board) growWidth(w int) {
-	buf := bytes.Repeat([]byte{' '}, w*b.lines())
-	for i := 0; i < b.lines(); i++ {
+	buf := make([]rune, w*b.lines())
+	for i := range buf {
+		buf[i] = ' '
+	}
+	for i := range b.lines() {
 		copy(buf[i*w:(i+1)*w], b.buf[i*b.width:(i+1)*b.width])
 	}
 	b.buf = buf
@@ -94,9 +107,9 @@ func (b *Board) lines() int {
 	return len(b.buf) / b.width
 }
 
-func (b *Board) row(r int) []byte {
+func (b *Board) row(r int) []rune {
 	if sz := (r + 1) * b.width; sz > len(b.buf) {
-		b.buf = append(b.buf, bytes.Repeat([]byte{' '}, sz-len(b.buf))...)
+		b.growBuf(sz - len(b.buf))
 	}
 	return b.buf[r*b.width : (r+1)*b.width]
 }
@@ -179,17 +192,17 @@ func (c Cursor) WriteString(s string) Cursor {
 			s = s[i+1:]
 		} else {
 			c.b.write(c.r, c.c, s)
-			c.c += len(s)
+			c.c += len([]rune(s))
 			break
 		}
 	}
 	return c
 }
 
-// RepeatByte writes the given byte n times starting at the cursor, returning a
+// Repeat writes the given character n times starting at the cursor, returning a
 // cursor where the written bytes end.
-func (c Cursor) RepeatByte(n int, b byte) Cursor {
-	c.b.repeat(c.r, c.c, n, b)
+func (c Cursor) Repeat(n int, ch rune) Cursor {
+	c.b.repeat(c.r, c.c, n, ch)
 	return c.Right(n)
 }
 

internal/manifest/version.go

@@ -171,6 +171,10 @@ type Version struct {
 	prev, next *Version
 }
 
+func (v *Version) Comparer() *base.Comparer {
+	return v.cmp
+}
+
 // String implements fmt.Stringer, printing the TableMetadata for each level in
 // the Version.
 func (v *Version) String() string {

sstable/compressionanalyzer/welford.go → internal/metricsutil/welford.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package compressionanalyzer
+package metricsutil
 
 import "math"
 
@@ -34,17 +34,17 @@ func (w *Welford) Mean() float64 {
 	return w.mean
 }
 
-// SampleVariance returns the sample variance (M2/(n-1)). Returns 0 if fewer than 2 values.
-func (w *Welford) SampleVariance() float64 {
+// Variance returns the sample variance (M2/(n-1)). Returns 0 if fewer than 2 values.
+func (w *Welford) Variance() float64 {
 	if w.count < 2 {
 		return 0
 	}
 	return w.m2 / float64(w.count-1)
 }
 
-// SampleStandardDeviation returns the sample standard deviation.
-func (w *Welford) SampleStandardDeviation() float64 {
-	return math.Sqrt(w.SampleVariance())
+// StdDev returns the sample standard deviation.
+func (w *Welford) StdDev() float64 {
+	return math.Sqrt(w.Variance())
 }
 
 // WeightedWelford maintains running statistics for mean and variance using
@@ -75,16 +75,16 @@ func (ww *WeightedWelford) Mean() float64 {
 	return ww.mean
 }
 
-// SampleVariance returns the sample variance (M2/(n-1)). Returns 0 if the sum
-// of added frequencies is less than 2.
-func (ww *WeightedWelford) SampleVariance() float64 {
+// Variance returns the sample variance (M2/(n-1)). Returns 0 if the sum of
+// added frequencies is less than 2.
+func (ww *WeightedWelford) Variance() float64 {
 	if ww.wSum < 2 {
 		return 0
 	}
 	return ww.s / (ww.wSum - 1)
 }
 
-// SampleStandardDeviation returns the sample standard deviation.
-func (ww *WeightedWelford) SampleStandardDeviation() float64 {
-	return math.Sqrt(ww.SampleVariance())
+// StdDev returns the sample standard deviation.
+func (ww *WeightedWelford) StdDev() float64 {
+	return math.Sqrt(ww.Variance())
 }

sstable/compressionanalyzer/welford_test.go → internal/metricsutil/welford_test.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package compressionanalyzer
+package metricsutil
 
 import (
 	"math"
@@ -73,7 +73,7 @@ func TestWelfordBasic(t *testing.T) {
 			if !almostEqual(gotMean, tc.wantMean) {
 				t.Errorf("Mean = %v; want %v", gotMean, tc.wantMean)
 			}
-			gotSample := w.SampleVariance()
+			gotSample := w.Variance()
 			if !almostEqual(gotSample, tc.wantVarSample) {
 				t.Errorf("Variance (sample) = %v; want %v", gotSample, tc.wantVarSample)
 			}
@@ -135,7 +135,7 @@ func TestWeightedWelford(t *testing.T) {
 			if !almostEqual(gotMean, tc.wantMean) {
 				t.Errorf("Mean = %v; want %v", gotMean, tc.wantMean)
 			}
-			gotSample := w.SampleVariance()
+			gotSample := w.Variance()
 			if !almostEqual(gotSample, tc.wantVarSample) {
 				t.Errorf("Variance (sample) = %v; want %v", gotSample, tc.wantVarSample)
 			}

sstable/compressionanalyzer/buckets.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/pebble/internal/compression"
+	"github.com/cockroachdb/pebble/internal/metricsutil"
 	"github.com/cockroachdb/pebble/sstable/block"
 	"github.com/cockroachdb/pebble/sstable/block/blockkind"
 )
@@ -156,17 +157,17 @@ type Buckets [blockkind.NumKinds][numBlockSizes][numCompressibility]Bucket
 // Bucket aggregates results for blocks of the same kind, size range, and
 // compressibility.
 type Bucket struct {
-	UncompressedSize Welford
+	UncompressedSize metricsutil.Welford
 	Experiments      [numProfiles]PerProfile
 }
 
 // PerProfile holds statistics from experiments on blocks in a bucket with a
 // specific compression.Setting.
 type PerProfile struct {
-	CompressionRatio WeightedWelford
+	CompressionRatio metricsutil.WeightedWelford
 	// CPU times are in nanoseconds per uncompressed byte.
-	CompressionTime   WeightedWelford
-	DecompressionTime WeightedWelford
+	CompressionTime   metricsutil.WeightedWelford
+	DecompressionTime metricsutil.WeightedWelford
 }
 
 func (b *Buckets) String(minSamples int) string {
@@ -185,21 +186,21 @@ func (b *Buckets) String(minSamples int) string {
 				if bucket.UncompressedSize.Count() < int64(minSamples) {
 					continue
 				}
-				fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%.1fKB %s\tCR", k, sz, c, bucket.UncompressedSize.Count(), bucket.UncompressedSize.Mean()/1024, stdDevStr(bucket.UncompressedSize.Mean(), bucket.UncompressedSize.SampleStandardDeviation()))
+				fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%.1fKB %s\tCR", k, sz, c, bucket.UncompressedSize.Count(), bucket.UncompressedSize.Mean()/1024, stdDevStr(bucket.UncompressedSize.Mean(), bucket.UncompressedSize.StdDev()))
 				for _, e := range (*b)[k][sz][c].Experiments {
-					mean, stdDev := e.CompressionRatio.Mean(), e.CompressionRatio.SampleStandardDeviation()
+					mean, stdDev := e.CompressionRatio.Mean(), e.CompressionRatio.StdDev()
 					fmt.Fprintf(tw, "\t%.2f %s", mean, stdDevStr(mean, stdDev))
 				}
 				fmt.Fprintf(tw, "\n")
 				fmt.Fprintf(tw, "\t\t\t\t\tComp")
 				for _, e := range (*b)[k][sz][c].Experiments {
-					mean, stdDev := e.CompressionTime.Mean(), e.CompressionTime.SampleStandardDeviation()
+					mean, stdDev := e.CompressionTime.Mean(), e.CompressionTime.StdDev()
 					fmt.Fprintf(tw, "\t%.0fMBps %s", toMBPS(mean), stdDevStr(mean, stdDev))
 				}
 				fmt.Fprintf(tw, "\n")
 				fmt.Fprintf(tw, "\t\t\t\t\tDecomp")
 				for _, e := range (*b)[k][sz][c].Experiments {
-					mean, stdDev := e.DecompressionTime.Mean(), e.DecompressionTime.SampleStandardDeviation()
+					mean, stdDev := e.DecompressionTime.Mean(), e.DecompressionTime.StdDev()
 					fmt.Fprintf(tw, "\t%.0fMBps %s", toMBPS(mean), stdDevStr(mean, stdDev))
 				}
 				fmt.Fprintf(tw, "\n")
@@ -247,11 +248,11 @@ func (b *Buckets) ToCSV(minSamples int) string {
 				if bucket.UncompressedSize.Count() < int64(minSamples) {
 					continue
 				}
-				fmt.Fprintf(&buf, "%s,%s,%s,%d,%.0f,%.0f", k, sz, c, bucket.UncompressedSize.Count(), bucket.UncompressedSize.Mean(), bucket.UncompressedSize.SampleStandardDeviation())
+				fmt.Fprintf(&buf, "%s,%s,%s,%d,%.0f,%.0f", k, sz, c, bucket.UncompressedSize.Count(), bucket.UncompressedSize.Mean(), bucket.UncompressedSize.StdDev())
 				for _, e := range (*b)[k][sz][c].Experiments {
-					fmt.Fprintf(&buf, ",%.3f,%.3f", e.CompressionRatio.Mean(), e.CompressionRatio.SampleStandardDeviation())
-					fmt.Fprintf(&buf, ",%.3f,%.3f", e.CompressionTime.Mean(), e.CompressionTime.SampleStandardDeviation())
-					fmt.Fprintf(&buf, ",%.3f,%.3f", e.DecompressionTime.Mean(), e.DecompressionTime.SampleStandardDeviation())
+					fmt.Fprintf(&buf, ",%.3f,%.3f", e.CompressionRatio.Mean(), e.CompressionRatio.StdDev())
+					fmt.Fprintf(&buf, ",%.3f,%.3f", e.CompressionTime.Mean(), e.CompressionTime.StdDev())
+					fmt.Fprintf(&buf, ",%.3f,%.3f", e.DecompressionTime.Mean(), e.DecompressionTime.StdDev())
 				}
 				fmt.Fprintf(&buf, "\n")
 			}

sstable/compressionanalyzer/file_analyzer_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/cockroachdb/crlib/crstrings"
 	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/pebble/internal/metricsutil"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
@@ -44,10 +45,10 @@ func TestFileAnalyzer(t *testing.T) {
 							// Snappy always has the same output in all configurations and on
 							// all platforms.
 							if Profiles[l].Name != "Snappy" {
-								bucket.Experiments[l].CompressionRatio = WeightedWelford{}
+								bucket.Experiments[l].CompressionRatio = metricsutil.WeightedWelford{}
 							}
-							bucket.Experiments[l].CompressionTime = WeightedWelford{}
-							bucket.Experiments[l].DecompressionTime = WeightedWelford{}
+							bucket.Experiments[l].CompressionTime = metricsutil.WeightedWelford{}
+							bucket.Experiments[l].DecompressionTime = metricsutil.WeightedWelford{}
 						}
 					}
 				}