Merge pull request #440 from zenador/sparsehistograms

Optimise JSON marshalling for sparse histograms

Author: codesome

go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/go-kit/log v0.2.1
 	github.com/golang/protobuf v1.5.2
+	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/matttproud/golang_protobuf_extensions v1.0.4
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
@@ -23,6 +24,8 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/stretchr/testify v1.8.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect

go.sum

@@ -139,6 +139,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -159,9 +160,11 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=

model/value.go

@@ -100,37 +100,19 @@ func (s Sample) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&v)
 }
 
-type sampleHistogramPairPtr struct {
-	Timestamp Time
-	Histogram *SampleHistogram
-}
-
-func (s *sampleHistogramPairPtr) UnmarshalJSON(buf []byte) error {
-	tmp := []interface{}{&s.Timestamp, &s.Histogram}
-	wantLen := len(tmp)
-	if err := json.Unmarshal(buf, &tmp); err != nil {
-		return err
-	}
-	if gotLen := len(tmp); gotLen != wantLen {
-		return fmt.Errorf("wrong number of fields: %d != %d", gotLen, wantLen)
-	}
-	return nil
-}
-
 // UnmarshalJSON implements json.Unmarshaler.
-// TODO: simplify and remove the need for both sampleHistogramPairPtr and SampleHistogramPair
 func (s *Sample) UnmarshalJSON(b []byte) error {
 	v := struct {
-		Metric    Metric                 `json:"metric"`
-		Value     SamplePair             `json:"value"`
-		Histogram sampleHistogramPairPtr `json:"histogram"`
+		Metric    Metric              `json:"metric"`
+		Value     SamplePair          `json:"value"`
+		Histogram SampleHistogramPair `json:"histogram"`
 	}{
 		Metric: s.Metric,
 		Value: SamplePair{
 			Timestamp: s.Timestamp,
 			Value:     s.Value,
 		},
-		Histogram: sampleHistogramPairPtr{
+		Histogram: SampleHistogramPair{
 			Timestamp: s.Timestamp,
 			Histogram: s.Histogram,
 		},

model/value_float.go

@@ -18,8 +18,15 @@ import (
 	"fmt"
 	"math"
 	"strconv"
+	"unsafe"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
+func init() {
+	jsoniter.RegisterTypeEncoderFunc("model.SamplePair", marshalSamplePairJSON, marshalJSONIsEmpty)
+}
+
 var (
 	// ZeroSamplePair is the pseudo zero-value of SamplePair used to signal a
 	// non-existing sample pair. It is a SamplePair with timestamp Earliest and
@@ -71,17 +78,18 @@ type SamplePair struct {
 	Value     SampleValue
 }
 
-// MarshalJSON implements json.Marshaler.
+// marshalSamplePairJSON writes `[ts, "val"]`.
+func marshalSamplePairJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	p := *((*SamplePair)(ptr))
+	stream.WriteArrayStart()
+	MarshalTimestamp(int64(p.Timestamp), stream)
+	stream.WriteMore()
+	MarshalValue(float64(p.Value), stream)
+	stream.WriteArrayEnd()
+}
+
 func (s SamplePair) MarshalJSON() ([]byte, error) {
-	t, err := json.Marshal(s.Timestamp)
-	if err != nil {
-		return nil, err
-	}
-	v, err := json.Marshal(s.Value)
-	if err != nil {
-		return nil, err
-	}
-	return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
+	return jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(s)
 }
 
 // UnmarshalJSON implements json.Unmarshaler.

model/value_float_test.go

@@ -20,6 +20,42 @@ import (
 	"testing"
 )
 
+var (
+	samplePairMatrixPlain = `[{"metric":{"__name__":"test_metric"},"values":[[1234.567,"123.1"],[12345.678,"123.12"]]},{"metric":{"foo":"bar"},"values":[[2234.567,"223.1"],[22345.678,"223.12"]]}]`
+	samplePairMatrixValue = Matrix{
+		&SampleStream{
+			Metric: Metric{
+				MetricNameLabel: "test_metric",
+			},
+			Values: []SamplePair{
+				{
+					Value:     123.1,
+					Timestamp: 1234567,
+				},
+				{
+					Value:     123.12,
+					Timestamp: 12345678,
+				},
+			},
+		},
+		&SampleStream{
+			Metric: Metric{
+				"foo": "bar",
+			},
+			Values: []SamplePair{
+				{
+					Value:     223.1,
+					Timestamp: 2234567,
+				},
+				{
+					Value:     223.12,
+					Timestamp: 22345678,
+				},
+			},
+		},
+	}
+)
+
 func TestEqualValues(t *testing.T) {
 	tests := map[string]struct {
 		in1, in2 SampleValue
@@ -231,39 +267,8 @@ func TestMatrixJSON(t *testing.T) {
 			value: Matrix{},
 		},
 		{
-			plain: `[{"metric":{"__name__":"test_metric"},"values":[[1234.567,"123.1"],[12345.678,"123.12"]]},{"metric":{"foo":"bar"},"values":[[2234.567,"223.1"],[22345.678,"223.12"]]}]`,
-			value: Matrix{
-				&SampleStream{
-					Metric: Metric{
-						MetricNameLabel: "test_metric",
-					},
-					Values: []SamplePair{
-						{
-							Value:     123.1,
-							Timestamp: 1234567,
-						},
-						{
-							Value:     123.12,
-							Timestamp: 12345678,
-						},
-					},
-				},
-				&SampleStream{
-					Metric: Metric{
-						"foo": "bar",
-					},
-					Values: []SamplePair{
-						{
-							Value:     223.1,
-							Timestamp: 2234567,
-						},
-						{
-							Value:     223.12,
-							Timestamp: 22345678,
-						},
-					},
-				},
-			},
+			plain: samplePairMatrixPlain,
+			value: samplePairMatrixValue,
 		},
 	}
 
@@ -291,3 +296,12 @@ func TestMatrixJSON(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkJSONMarshallingSamplePairMatrix(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_, err := json.Marshal(samplePairMatrixValue)
+		if err != nil {
+			b.Fatal("error marshalling")
+		}
+	}
+}

model/value_histogram.go

@@ -18,8 +18,16 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"unsafe"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
+func init() {
+	jsoniter.RegisterTypeEncoderFunc("model.HistogramBucket", marshalHistogramBucketJSON, marshalJSONIsEmpty)
+	jsoniter.RegisterTypeEncoderFunc("model.SampleHistogramPair", marshalSampleHistogramPairJSON, marshalJSONIsEmpty)
+}
+
 type FloatString float64
 
 func (v FloatString) String() string {
@@ -49,24 +57,10 @@ type HistogramBucket struct {
 	Count      FloatString
 }
 
-func (s HistogramBucket) MarshalJSON() ([]byte, error) {
-	b, err := json.Marshal(s.Boundaries)
-	if err != nil {
-		return nil, err
-	}
-	l, err := json.Marshal(s.Lower)
-	if err != nil {
-		return nil, err
-	}
-	u, err := json.Marshal(s.Upper)
-	if err != nil {
-		return nil, err
-	}
-	c, err := json.Marshal(s.Count)
-	if err != nil {
-		return nil, err
-	}
-	return []byte(fmt.Sprintf("[%s,%s,%s,%s]", b, l, u, c)), nil
+// marshalHistogramBucketJSON writes fmt.Sprintf("[%s,%s,%s,%s]", b.Boundaries, b.Lower, b.Upper, b.Count).
+func marshalHistogramBucketJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	b := *((*HistogramBucket)(ptr))
+	MarshalHistogramBucket(b, stream)
 }
 
 func (s *HistogramBucket) UnmarshalJSON(buf []byte) error {
@@ -139,19 +133,21 @@ type SampleHistogramPair struct {
 	Histogram *SampleHistogram
 }
 
+// marshalSampleHistogramPairJSON writes `[ts, "val"]`.
+func marshalSampleHistogramPairJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	p := *((*SampleHistogramPair)(ptr))
+	stream.WriteArrayStart()
+	MarshalTimestamp(int64(p.Timestamp), stream)
+	stream.WriteMore()
+	MarshalHistogram(*p.Histogram, stream)
+	stream.WriteArrayEnd()
+}
+
 func (s SampleHistogramPair) MarshalJSON() ([]byte, error) {
-	t, err := json.Marshal(s.Timestamp)
-	if err != nil {
-		return nil, err
-	}
 	if s.Histogram == nil {
 		return nil, fmt.Errorf("histogram is nil")
 	}
-	v, err := json.Marshal(s.Histogram)
-	if err != nil {
-		return nil, err
-	}
-	return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
+	return jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(s)
 }
 
 func (s *SampleHistogramPair) UnmarshalJSON(buf []byte) error {