[perforator] Introduce profile statistics

commit_hash:1c8b5f2fc38f54d638ce2bb723f73e46a70efa30

perforator/internal/symbolizer/proxy/server/server.go

@@ -50,6 +50,7 @@ import (
 	"github.com/yandex/perforator/perforator/pkg/polyheapprof"
 	"github.com/yandex/perforator/perforator/pkg/profile/flamegraph/render"
 	"github.com/yandex/perforator/perforator/pkg/profile/python"
+	"github.com/yandex/perforator/perforator/pkg/profile/quality"
 	"github.com/yandex/perforator/perforator/pkg/profile/samplefilter"
 	"github.com/yandex/perforator/perforator/pkg/profilequerylang"
 	"github.com/yandex/perforator/perforator/pkg/sampletype"
@@ -1084,15 +1085,19 @@ func (s *PerforatorServer) MergeProfiles(
 		return nil, err
 	}
 
+	statistics := quality.CalculateProfileStatistics(mergedProfile)
+
 	if url != "" {
 		return &perforator.MergeProfilesResponse{
 			Result:      &perforator.MergeProfilesResponse_ProfileURL{ProfileURL: url},
 			ProfileMeta: extractProtoMetasFromRawProfiles(rawProfiles),
+			Statistics:  statistics,
 		}, nil
 	} else {
 		return &perforator.MergeProfilesResponse{
 			Result:      &perforator.MergeProfilesResponse_Profile{Profile: buf},
 			ProfileMeta: extractProtoMetasFromRawProfiles(rawProfiles),
+			Statistics:  statistics,
 		}, nil
 	}
 }

perforator/pkg/profile/quality/statistics.go

@@ -0,0 +1,60 @@
+package quality
+
+import (
+	"github.com/google/pprof/profile"
+
+	"github.com/yandex/perforator/perforator/proto/perforator"
+)
+
+func CalculateProfileStatistics(profile *profile.Profile) *perforator.ProfileStatistics {
+	stats := &perforator.ProfileStatistics{
+		SampleValueSum:         make(map[string]float64),
+		UniqueSampleCount:      uint64(len(profile.Sample)),
+		TotalFrameCount:        0,
+		UnmappedFrameCount:     0,
+		UnsymbolizedFrameCount: 0,
+		TotalBinaryCount:       0,
+		UnavailableBinaryCount: 0,
+	}
+
+	referencedBinaries := make(map[string]struct{})
+	symbolizedBinaries := make(map[string]struct{})
+
+	types := make([]string, len(profile.SampleType))
+	for i, typ := range profile.SampleType {
+		types[i] = typ.Type + "." + typ.Unit
+	}
+
+	for _, sample := range profile.Sample {
+		for i, value := range sample.Value {
+			stats.SampleValueSum[types[i]] += float64(value)
+		}
+
+		stats.TotalFrameCount += uint64(len(sample.Location))
+		for _, location := range sample.Location {
+			symbolized := len(location.Line) > 0
+			if !symbolized {
+				stats.UnsymbolizedFrameCount++
+			}
+
+			if location.Mapping == nil {
+				stats.UnmappedFrameCount++
+				continue
+			}
+
+			referencedBinaries[location.Mapping.BuildID] = struct{}{}
+			if len(location.Line) > 0 {
+				symbolizedBinaries[location.Mapping.BuildID] = struct{}{}
+			}
+		}
+	}
+
+	stats.TotalBinaryCount = uint64(len(referencedBinaries))
+	for id := range referencedBinaries {
+		if _, ok := symbolizedBinaries[id]; !ok {
+			stats.UnavailableBinaryCount++
+		}
+	}
+
+	return stats
+}

perforator/pkg/profile/quality/ya.make

@@ -0,0 +1,7 @@
+GO_LIBRARY()
+
+SRCS(
+    statistics.go
+)
+
+END()

perforator/pkg/profile/ya.make

@@ -3,6 +3,7 @@ RECURSE(
     labels
     parse
     python
+    quality
     samplefilter
 )
 

perforator/proto/perforator/perforator.proto

@@ -231,6 +231,46 @@ message MergeProfilesResponse {
 
     // Info about profiles that participated in the merged profile.
     repeated ProfileMeta ProfileMeta = 2;
+
+    // Merged profile statistics.
+    // Can be used to estimate quality of the profile.
+    // See perforator/pkg/profile/quality.
+    ProfileStatistics Statistics = 4;
+}
+
+message ProfileStatistics {
+    // Sum of the profile samples values. A profile sample can contain multiple
+    // values. For example, CPU profiles produced by Perforator often contain
+    // two sample types: "cpu.cycles" & "wall.seconds". For such profiles, the
+    // sample_value_sum will have two entries:
+    // {"cpu.cycles": sum_of_cpu_cycles, "wall.seconds": sum_of_walltime_seconds}.
+    map<string, double> sample_value_sum = 1;
+
+    // Number of unique sample keys in the profile.
+    // Sample key is a unique combination of sample stack and labels.
+    // If this value is low, the profile is probably malformed.
+    uint64 unique_sample_count = 2;
+
+    // Number of stack frames in the profile. Average stack depth can be
+    // computed as total_frame_count / unique_sample_count.
+    uint64 total_frame_count = 3;
+
+    // Number of unmapped frames (frames which were not attributed to a binary).
+    // If this value is high, then the profile is probably malformed.
+    uint64 unmapped_frame_count = 5;
+
+    // Number of unsymbolized frames (frames without source line info).
+    // If this value is high, then the profile is probably not readable.
+    uint64 unsymbolized_frame_count = 6;
+
+    // Number of different executable binaries in the profile.
+    uint64 total_binary_count = 7;
+
+    // Number of unavailable binaries in the profile. The binary is unavailable
+    // when the Perforator backend was not available to fetch it by build id,
+    // so frames from that binary are probably not symbolized.
+    // If this value is high, the profile is probably not readable.
+    uint64 unavailable_binary_count = 8;
 }
 
 ////////////////////////////////////////////////////////////////////////////////