Skip to content

Add better support for thread id #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: master
Choose a base branch
from
Open

Add better support for thread id #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
624fb49
Add better support for thread id
winmutt Sep 12, 2018
32c312f
Fmted
winmutt Sep 12, 2018
d4cf8d8
Moved Thread_id out of NumberMetrics and names it ThreadId in the eve…
winmutt Oct 16, 2018
ddc411e
add LastOffset, fix Offset update
gywndi Aug 27, 2018
8644b73
PMM-2859: Add test with new time format.
arvenil Sep 18, 2018
40896b2
Fixed some tests failing around thread_id being removed from NumberMe…
winmutt Oct 16, 2018
290bf29
Merge branch 'master' of https://github.com/percona/go-mysql into thr…
winmutt Oct 16, 2018
cf6963e
Making houndci happy with linting
winmutt Oct 16, 2018
1315ef2
Removed odd commit effect from rebase
winmutt Oct 16, 2018
b39efcb
Added support for Ts_min and Ts_max. Added support for ThreadID to ag…
winmutt Oct 22, 2018
37cd562
Linted and fmted
winmutt Oct 22, 2018
8904558
Linted
winmutt Nov 5, 2018
ccc6f59
Updated testdata to reflect Id->ID
winmutt Nov 14, 2018
9e9def2
Merge branch 'master' of https://github.com/percona/go-mysql into thr…
Mar 5, 2019
956c72a
Removed Thread_id as Id is the only part that is needed to match
Mar 8, 2019
b6be54d
Merge branch 'master' of https://github.com/percona/go-mysql into thr…
winmutt Mar 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions event/aggregator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,18 @@ func (a *Aggregator) Finalize() Result {
a.global.Finalize(a.rateLimit)
a.global.UniqueQueries = uint(len(a.classes))
for _, class := range a.classes {
if class.LastThreadID != 0 {
a.global.LastThreadID = class.LastThreadID
}

if !class.TsMin.IsZero() && class.TsMin.Before(a.global.TsMin) {
a.global.TsMin = class.TsMin
}

if !class.TsMax.IsZero() && class.TsMax.After(a.global.TsMax) {
a.global.TsMax = class.TsMax
}

class.Finalize(a.rateLimit)
class.UniqueQueries = 1
if class.Example != nil && class.Example.Ts != "" {
Expand Down
8 changes: 7 additions & 1 deletion event/aggregator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"strings"
"testing"
"time"

"github.com/percona/go-mysql/event"
"github.com/percona/go-mysql/log"
parser "github.com/percona/go-mysql/log/slow"
Expand Down Expand Up @@ -148,10 +147,16 @@ func TestAddClassSlow001(t *testing.T) {
for _, class := range expectEventResult.Class {
global.AddClass(class)
}

var emptyTime time.Time
expectEventResult.Global.TsMin = emptyTime;
expectEventResult.Global.TsMax = emptyTime;
expectEventResult.Global.LastThreadID = 0;
expectGlobalBytes, err := json.Marshal(expectEventResult.Global)
if err != nil {
t.Fatal(err)
}

gotGlobalBytes, err := json.Marshal(global)
if err != nil {
t.Fatal(err)
Expand All @@ -171,6 +176,7 @@ func TestAddClassSlow023(t *testing.T) {
for _, class := range ordered(expectEventResult.Class) {
global.AddClass(class)
}
expectEventResult.Global.LastThreadID = 0;
expectGlobalBytes, err := json.Marshal(expectEventResult.Global)
if err != nil {
t.Fatal(err)
Expand Down
36 changes: 26 additions & 10 deletions event/class.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package event

import (
"time"

"github.com/percona/go-mysql/log"
)

Expand All @@ -33,16 +35,19 @@ const (
// This is only enforced by convention, so be careful not to mix events from
// different classes.
type Class struct {
Id string // 32-character hex checksum of fingerprint
Fingerprint string // canonical form of query: values replaced with "?"
Metrics *Metrics // statistics for each metric, e.g. max Query_time
TotalQueries uint // total number of queries in class
UniqueQueries uint // unique number of queries in class
Example *Example `json:",omitempty"` // sample query with max Query_time
ID string // 32-character hex checksum of fingerprint
Fingerprint string // canonical form of query: values replaced with "?"
TsMin time.Time // timestamp of first event
TsMax time.Time // timestamp of last event
Metrics *Metrics // statistics for each metric, e.g. max Query_time
TotalQueries uint // total number of queries in class
UniqueQueries uint // unique number of queries in class
Example *Example `json:",omitempty"` // sample query with max Query_time
// --
outliers uint
lastDb string
sample bool
LastThreadID uint64 // Thread_id of query
outliers uint
lastDb string
sample bool
}

// A Example is a real query and its database, timestamp, and Query_time.
Expand All @@ -60,7 +65,7 @@ type Example struct {
// If sample is true, the query with the greatest Query_time is saved.
func NewClass(id, fingerprint string, sample bool) *Class {
class := &Class{
Id: id,
ID: id,
Fingerprint: fingerprint,
Metrics: NewMetrics(),
TotalQueries: 0,
Expand All @@ -80,6 +85,17 @@ func (c *Class) AddEvent(e *log.Event, outlier bool) {

c.Metrics.AddEvent(e, outlier)

if e.ThreadID != 0 {
c.LastThreadID = e.ThreadID
}

if !e.Ts.IsZero() {
if c.TsMin.IsZero() {
c.TsMin = e.Ts
}
c.TsMax = e.Ts
}

// Save last db seen for this query. This helps ensure the sample query
// has a db.
if e.Db != "" {
Expand Down
30 changes: 10 additions & 20 deletions event/testdata/mariadb102-with-explain.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"Global": {
"Id": "",
"ID": "",
"Fingerprint": "",
"TsMin": "2018-02-14T16:18:07Z",
"TsMax": "2018-02-14T16:18:07Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -45,14 +47,6 @@
"Med": 1,
"P95": 1,
"Max": 1
},
"Thread_id": {
"Sum": 8,
"Min": 8,
"Avg": 8,
"Med": 8,
"P95": 8,
"Max": 8
}
},
"BoolMetrics": {
Expand All @@ -62,12 +56,15 @@
}
},
"TotalQueries": 1,
"UniqueQueries": 1
"UniqueQueries": 1,
"LastThreadID": 8
},
"Class": {
"16219655761820A2": {
"Id": "16219655761820A2",
"ID": "16219655761820A2",
"Fingerprint": "select ?",
"TsMin": "2018-02-14T16:18:07Z",
"TsMax": "2018-02-14T16:18:07Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -111,14 +108,6 @@
"Med": 1,
"P95": 1,
"Max": 1
},
"Thread_id": {
"Sum": 8,
"Min": 8,
"Avg": 8,
"Med": 8,
"P95": 8,
"Max": 8
}
},
"BoolMetrics": {
Expand All @@ -135,7 +124,8 @@
"Query": "SELECT 1",
"Size": 8,
"Ts": "2018-02-14 16:18:07"
}
},
"LastThreadID": 8
}
},
"RateLimit": 0,
Expand Down
21 changes: 15 additions & 6 deletions event/testdata/slow001-no-examples.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"Global": {
"Id": "",
"ID": "",
"Fingerprint": "",
"TsMin": "2007-10-15T21:43:52Z",
"TsMax": "2007-10-15T21:45:10Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -41,12 +43,15 @@
}
},
"TotalQueries": 2,
"UniqueQueries": 2
"UniqueQueries": 2,
"LastThreadID": 0
},
"Class": {
"3A99CC42AEDCCFCD": {
"Id": "3A99CC42AEDCCFCD",
"ID": "3A99CC42AEDCCFCD",
"Fingerprint": "select sleep(?) from test.n",
"TsMin": "2007-10-15T21:45:10Z",
"TsMax": "2007-10-15T21:45:10Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -86,11 +91,14 @@
}
},
"TotalQueries": 1,
"UniqueQueries": 1
"UniqueQueries": 1,
"LastThreadID": 0
},
"7F7D57ACDD8A346E": {
"Id": "7F7D57ACDD8A346E",
"ID": "7F7D57ACDD8A346E",
"Fingerprint": "select sleep(?) from n",
"TsMin": "2007-10-15T21:43:52Z",
"TsMax": "2007-10-15T21:43:52Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -130,7 +138,8 @@
}
},
"TotalQueries": 1,
"UniqueQueries": 1
"UniqueQueries": 1,
"LastThreadID": 0
}
},
"RateLimit": 0,
Expand Down
21 changes: 15 additions & 6 deletions event/testdata/slow001.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"Global": {
"Id": "",
"ID": "",
"Fingerprint": "",
"TsMin": "2007-10-15T21:43:52Z",
"TsMax": "2007-10-15T21:45:10Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -41,12 +43,15 @@
}
},
"TotalQueries": 2,
"UniqueQueries": 2
"UniqueQueries": 2,
"LastThreadID": 0
},
"Class": {
"3A99CC42AEDCCFCD": {
"Id": "3A99CC42AEDCCFCD",
"ID": "3A99CC42AEDCCFCD",
"Fingerprint": "select sleep(?) from test.n",
"TsMin": "2007-10-15T21:45:10Z",
"TsMax": "2007-10-15T21:45:10Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -93,11 +98,14 @@
"Query": "select sleep(2) from test.n",
"Size": 27,
"Ts": "2007-10-15 21:45:10"
}
},
"LastThreadID": 0
},
"7F7D57ACDD8A346E": {
"Id": "7F7D57ACDD8A346E",
"ID": "7F7D57ACDD8A346E",
"Fingerprint": "select sleep(?) from n",
"TsMin": "2007-10-15T21:43:52Z",
"TsMax": "2007-10-15T21:43:52Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -144,7 +152,8 @@
"Query": "select sleep(2) from n",
"Size": 22,
"Ts": "2007-10-15 21:43:52"
}
},
"LastThreadID": 0
}
},
"RateLimit": 0,
Expand Down
14 changes: 10 additions & 4 deletions event/testdata/slow010.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"Global": {
"Id": "",
"ID": "",
"Fingerprint": "",
"TsMin": "2007-12-18T11:48:27Z",
"TsMax": "2007-12-18T11:48:27Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -33,12 +35,15 @@
}
},
"TotalQueries": 36,
"UniqueQueries": 1
"UniqueQueries": 1,
"LastThreadID": 0
},
"Class": {
"CB5621E548E5497F": {
"Id": "CB5621E548E5497F",
"ID": "CB5621E548E5497F",
"Fingerprint": "select c from t where id=?",
"TsMin": "2007-12-18T11:48:27Z",
"TsMax": "2007-12-18T11:48:27Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -77,7 +82,8 @@
"Query": "SELECT c FROM t WHERE id=1",
"Size": 26,
"Ts": "2007-12-18 11:48:27"
}
},
"LastThreadID": 0
}
},
"RateLimit": 0,
Expand Down
14 changes: 10 additions & 4 deletions event/testdata/slow018.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"Global": {
"Id": "",
"ID": "",
"Fingerprint": "",
"TsMin": "0001-01-01T00:00:00Z",
"TsMax": "0001-01-01T00:00:00Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -72,12 +74,15 @@
}
},
"TotalQueries": 4,
"UniqueQueries": 1
"UniqueQueries": 1,
"LastThreadID": 0
},
"Class": {
"B2414E722E8A89DD": {
"Id": "B2414E722E8A89DD",
"ID": "B2414E722E8A89DD",
"Fingerprint": "select * from t where id in(?+)",
"TsMin": "0001-01-01T00:00:00Z",
"TsMax": "0001-01-01T00:00:00Z",
"Metrics": {
"TimeMetrics": {
"Lock_time": {
Expand Down Expand Up @@ -154,7 +159,8 @@
"Db": "db1",
"Query": "select * from t where id in (10000,20000,30000)",
"Size": 47
}
},
"LastThreadID": 0
}
},
"RateLimit": 0,
Expand Down
Loading