Skip to content

Commit 8676d3d

Browse files
dongjiang1989SuperQ
dongjiang1989
and
SuperQ
authored
dongjiang, add nfs over RDMA proc parse (#558)
Signed-off-by: dongjiang1989 <[email protected]> Co-authored-by: Ben Kochie <[email protected]>
1 parent 607adcf commit 8676d3d

File tree

2 files changed

+229
-7
lines changed

2 files changed

+229
-7
lines changed

‎mountstats.go

Lines changed: 76 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ const (
4444

4545
fieldTransport11TCPLen = 13
4646
fieldTransport11UDPLen = 10
47+
48+
// kernel version >= 4.14 MaxLen
49+
// See: https://elixir.bootlin.com/linux/v6.4.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L393
50+
fieldTransport11RDMAMaxLen = 28
51+
52+
// kernel version <= 4.2 MinLen
53+
// See: https://elixir.bootlin.com/linux/v4.2.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L331
54+
fieldTransport11RDMAMinLen = 20
4755
)
4856

4957
// A Mount is a device mount parsed from /proc/[pid]/mountstats.
@@ -233,6 +241,33 @@ type NFSTransportStats struct {
233241
// A running counter, incremented on each request as the current size of the
234242
// pending queue.
235243
CumulativePendingQueue uint64
244+
245+
// Stats below only available with stat version 1.1.
246+
// Transport over RDMA
247+
248+
// accessed when sending a call
249+
ReadChunkCount uint64
250+
WriteChunkCount uint64
251+
ReplyChunkCount uint64
252+
TotalRdmaRequest uint64
253+
254+
// rarely accessed error counters
255+
PullupCopyCount uint64
256+
HardwayRegisterCount uint64
257+
FailedMarshalCount uint64
258+
BadReplyCount uint64
259+
MrsRecovered uint64
260+
MrsOrphaned uint64
261+
MrsAllocated uint64
262+
EmptySendctxQ uint64
263+
264+
// accessed when receiving a reply
265+
TotalRdmaReply uint64
266+
FixupCopyCount uint64
267+
ReplyWaitsForSend uint64
268+
LocalInvNeeded uint64
269+
NomsgCallCount uint64
270+
BcallCount uint64
236271
}
237272

238273
// parseMountStats parses a /proc/[pid]/mountstats file and returns a slice
@@ -587,14 +622,17 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
587622
expectedLength = fieldTransport11TCPLen
588623
} else if protocol == "udp" {
589624
expectedLength = fieldTransport11UDPLen
625+
} else if protocol == "rdma" {
626+
expectedLength = fieldTransport11RDMAMinLen
590627
} else {
591628
return nil, fmt.Errorf("%w: invalid NFS protocol \"%s\" in stats 1.1 statement: %v", ErrFileParse, protocol, ss)
592629
}
593-
if len(ss) != expectedLength {
594-
return nil, fmt.Errorf("%w: invalid NFS transport stats 1.1 statement: %v", ErrFileParse, ss)
630+
if (len(ss) != expectedLength && (protocol == "tcp" || protocol == "udp")) ||
631+
(protocol == "rdma" && len(ss) < expectedLength) {
632+
return nil, fmt.Errorf("%w: invalid NFS transport stats 1.1 statement: %v, protocol: %v", ErrFileParse, ss, protocol)
595633
}
596634
default:
597-
return nil, fmt.Errorf("%s: Unrecognized NFS transport stats version: %q", ErrFileParse, statVersion)
635+
return nil, fmt.Errorf("%s: Unrecognized NFS transport stats version: %q, protocol: %v", ErrFileParse, statVersion, protocol)
598636
}
599637

600638
// Allocate enough for v1.1 stats since zero value for v1.1 stats will be okay
@@ -604,7 +642,9 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
604642
// Note: slice length must be set to length of v1.1 stats to avoid a panic when
605643
// only v1.0 stats are present.
606644
// See: https://github.com/prometheus/node_exporter/issues/571.
607-
ns := make([]uint64, fieldTransport11TCPLen)
645+
//
646+
// Note: NFS Over RDMA slice length is fieldTransport11RDMAMaxLen
647+
ns := make([]uint64, fieldTransport11RDMAMaxLen+3)
608648
for i, s := range ss {
609649
n, err := strconv.ParseUint(s, 10, 64)
610650
if err != nil {
@@ -622,9 +662,14 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
622662
// we set them to 0 here.
623663
if protocol == "udp" {
624664
ns = append(ns[:2], append(make([]uint64, 3), ns[2:]...)...)
665+
} else if protocol == "tcp" {
666+
ns = append(ns[:fieldTransport11TCPLen], make([]uint64, fieldTransport11RDMAMaxLen-fieldTransport11TCPLen+3)...)
667+
} else if protocol == "rdma" {
668+
ns = append(ns[:fieldTransport10TCPLen], append(make([]uint64, 3), ns[fieldTransport10TCPLen:]...)...)
625669
}
626670

627671
return &NFSTransportStats{
672+
// NFS xprt over tcp or udp
628673
Protocol: protocol,
629674
Port: ns[0],
630675
Bind: ns[1],
@@ -636,8 +681,32 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
636681
BadTransactionIDs: ns[7],
637682
CumulativeActiveRequests: ns[8],
638683
CumulativeBacklog: ns[9],
639-
MaximumRPCSlotsUsed: ns[10],
640-
CumulativeSendingQueue: ns[11],
641-
CumulativePendingQueue: ns[12],
684+
685+
// NFS xprt over tcp or udp
686+
// And statVersion 1.1
687+
MaximumRPCSlotsUsed: ns[10],
688+
CumulativeSendingQueue: ns[11],
689+
CumulativePendingQueue: ns[12],
690+
691+
// NFS xprt over rdma
692+
// And stat Version 1.1
693+
ReadChunkCount: ns[13],
694+
WriteChunkCount: ns[14],
695+
ReplyChunkCount: ns[15],
696+
TotalRdmaRequest: ns[16],
697+
PullupCopyCount: ns[17],
698+
HardwayRegisterCount: ns[18],
699+
FailedMarshalCount: ns[19],
700+
BadReplyCount: ns[20],
701+
MrsRecovered: ns[21],
702+
MrsOrphaned: ns[22],
703+
MrsAllocated: ns[23],
704+
EmptySendctxQ: ns[24],
705+
TotalRdmaReply: ns[25],
706+
FixupCopyCount: ns[26],
707+
ReplyWaitsForSend: ns[27],
708+
LocalInvNeeded: ns[28],
709+
NomsgCallCount: ns[29],
710+
BcallCount: ns[30],
642711
}, nil
643712
}

‎mountstats_test.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,102 @@ func TestMountStats(t *testing.T) {
372372
},
373373
},
374374
},
375+
{
376+
name: "NFS xprt over rdma proto",
377+
s: `device <nfsserver>:<nfsmount> mounted on <mountpoint> with fstype nfs statvers=1.1
378+
opts: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=120,acregmax=120,acdirmin=120,acdirmax=120,hard,nocto,forcerdirplus,proto=rdma,nconnect=16,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=172.16.40.20,mountvers=3,mountport=0,mountproto=tcp,local_lock=none
379+
age: 1270876
380+
caps: caps=0xf,wtmult=4096,dtsize=131072,bsize=0,namlen=255
381+
sec: flavor=1,pseudoflavor=1
382+
events: 512052 36601115 0 68 1498583 16514 38815015 0 41584 2654459933 0 0 0 0 1527715 0 0 1498575 0 0 0 0 0 0 0 0 0
383+
bytes: 3104202770327296 0 0 0 2013200952170479 0 491504202537 0
384+
RPC iostats version: 1.1 p/v: 100003/3 (nfs)
385+
xprt: rdma 0 0 5808 62 0 494490723 494490687 36 10032963746 1282789 107150285 1226637531 2673889 135120843409861 135119397156505 266368832 75716996 0 7853 0 0 0 0 119328 1336431717 0 96
386+
per-op statistics
387+
NULL: 16 16 0 640 384 320 11 331 0
388+
`,
389+
mounts: []*Mount{{
390+
Device: "<nfsserver>:<nfsmount>",
391+
Mount: "<mountpoint>",
392+
Type: "nfs",
393+
Stats: &MountStatsNFS{
394+
StatVersion: "1.1",
395+
Opts: map[string]string{"acdirmax": "120", "acdirmin": "120", "acregmax": "120",
396+
"acregmin": "120", "forcerdirplus": "", "hard": "", "local_lock": "none",
397+
"mountaddr": "172.16.40.20", "mountport": "0", "mountproto": "tcp", "mountvers": "3",
398+
"namlen": "255", "nconnect": "16", "nocto": "", "port": "20049", "proto": "rdma",
399+
"retrans": "2", "ro": "", "rsize": "1048576", "sec": "sys", "timeo": "600",
400+
"vers": "3", "wsize": "1048576"},
401+
Age: 1270876 * time.Second,
402+
Bytes: NFSBytesStats{
403+
Read: 3104202770327296,
404+
ReadTotal: 2013200952170479,
405+
ReadPages: 491504202537,
406+
},
407+
Events: NFSEventsStats{
408+
InodeRevalidate: 512052,
409+
DnodeRevalidate: 36601115,
410+
AttributeInvalidate: 68,
411+
VFSOpen: 1498583,
412+
VFSLookup: 16514,
413+
VFSAccess: 38815015,
414+
VFSReadPage: 41584,
415+
VFSReadPages: 2654459933,
416+
VFSFlush: 1527715,
417+
VFSFileRelease: 1498575,
418+
},
419+
Operations: []NFSOperationStats{
420+
{
421+
Operation: "NULL",
422+
Requests: 16,
423+
Transmissions: 16,
424+
MajorTimeouts: 0,
425+
BytesSent: 640,
426+
BytesReceived: 384,
427+
CumulativeQueueMilliseconds: 320,
428+
CumulativeTotalResponseMilliseconds: 11,
429+
CumulativeTotalRequestMilliseconds: 331,
430+
AverageRTTMilliseconds: 0.6875,
431+
Errors: 0,
432+
},
433+
},
434+
Transport: NFSTransportStats{
435+
Protocol: "rdma",
436+
Port: 0,
437+
Bind: 0,
438+
Connect: 5808,
439+
ConnectIdleTime: 62,
440+
IdleTimeSeconds: 0,
441+
Sends: 494490723,
442+
Receives: 494490687,
443+
BadTransactionIDs: 36,
444+
CumulativeActiveRequests: 10032963746,
445+
CumulativeBacklog: 1282789,
446+
MaximumRPCSlotsUsed: 0,
447+
CumulativeSendingQueue: 0,
448+
CumulativePendingQueue: 0,
449+
ReadChunkCount: 107150285,
450+
WriteChunkCount: 1226637531,
451+
ReplyChunkCount: 2673889,
452+
TotalRdmaRequest: 135120843409861,
453+
PullupCopyCount: 135119397156505,
454+
HardwayRegisterCount: 266368832,
455+
FailedMarshalCount: 75716996,
456+
BadReplyCount: 0,
457+
MrsRecovered: 7853,
458+
MrsOrphaned: 0,
459+
MrsAllocated: 0,
460+
EmptySendctxQ: 0,
461+
TotalRdmaReply: 0,
462+
FixupCopyCount: 119328,
463+
ReplyWaitsForSend: 1336431717,
464+
LocalInvNeeded: 0,
465+
NomsgCallCount: 96,
466+
BcallCount: 0,
467+
},
468+
},
469+
}},
470+
},
375471
}
376472

377473
for i, tt := range tests {
@@ -518,4 +614,61 @@ device fs.example.com:/volume4/apps/home-automation/node-red-data mounted on /va
518614
LOOKUPP: 0 0 0 0 0 0 0 0 0
519615
LAYOUTERROR: 0 0 0 0 0 0 0 0 0
520616
`
617+
618+
extendedRDMAExampleALLMountstats = `device <nfsserver>:<nfsmount> mounted on <mountpoint> with fstype nfs statvers=1.1
619+
opts: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=120,acregmax=120,acdirmin=120,acdirmax=120,hard,nocto,forcerdirplus,proto=rdma,nconnect=16,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=172.16.40.20,mountvers=3,mountport=0,mountproto=tcp,local_lock=none
620+
age: 1270876
621+
caps: caps=0xf,wtmult=4096,dtsize=131072,bsize=0,namlen=255
622+
sec: flavor=1,pseudoflavor=1
623+
events: 512052 36601115 0 68 1498583 16514 38815015 0 41584 2654459933 0 0 0 0 1527715 0 0 1498575 0 0 0 0 0 0 0 0 0
624+
bytes: 3104202770327296 0 0 0 2013200952170479 0 491504202537 0
625+
RPC iostats version: 1.1 p/v: 100003/3 (nfs)
626+
xprt: rdma 0 0 5808 62 0 494490723 494490687 36 10032963746 1282789 107150285 1226637531 2673889 135120843409861 135119397156505 266368832 75716996 0 7853 0 0 0 0 119328 1336431717 0 96
627+
xprt: rdma 0 0 14094 145 0 492392334 492392307 27 7078693624 2509627 105561370 1280878332 2659446 142218924010291 142217463504063 276368040 94761838 0 7610 0 0 0 0 207977 1389069860 0 103
628+
xprt: rdma 0 0 16107 156 0 522755125 522755092 33 9119562599 1147699 109077860 1491898147 2566003 167152062826463 167149287506014 284931680 83011025 0 6229 0 0 0 0 221408 1603518232 0 82
629+
xprt: rdma 0 0 7808 82 0 441542046 441542010 36 7226132207 2519174 111096004 955223347 2676765 105741904708009 105740125663595 275613584 80373159 0 8893 0 0 0 0 149479 1068962768 0 76
630+
xprt: rdma 0 0 15018 167 0 508091827 508091764 63 19817677255 36702583 108265928 1258185459 2438516 138247436686102 138246196289594 270162080 74962306 0 13328 0 0 0 0 268433 1368837472 0 66
631+
xprt: rdma 0 0 14321 149 0 530246310 530246275 35 9723190432 2392024 111099700 1494204555 2589805 166691166581904 166689567426908 289995492 85067377 0 8010 0 0 0 0 214511 1607864447 0 100
632+
xprt: rdma 0 0 7863 84 0 459019689 459019642 47 11809253102 1716688 111825219 1032758664 2564226 114416685286438 114414936423706 290494252 73702102 0 6927 0 0 0 0 134453 1147121864 0 79
633+
xprt: rdma 0 0 7702 84 3 497598986 497598931 55 11816221496 3924722 106922130 1382063307 2506108 153967067193941 153965665472218 286222584 84094006 0 5875 0 0 0 0 127347 1491469045 0 66
634+
xprt: rdma 0 0 18341 202 0 477721151 477721073 78 15204400959 40562626 106645745 1291616653 3091375 144533696686651 144529688231163 278135800 73821525 0 6795 0 0 0 0 251097 1401327563 0 64
635+
xprt: rdma 0 0 8228 90 4 453155092 453155063 29 7884786894 1591225 112197590 1026006338 2742688 114591819605673 114590175821191 275541944 85857259 0 7487 0 0 0 0 143044 1140917892 0 76
636+
xprt: rdma 0 0 7843 83 0 446480377 446480324 53 12267986428 2958997 111971246 963162784 2693433 107176282309753 107174637802555 290269096 101100410 0 7825 0 0 0 0 141735 1077797328 0 83
637+
xprt: rdma 0 0 7582 86 0 423315608 423315567 41 10197484604 2076993 109207538 785978455 2650354 86090211449474 86088475571312 279912524 87676008 0 7491 0 0 0 0 137533 897807641 0 101
638+
xprt: rdma 0 0 7767 84 0 482538465 482538424 41 8935200479 1344778 112200583 1192341640 2644896 132860698423762 132858881459050 273354060 75337030 0 5941 0 0 0 0 127842 1307164736 0 97
639+
xprt: rdma 0 0 14526 148 2 537745063 537745007 56 20756072620 3970332320 109539564 1363647371 2503250 148793734936250 148791264145401 291888720 90344151 0 7471 0 0 0 0 211057 1475661285 0 82
640+
xprt: rdma 0 0 14300 151 0 495357347 495357316 31 8703101643 1451809 112315311 1303804607 2620502 145680743007170 145678880292235 288046696 98018259 0 7241 0 0 0 0 209396 1418712657 0 139
641+
xprt: rdma 0 0 7700 82 0 466611083 466611050 33 8540498291 4082864 114740300 1059770596 2523155 117376668239921 117375375683167 260927576 78437075 0 6691 0 0 0 0 130878 1177008175 1 76
642+
per-op statistics
643+
NULL: 16 16 0 640 384 320 11 331 0
644+
GETATTR: 512052 512052 0 79823516 57349824 107131 612667 751847 0
645+
SETATTR: 0 0 0 0 0 0 0 0 0
646+
LOOKUP: 16713 16713 0 3040536 3706344 560 17488 20232 346
647+
ACCESS: 211705 211705 0 33860920 25404600 37059 229754 283822 0
648+
READLINK: 0 0 0 0 0 0 0 0 0
649+
READ: 2654501510 2654501510 0 445911966900 2013540728551504 6347457114 31407021389 37927280438 0
650+
WRITE: 0 0 0 0 0 0 0 0 0
651+
CREATE: 0 0 0 0 0 0 0 0 0
652+
MKDIR: 0 0 0 0 0 0 0 0 0
653+
SYMLINK: 0 0 0 0 0 0 0 0 0
654+
MKNOD: 0 0 0 0 0 0 0 0 0
655+
REMOVE: 0 0 0 0 0 0 0 0 0
656+
RMDIR: 0 0 0 0 0 0 0 0 0
657+
RENAME: 0 0 0 0 0 0 0 0 0
658+
LINK: 0 0 0 0 0 0 0 0 0
659+
READDIR: 0 0 0 0 0 0 0 0 0
660+
READDIRPLUS: 0 0 0 0 0 0 0 0 0
661+
FSSTAT: 56356 56356 0 6243572 9467808 82068 74356 159001 0
662+
FSINFO: 2 2 0 184 328 0 0 0 0
663+
PATHCONF: 1 1 0 92 140 0 0 0 0
664+
COMMIT: 0 0 0 0 0 0 0 0 0
665+
`
521666
)
667+
668+
func TestMountStatsExtendedRDMAStats(t *testing.T) {
669+
r := strings.NewReader(extendedRDMAExampleALLMountstats)
670+
_, err := parseMountStats(r)
671+
if err != nil {
672+
t.Errorf("failed to parse mount stats with extended RDMA statistics: %v", err)
673+
}
674+
}

0 commit comments

Comments
 (0)