From 04a22381b82164588519055b6b85aec1821c0e6a Mon Sep 17 00:00:00 2001 From: Zichao LI Date: Wed, 28 Apr 2021 23:32:16 +0800 Subject: [PATCH] 1st commit --- .gitignore | 1 + extractor.go | 45 ++++++++++++++++ go.mod | 9 ++++ go.sum | 110 +++++++++++++++++++++++++++++++++++++++ ignore.go | 26 ++++++++++ log-writter/file.go | 62 ++++++++++++++++++++++ log-writter/redis.go | 24 +++++++++ logger.go | 121 +++++++++++++++++++++++++++++++++++++++++++ util.go | 37 +++++++++++++ 9 files changed, 435 insertions(+) create mode 100644 extractor.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 ignore.go create mode 100644 log-writter/file.go create mode 100644 log-writter/redis.go create mode 100644 logger.go create mode 100644 util.go diff --git a/.gitignore b/.gitignore index 66fd13c..d1baad6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ # Dependency directories (remove the comment below to include it) # vendor/ +.idea \ No newline at end of file diff --git a/extractor.go b/extractor.go new file mode 100644 index 0000000..16f0bab --- /dev/null +++ b/extractor.go @@ -0,0 +1,45 @@ +package traffic_logger + +import ( + "net" + "net/http" + "strings" +) + +type FieldsExtractor interface { + Host(r *http.Request) string + ClientIP(r *http.Request) string + APIName(r *http.Request) string + Operator(r *http.Request) string +} + +type DefaultExtractor struct{} + +func (DefaultExtractor) Host(r *http.Request) string { + if host := r.Header.Get("X-Forwarded-Host"); len(host) > 0 { + return host + } + return r.Host +} + +func (DefaultExtractor) ClientIP(r *http.Request) string { + if clientIP := strings.TrimSpace(r.Header.Get("X-Real-Ip")); clientIP != "" { + return clientIP + } + clientIP := r.Header.Get("X-Forwarded-For") + if clientIP = strings.TrimSpace(strings.Split(clientIP, ",")[0]); clientIP != "" { + return clientIP + } + if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil { + return ip + } + return "" +} + +func (DefaultExtractor) APIName(r *http.Request) string { + return r.Header.Get("X-Api-Name") +} + +func (DefaultExtractor) Operator(r *http.Request) string { + return r.Header.Get("X-Forwarded-User-Name") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b95ac83 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/haxii/traffic-logger + +go 1.15 + +require ( + github.com/go-redis/redis/v8 v8.8.2 + github.com/rs/zerolog v1.21.0 + github.com/valyala/bytebufferpool v1.0.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4383a4f --- /dev/null +++ b/go.sum @@ -0,0 +1,110 @@ +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis/v8 v8.8.2 h1:O/NcHqobw7SEptA0yA6up6spZVFtwE06SXM8rgLtsP8= +github.com/go-redis/redis/v8 v8.8.2/go.mod h1:F7resOH5Kdug49Otu24RjHWwgK7u9AmtqWMnCV1iP5Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.21.0 h1:Q3vdXlfLNT+OftyBHsU0Y445MD+8m8axjKgf2si0QcM= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/otel v0.19.0 h1:Lenfy7QHRXPZVsw/12CWpxX6d/JkrX8wrx2vO8G80Ng= +go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= +go.opentelemetry.io/otel/metric v0.19.0 h1:dtZ1Ju44gkJkYvo+3qGqVXmf88tc+a42edOywypengg= +go.opentelemetry.io/otel/metric v0.19.0/go.mod h1:8f9fglJPRnXuskQmKpnad31lcLJ2VmNNqIsx/uIwBSc= +go.opentelemetry.io/otel/oteltest v0.19.0 h1:YVfA0ByROYqTwOxqHVZYZExzEpfZor+MU1rU+ip2v9Q= +go.opentelemetry.io/otel/oteltest v0.19.0/go.mod h1:tI4yxwh8U21v7JD6R3BcA/2+RBoTKFexE/PJ/nSO7IA= +go.opentelemetry.io/otel/trace v0.19.0 h1:1ucYlenXIDA1OlHVLDZKX0ObXV5RLaq06DtUKz5e5zc= +go.opentelemetry.io/otel/trace v0.19.0/go.mod h1:4IXiNextNOpPnRlI4ryK69mn5iC84bjBWZQA5DXz/qg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ignore.go b/ignore.go new file mode 100644 index 0000000..a79e672 --- /dev/null +++ b/ignore.go @@ -0,0 +1,26 @@ +package traffic_logger + +type Ignore interface { + Req(apiName string) bool + Resp(apiName string) bool +} + +type DefaultIgnore struct{} + +func (DefaultIgnore) Req(apiName string) bool { + return false +} + +func (DefaultIgnore) Resp(apiName string) bool { + return false +} + +type IgnoreAll struct{} + +func (IgnoreAll) Req(apiName string) bool { + return true +} + +func (IgnoreAll) Resp(apiName string) bool { + return true +} diff --git a/log-writter/file.go b/log-writter/file.go new file mode 100644 index 0000000..70afa8f --- /dev/null +++ b/log-writter/file.go @@ -0,0 +1,62 @@ +package log_writter + +import ( + "fmt" + "os" + "path/filepath" + "time" +) + +// NewFileWriter make a log writer to save traffic into file +func NewFileWriter(logDir, name string) (*os.File, error) { + timeNOW := func() string { + return time.Now().Format("2006-01-02-15.04.05.999999999") + } + + logFileName := name + ".log" + logFilePath := filepath.Join(logDir, logFileName) + + if fileExists(logFilePath) { + _ = os.Rename(logFilePath, filepath.Join(logDir, name+"."+timeNOW()+".log")) + } + return makefile(logDir, logFileName) +} + +// log file helper +func makefile(dir string, filename string) (f *os.File, e error) { + if err := createDirectories(dir); err != nil { + return nil, err + } + filePath := filepath.Join(dir, filename) + fileWriter, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return nil, err + } + return fileWriter, nil +} + +func createDirectories(dir string) error { + if fi, err := os.Stat(dir); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("%v (checking directory)", err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("%v (creating directory)", err) + } + } else if !fi.IsDir() { + return fmt.Errorf("destination path is not directory") + } + return nil +} + +func fileExists(path string) bool { + if len(path) == 0 { + return false + } + if fi, err := os.Stat(path); os.IsNotExist(err) { + return false + } else if fi.IsDir() { + return false + } + return true +} diff --git a/log-writter/redis.go b/log-writter/redis.go new file mode 100644 index 0000000..9f76211 --- /dev/null +++ b/log-writter/redis.go @@ -0,0 +1,24 @@ +package log_writter + +import ( + "context" + "github.com/go-redis/redis/v8" +) + +type RedisListWriter struct { + rdb *redis.Client + key string +} + +// NewRedisListWriter make a redis writer to push every traffic into a redis list +func NewRedisListWriter(rdbLogKey string, rdb *redis.Client) *RedisListWriter { + return &RedisListWriter{rdb: rdb, key: rdbLogKey} +} + +func (r *RedisListWriter) Write(p []byte) (n int, err error) { + _, err = r.rdb.LPush(context.Background(), r.key, p).Result() + if err != nil { + return 0, err + } + return len(p), nil +} diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..b7f6d10 --- /dev/null +++ b/logger.go @@ -0,0 +1,121 @@ +package traffic_logger + +import ( + "bytes" + "encoding/json" + "github.com/rs/zerolog" + "github.com/valyala/bytebufferpool" + "io/ioutil" + "net/http" + "time" +) + +type Options struct { + Logger *zerolog.Logger + Extractor FieldsExtractor + Ignore Ignore +} + +type TrafficLogger struct { + logger *zerolog.Logger + extractor FieldsExtractor + ignore Ignore +} + +func New(options *Options) *TrafficLogger { + extractor := options.Extractor + if extractor == nil { + extractor = &DefaultExtractor{} + } + ignore := options.Ignore + if ignore == nil { + ignore = &DefaultIgnore{} + } + return &TrafficLogger{logger: options.Logger, extractor: extractor, ignore: ignore} +} + +func (l *TrafficLogger) Handler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // we don't track traffic without a api name + apiName := l.extractor.APIName(r) + if len(apiName) == 0 { + next.ServeHTTP(w, r) + return + } + + // request time + reqStartTime := time.Now() + + // get request body + var reqBuffer *bytebufferpool.ByteBuffer + if !l.ignore.Req(apiName) { + reqBuffer = bytebufferpool.Get() + defer bytebufferpool.Put(reqBuffer) + n, err := reqBuffer.ReadFrom(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _ = r.Body.Close() + if n > 0 { + r.Body = ioutil.NopCloser(bytes.NewReader(reqBuffer.B)) + } else { + r.Body = &nullReadCloser{} + } + } + + // get resp + var respBuffer *bytebufferpool.ByteBuffer + if !l.ignore.Resp(apiName) { + respBuffer = bytebufferpool.Get() + defer bytebufferpool.Put(respBuffer) + } + nw := &recordableResponseWriter{ResponseWriter: w, buffer: respBuffer} + + // wrap + next.ServeHTTP(nw, r) + + // request log + logRequestEvent := zerolog.Dict(). + Str("method", r.Method). + Str("host", l.extractor.Host(r)). + Str("path", r.URL.Path). + Str("query", r.URL.RawQuery) + if reqBuffer != nil { + logRequestEvent = logBodyEvent(logRequestEvent, reqBuffer.B) + } + + // response log + logResponseEvent := zerolog.Dict().Int("status", nw.status) + if respBuffer != nil { + logResponseEvent = logBodyEvent(logResponseEvent, respBuffer.B) + } + + // full log + l.logger.Log(). + Int64("timestamp", reqStartTime.Unix()). + Str("api_name", apiName). + Str("ip", l.extractor.ClientIP(r)). + Str("operator", l.extractor.Operator(r)). + Dur("latency", time.Now().Sub(reqStartTime)). + Dict("request", logRequestEvent). + Dict("response", logResponseEvent). + Send() + }) +} + +func logBodyEvent(e *zerolog.Event, b []byte) *zerolog.Event { + if len(b) == 0 { + return e + } + if json.Valid(b) { + // remove carriage returns + for i, _b := range b { + if _b == '\n' || _b == '\r' { + b[i] = ' ' + } + } + return e.RawJSON("body", b) + } + return e.Bytes("body", b) +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..75a0b6a --- /dev/null +++ b/util.go @@ -0,0 +1,37 @@ +package traffic_logger + +import ( + "github.com/valyala/bytebufferpool" + "io" + "net/http" +) + +// nullReadCloser 假的 request body +type nullReadCloser struct{} + +func (r *nullReadCloser) Read([]byte) (int, error) { + return 0, io.EOF +} + +func (r *nullReadCloser) Close() error { + return nil +} + +// recordableResponseWriter 支持 response 记录的 response writer +type recordableResponseWriter struct { + http.ResponseWriter + buffer *bytebufferpool.ByteBuffer + status int +} + +func (r *recordableResponseWriter) WriteHeader(statusCode int) { + r.status = statusCode + r.ResponseWriter.WriteHeader(statusCode) +} + +func (r *recordableResponseWriter) Write(p []byte) (int, error) { + if r.buffer != nil { + r.buffer.Write(p) + } + return r.ResponseWriter.Write(p) +}