diff --git a/Makefile b/Makefile index e683417..58a322a 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,11 @@ test-with-js: clean test-with-self: clean go run . go run . curl https://www.example.com +# Test HAR output + +test-with-har: + go run . --dump-har out.har -- curl -Lso /dev/null https://monasticacademy.org + # These tests are currently broken broken-test-with-nonroot-user: clean diff --git a/README.md b/README.md index bf99d2c..0472750 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,21 @@ buddhismforaisutraco� ... ``` -Here the `--head` option tells httptap to print the HTTP headers, and `--body` tells it to print the raw HTTP payloads. To keep it short I'm showing just the first request/response pair above. +Here the `--head` option tells httptap to print the HTTP headers, and `--body` tells it to print the raw HTTP payloads. To keep it short I'm showing just the first request/response pair. + +# HAR output + +You can dump the HTTP requests and responses to a HAR file like this: + +``` +$ httptap --dump-har out.har -- curl -Lso /dev/null https://monasticacademy.org +``` + +There are many HAR viewers out there that can visualize this dump file. For example here is how the above looks in the Google HAR Analyzer](https://toolbox.googleapps.com/apps/har_analyzer/): + +![HAR Analyzer Screenshot](docs/har-screenshot.png) + +Again, what you're looking at here is one HTTP request to https://monasticacademy.org that returns a 308 Redirect, followed by a second HTTP request to https://www.monasticacademy.org that return a 200 OK. # How it works diff --git a/context.go b/context.go new file mode 100644 index 0000000..d0a9818 --- /dev/null +++ b/context.go @@ -0,0 +1,9 @@ +package main + +// the type for context keys created by this package +type contextKey string + +// a value for this context key is set on all HTTP requests intercepted by httptap, and is used +// in the DialContext function associated with http transports to dial the same hostname that +// the subprocess was dialing, regardless of what hostname is in HTTP request. +var dialToContextKey contextKey = "httptap.dialTo" diff --git a/docs/har-screenshot.png b/docs/har-screenshot.png new file mode 100644 index 0000000..fbdaeb4 Binary files /dev/null and b/docs/har-screenshot.png differ diff --git a/go.mod b/go.mod index f01ddb6..e944287 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/fatih/color v1.17.0 github.com/google/gopacket v1.1.19 github.com/joemiller/certin v0.3.5 + golang.org/x/lint v0.0.0-20200302205851-738671d3881b + golang.org/x/tools v0.22.0 software.sslmate.com/src/go-pkcs12 v0.5.0 ) @@ -19,7 +21,6 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/tools v0.22.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 33fc2bb..95324d0 100644 --- a/go.sum +++ b/go.sum @@ -53,18 +53,15 @@ github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1Y github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -76,8 +73,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/http.go b/http.go index 92d71db..9ccd5d4 100644 --- a/http.go +++ b/http.go @@ -13,7 +13,6 @@ import ( "net/http" "strings" "sync" - "time" "github.com/joemiller/certin" ) @@ -140,9 +139,8 @@ func ipFromAddr(addr net.Addr) net.IP { } } -// listen for incomming connections on l and proxy each one to the outside world, while sending -// information about the request/response pairs to all HTTP listeners -func proxyHTTPS(conn net.Conn, root *certin.KeyAndCert) { +// service an incoming HTTPS connection on conn by sending a request out to the world through dst. +func proxyHTTPS(dst http.RoundTripper, conn net.Conn, root *certin.KeyAndCert) { defer handlePanic() defer conn.Close() @@ -174,12 +172,12 @@ func proxyHTTPS(conn net.Conn, root *certin.KeyAndCert) { verbosef("reading request sent to %v (%v) ...", conn.LocalAddr(), serverName) - proxyHTTP(tlsconn) + proxyHTTP(dst, tlsconn) } -// listen for incomming connections on l and proxy each one to the outside world, while sending -// information about the request/response pairs to all HTTP listeners -func proxyHTTP(conn net.Conn) { +// Service an incoming HTTP connection on conn by sending a request out to the world through dst. +// All HTTP requests sent to dst will have a context containing a value for the key dialToContextKey. +func proxyHTTP(dst http.RoundTripper, conn net.Conn) { defer handlePanic() defer conn.Close() @@ -199,8 +197,7 @@ func proxyHTTP(conn net.Conn) { } defer req.Body.Close() - // the request may contain a relative URL but we need an absolute URL for RoundTrip to know - // where to dial + // the request may contain a relative URL but we need an absolute URL for call to RoundTrip if req.URL.Host == "" { req.URL.Host = req.Host if req.URL.Host == "" { @@ -211,37 +208,23 @@ func proxyHTTP(conn net.Conn) { req.URL.Scheme = "https" } - // create a RoundTripper that always dials the IP we intercepted packets to + // add the IP to which we intercepted packets as a context variable dialTo := req.URL.Host if !strings.Contains(dialTo, ":") { dialTo += ":https" } - - // these parameters copied from http.DefaultTransport - roundTripper := http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - if network != "tcp" { - return nil, fmt.Errorf("network %q was requested of dialer pinned to tcp (%v)", network, dialTo) - } - verbosef("pinned dialer ignoring %q and dialing %v", address, dialTo) - return net.Dial("tcp", dialTo) - }, - ForceAttemptHTTP2: true, - MaxIdleConns: 5, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } + req = req.WithContext(context.WithValue(req.Context(), dialToContextKey, dialTo)) // capture the request body into memory for inspection later var reqbody bytes.Buffer req.Body = TeeReadCloser(req.Body, &reqbody) + // it seems that harlog assumes that request.GetBody will be non-nil whenever request.Body is non-nil + req.GetBody = func() (io.ReadCloser, error) { return req.Body, nil } + // do roundtrip to the actual server in the world -- we use RoundTrip here because // we do not want to follow redirects or accumulate our own cookies - resp, err := roundTripper.RoundTrip(req) + resp, err := dst.RoundTrip(req) if err != nil { // error here means the server hostname could not be resolved, or a TCP connection could not be made, // or TLS could not be negotiated, or something like that diff --git a/httptap.go b/httptap.go index 1f1ed18..1b384b2 100644 --- a/httptap.go +++ b/httptap.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -16,6 +17,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/alexflint/go-arg" "github.com/fatih/color" @@ -24,6 +26,7 @@ import ( "github.com/joemiller/certin" "github.com/mdlayher/packet" "github.com/monasticacademy/httptap/pkg/certfile" + "github.com/monasticacademy/httptap/pkg/harlog" "github.com/monasticacademy/httptap/pkg/opensslpaths" "github.com/monasticacademy/httptap/pkg/overlay" "github.com/songgao/water" @@ -160,17 +163,18 @@ func Main() error { var args struct { Verbose bool `arg:"-v,--verbose,env:HTTPTAP_VERBOSE"` NoNewUserNamespace bool `arg:"--no-new-user-namespace,env:HTTPTAP_NO_NEW_USER_NAMESPACE" help:"do not create a new user namespace (must be run as root)"` - Stderr bool `arg:"env:HTTPTAP_LOG_TO_STDERR" help:"log to stderr (default is stdout)"` - Tun string `default:"httptap" help:"name of the network device to create"` + Stderr bool `arg:"env:HTTPTAP_LOG_TO_STDERR" help:"log to standard error (default is standard out)"` + Tun string `default:"httptap" help:"name of the TUN device that will be created"` Subnet string `default:"10.1.1.100/24" help:"IP address of the network interface that the subprocess will see"` Gateway string `default:"10.1.1.1" help:"IP address of the gateway that intercepts and proxies network packets"` WebUI string `arg:"env:HTTPTAP_WEB_UI" help:"address and port to serve API on"` User string `help:"run command as this user (username or id)"` NoOverlay bool `arg:"--no-overlay,env:HTTPTAP_NO_OVERLAY" help:"do not mount any overlay filesystems"` - Stack string `arg:"env:HTTPTAP_STACK" default:"gvisor" help:"'gvisor' or 'homegrown'"` - Dump bool `arg:"env:HTTPTAP_DUMP" help:"dump all packets sent and received"` - HTTPPorts []int `arg:"--http"` - HTTPSPorts []int `arg:"--https"` + Stack string `arg:"env:HTTPTAP_STACK" default:"gvisor" help:"which tcp implementation to use: 'gvisor' or 'homegrown'"` + DumpTCP bool `arg:"--dump-tcp,env:HTTPTAP_DUMP_TCP" help:"dump all TCP packets sent and received to standard out"` + DumpHAR string `arg:"--dump-har,env:HTTPTAP_DUMP_HAR" help:"path to dump HAR capture to"` + HTTPPorts []int `arg:"--http" help:"list of TCP ports to intercept HTTPS traffic on"` + HTTPSPorts []int `arg:"--https" help:"list of TCP ports to intercept HTTP traffic on"` Head bool `help:"whether to include HTTP headers in terminal output"` Body bool `help:"whether to include HTTP payloads in terminal output"` Command []string `arg:"positional"` @@ -193,15 +197,16 @@ func Main() error { verbosef("re-execing in a new user namespace...") // Here we move to a new user namespace, which is an unpriveleged operation, and which - // allows us to do everything else we need to do in unpriveleged mode. + // allows us to do everything else without being root. // // In a C program, we could run unshare(CLONE_NEWUSER) and directly be in a new user // namespace. In a Go program that is not possible because all Go programs are multithreaded // (even with GOMAXPROCS=1), and unshare(CLONE_NEWUSER) is only available to single-threaded // programs. // - // Our best option is then to launch ourselves in a subprocess that is in a new user namespace, - // using /proc/self/exe, which contains the executable code for the current process. + // Our best option is to launch ourselves in a subprocess that is in a new user namespace, + // using /proc/self/exe, which contains the executable code for the current process. This + // is the same approach taken by docker's reexec package. cmd := exec.Command("/proc/self/exe") cmd.Args = append([]string{"/proc/self/exe"}, os.Args[1:]...) @@ -354,7 +359,7 @@ func Main() error { } // if --dump was provided then start watching everything - if args.Dump { + if args.DumpTCP { iface, err := net.InterfaceByName(args.Tun) if err != nil { return err @@ -587,38 +592,72 @@ func Main() error { handleDNS(context.Background(), w, p.payload) }) - // set up a test TCP interceptor - mux.HandleTCP(":11223", func(conn net.Conn) { - fmt.Fprint(conn, "hello 11223\n") - conn.Close() - }) - - // set up a test UDP interceptor - mux.HandleUDP(":11223", func(w udpResponder, p *udpPacket) { - verbosef("got udp packet: %q, replying", string(p.payload)) - _, err = w.Write([]byte("hello udp 11223!\n")) + // create the transport that will proxy intercepted connections out to the world + var roundTripper http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + if network != "tcp" { + return nil, fmt.Errorf("network %q was requested of dialer pinned to tcp", network) + } + dialTo, ok := ctx.Value(dialToContextKey).(string) + if !ok { + return nil, fmt.Errorf("context on proxied request was missing dialTo key") + } + verbosef("pinned dialer ignoring %q and dialing %v", address, dialTo) + return net.Dial("tcp", dialTo) + }, + ForceAttemptHTTP2: true, + MaxIdleConns: 5, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + // set up middlewares for HAR file logging if requested + if args.DumpHAR != "" { + // open the file right away so that filesystem errors get surfaced as soon as possible + f, err := os.Create(args.DumpHAR) if err != nil { - errorf("error writing udp packet back to sender: %v", err) - return + log.Printf("error opening HAR file for writing: %w", err) + } + defer f.Close() + + // add the HAR middleware + harlogger := harlog.Transport{ + Transport: roundTripper, + UnusualError: func(err error) error { + verbosef("error in HAR log capture: %v, ignoring", err) + return nil + }, } - }) - // TODO: proxy all other UDP packets to the public internet - // go proxyUDP(udppstack.Listen("*")) + roundTripper = &harlogger - // intercept all TCP connections on port 443 and treat as HTTPS + // write the HAR log at program termination + defer func() { + err := json.NewEncoder(f).Encode(harlogger.HAR()) + if err != nil { + verbosef("error serializing HAR output: %v, ignoring", err) + } + }() + } + + // intercept TCP connections on requested HTTP ports and treat as HTTP for _, port := range args.HTTPPorts { - mux.HandleTCP(fmt.Sprintf(":%d", port), proxyHTTP) + mux.HandleTCP(fmt.Sprintf(":%d", port), func(conn net.Conn) { + proxyHTTP(roundTripper, conn) + }) } - // intercept all TCP connections on port 443 and treat as HTTPS + // intercept TCP connections on requested HTTPS ports and treat as HTTPS for _, port := range args.HTTPSPorts { mux.HandleTCP(fmt.Sprintf(":%d", port), func(conn net.Conn) { - proxyHTTPS(conn, ca) + proxyHTTPS(roundTripper, conn, ca) }) } - // listen for TCP connections and proxy each one to the world + // listen for other TCP connections and proxy to the world mux.HandleTCP("*", func(conn net.Conn) { proxyTCP(conn) }) diff --git a/pkg/harlog/.gitignore b/pkg/harlog/.gitignore new file mode 100644 index 0000000..d586eea --- /dev/null +++ b/pkg/harlog/.gitignore @@ -0,0 +1,3 @@ +.idea/ + +bin/ diff --git a/pkg/harlog/LICENSE b/pkg/harlog/LICENSE new file mode 100644 index 0000000..bf22b2b --- /dev/null +++ b/pkg/harlog/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Masahiro Wakame + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pkg/harlog/README.md b/pkg/harlog/README.md new file mode 100644 index 0000000..deee179 --- /dev/null +++ b/pkg/harlog/README.md @@ -0,0 +1,46 @@ +Forked from https://github.com/vvakame/go-harlog + +net/http client logging by HAR format. + +Take http request/response log by HAR (HTTP Archive) format. +It can visualize by [any](https://developers.google.com/web/updates/2017/08/devtools-release-notes#har-imports) [tools](https://toolbox.googleapps.com/apps/har_analyzer/). + +## How to use + +```shell script +$ go get github.com/vvakame/go-harlog +``` + +```go +har := &harlog.Transport{} +hc := &http.Client{ + Transport: har, +} + +// do something... + +b, err := json.MarshalIndent(har.HAR(), "", " ") +if err != nil { + return err +} +fmt.Println(string(b)) +``` + +See HAR file in Google Chrome DevTools. + +This screenshots are generated by this library. +Capture the log about Google Cloud Storage access by [cloud.google.com/go/storage](https://godoc.org/cloud.google.com/go/storage). + +![Headers](https://user-images.githubusercontent.com/125332/65957276-60f68f80-e487-11e9-86e4-dfcc5fe64f44.png) + +![Response](https://user-images.githubusercontent.com/125332/65957297-6c49bb00-e487-11e9-8762-2c176cacc83a.png) + +![Timing](https://user-images.githubusercontent.com/125332/65957313-779ce680-e487-11e9-9c00-f03c257b2ef3.png) + + +## Limitations + +* compressed response is not supported yet. +* `headersSize` is not calculated. + +patches welcome! diff --git a/pkg/harlog/client_tracer.go b/pkg/harlog/client_tracer.go new file mode 100644 index 0000000..88c0f79 --- /dev/null +++ b/pkg/harlog/client_tracer.go @@ -0,0 +1,80 @@ +package harlog + +import ( + "crypto/tls" + "net/http/httptrace" + "time" +) + +// NewTimingTrace creates a timing trace together with an http client tracer that populates it. +// The third parameter is a function to call when the request is complete. +func NewTimingTrace() (*TimingTrace, *httptrace.ClientTrace) { + timings := &TimingTrace{ + startAt: time.Now(), + } + tracer := &httptrace.ClientTrace{ + GetConn: timings.GetConn, + GotConn: timings.GotConn, + PutIdleConn: nil, + GotFirstResponseByte: timings.GotFirstResponseByte, + Got100Continue: nil, + Got1xxResponse: nil, + DNSStart: timings.DNSStart, + DNSDone: timings.DNSDone, + ConnectStart: nil, + ConnectDone: nil, + TLSHandshakeStart: timings.TLSHandshakeStart, + TLSHandshakeDone: timings.TLSHandshakeDone, + WroteHeaderField: nil, + WroteHeaders: nil, + Wait100Continue: nil, + WroteRequest: timings.WroteRequest, + } + + return timings, tracer +} + +type TimingTrace struct { + startAt time.Time + connStart time.Time + connObtained time.Time + firstResponseByte time.Time + dnsStart time.Time + dnsEnd time.Time + tlsHandshakeStart time.Time + tlsHandshakeEnd time.Time + writeRequest time.Time + endAt time.Time +} + +func (ct *TimingTrace) GetConn(hostPort string) { + ct.connStart = time.Now() +} + +func (ct *TimingTrace) GotConn(info httptrace.GotConnInfo) { + ct.connObtained = time.Now() +} + +func (ct *TimingTrace) GotFirstResponseByte() { + ct.firstResponseByte = time.Now() +} + +func (ct *TimingTrace) DNSStart(info httptrace.DNSStartInfo) { + ct.dnsStart = time.Now() +} + +func (ct *TimingTrace) DNSDone(info httptrace.DNSDoneInfo) { + ct.dnsEnd = time.Now() +} + +func (ct *TimingTrace) TLSHandshakeStart() { + ct.tlsHandshakeStart = time.Now() +} + +func (ct *TimingTrace) TLSHandshakeDone(tls.ConnectionState, error) { + ct.tlsHandshakeEnd = time.Now() +} + +func (ct *TimingTrace) WroteRequest(info httptrace.WroteRequestInfo) { + ct.writeRequest = time.Now() +} diff --git a/pkg/harlog/round_tripper.go b/pkg/harlog/round_tripper.go new file mode 100644 index 0000000..ec4b130 --- /dev/null +++ b/pkg/harlog/round_tripper.go @@ -0,0 +1,145 @@ +package harlog + +import ( + "bytes" + "io" + "log" + "net/http" + "net/http/httptrace" + "sync" + "time" +) + +var _ http.RoundTripper = (*Transport)(nil) + +// Transport is collecting http request/response log by HAR format. +type Transport struct { + // next Transport. if nil, use http.DefaultTransport. + Transport http.RoundTripper + // unusual (not network oriented) error occurred, handle error by this function. + // if nil, emit error log by log package, and ignore it. + UnusualError func(err error) error + + har *HARContainer + mutex sync.Mutex +} + +func (h *Transport) init() { + if h.har != nil { + return + } + + h.mutex.Lock() + defer h.mutex.Unlock() + if h.har != nil { + return + } + + h.har = &HARContainer{ + Log: &Log{ + Version: "1.2", + Creator: &Creator{ + Name: "github.com/vvakame/go-harlog", + Version: "0.0.1", + }, + }, + } +} + +// HAR returns HAR format log data. +func (h *Transport) HAR() *HARContainer { + h.init() + return h.har +} + +// RoundTrip executes a single HTTP transaction, returning +// a Response for the provided Request. +func (h *Transport) RoundTrip(r *http.Request) (*http.Response, error) { + h.init() + + baseRoundTripper := h.Transport + if baseRoundTripper == nil { + baseRoundTripper = http.DefaultTransport + } + + entry := &Entry{} + defer func() { + h.mutex.Lock() + h.har.Log.Entries = append(h.har.Log.Entries, entry) + h.mutex.Unlock() + }() + + err := h.preRoundTrip(r, entry) + if err != nil { + if h.UnusualError != nil { + err = h.UnusualError(err) + } else { + log.Println(err) + err = nil + } + if err != nil { + return nil, err + } + } + + // create a tracer to record timestamps of certain events internal to the HTTP stack + timings, tracer := NewTimingTrace() + r = r.WithContext(httptrace.WithClientTrace(r.Context(), tracer)) + + // do the HTTP roundtrip + resp, realErr := baseRoundTripper.RoundTrip(r) + + err = h.postRoundTrip(resp, entry) + + timings.endAt = time.Now() + UpdateEntryWithTimings(entry, timings) + + if err != nil { + if h.UnusualError != nil { + err = h.UnusualError(err) + } else { + log.Println(err) + err = nil + } + if err != nil { + return nil, err + } + } + + entry.Cache = &Cache{} + + return resp, realErr +} + +func (h *Transport) preRoundTrip(r *http.Request, entry *Entry) error { + var err error + reqBody := r.Body + if r.GetBody != nil { + reqBody, err = r.GetBody() + if err != nil { + return err + } + } + + var body []byte + if reqBody != nil { + body, err = io.ReadAll(reqBody) + if err != nil { + return err + } + } + + return UpdateEntryWithRequest(entry, r, body) +} + +func (h *Transport) postRoundTrip(resp *http.Response, entry *Entry) error { + respBodyBytes, err := io.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return err + } + resp.Body = io.NopCloser(bytes.NewBuffer(respBodyBytes)) + + UpdateEntryWithResponse(entry, resp, respBodyBytes) + return nil +} diff --git a/pkg/harlog/tools.go b/pkg/harlog/tools.go new file mode 100644 index 0000000..94cd063 --- /dev/null +++ b/pkg/harlog/tools.go @@ -0,0 +1,10 @@ +// +build tools + +package harlog + +// from https://github.com/golang/go/issues/25922#issuecomment-412992431 + +import ( + _ "golang.org/x/lint/golint" + _ "golang.org/x/tools/cmd/goimports" +) diff --git a/pkg/harlog/types.go b/pkg/harlog/types.go new file mode 100644 index 0000000..cf1c0cf --- /dev/null +++ b/pkg/harlog/types.go @@ -0,0 +1,335 @@ +package harlog + +import ( + "encoding/json" + "time" +) + +// copied from https://github.com/CyrusBiotechnology/go-har + +// from https://w3c.github.io/web-performance/specs/HAR/Overview.html + +var _ json.Marshaler = Time{} +var _ json.Unmarshaler = (*Time)(nil) +var _ json.Marshaler = Duration(0) +var _ json.Unmarshaler = (*Duration)(nil) + +// Time provides ISO 8601 format JSON data. +type Time time.Time + +// MarshalJSON to ISO 8601 format from time.Time. +func (t Time) MarshalJSON() ([]byte, error) { + if time.Time(t).IsZero() { + return []byte(`null`), nil + } + + v := time.Time(t).Format(time.RFC3339) + return json.Marshal(v) +} + +// UnmarshalJSON from ISO 8601 format to time.Time. +func (t *Time) UnmarshalJSON(data []byte) error { + if string(data) == "null" { + return nil + } + + v, err := time.Parse(`"`+time.RFC3339+`"`, string(data)) + if err != nil { + return err + } + vt := Time(v) + *t = vt + return nil +} + +// Duration provides milliseconds order JSON format. +type Duration time.Duration + +// MarshalJSON to milliseconds order number format from time.Duration. +func (d Duration) MarshalJSON() ([]byte, error) { + v := float64(d) / float64(time.Millisecond) + return json.Marshal(v) +} + +// UnmarshalJSON from milliseconds order number format to time.Duration. +func (d *Duration) UnmarshalJSON(data []byte) error { + if string(data) == "null" { + return nil + } + + var v float64 + err := json.Unmarshal(data, &v) + if err != nil { + return err + } + + *d = Duration(v * float64(time.Millisecond)) + + return nil +} + +// HARContainer is ... +// The HAR format is based on JSON, as described in RFC 4627. +type HARContainer struct { + Log *Log `json:"log"` +} + +// Log is... +// This object represents the root of the exported data. This object MUST be present and its name MUST be "log". The object contains the following name/value pairs: +type Log struct { + // Required. Version number of the format. + Version string `json:"version"` + // Required. An object of type creator that contains the name and version information of the log creator application. + Creator *Creator `json:"creator"` + // Optional. An object of type browser that contains the name and version information of the user agent. + Browser *Browser `json:"browser,omitempty"` + // Optional. An array of objects of type page, each representing one exported (tracked) page. Leave out this field if the application does not support grouping by pages. + Pages []*Page `json:"pages,omitempty"` + // Required. An array of objects of type entry, each representing one exported (tracked) HTTP request. + Entries []*Entry `json:"entries"` + // Optional. A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Creator is ... +// This object contains information about the log creator application and contains the following name/value pairs: +type Creator struct { + // Required. The name of the application that created the log. + Name string `json:"name"` + // Required. The version number of the application that created the log. + Version string `json:"version"` + // Optional. A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Browser is ... +// This object contains information about the browser that created the log and contains the following name/value pairs: +type Browser struct { + // Required. The name of the browser that created the log. + Name string `json:"name"` + // Required. The version number of the browser that created the log. + Version string `json:"version"` + // Optional. A comment provided by the user or the browser. + Comment string `json:"comment,omitempty"` +} + +// Page is ... +// This object represents list of exported pages. +type Page struct { + // Date and time stamp for the beginning of the page load (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00). + StartedDateTime Time `json:"startedDateTime"` + // Unique identifier of a page within the . Entries use it to refer the parent page. + ID string `json:"id"` + // Page title. + Title string `json:"title"` + // Detailed timing info about page load. + PageTiming *PageTiming `json:"pageTimings,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// PageTiming is ... +// This object describes timings for various events (states) fired during the page load. All times are specified in milliseconds. If a time info is not available appropriate field is set to -1. +type PageTiming struct { + // Content of the page loaded. Number of milliseconds since page load started (page.startedDateTime). Use -1 if the timing does not apply to the current request. + OnContentLoad Duration `json:"onContentLoad,omitempty"` + // Page is loaded (onLoad event fired). Number of milliseconds since page load started (page.startedDateTime). Use -1 if the timing does not apply to the current request. + OnLoad Duration `json:"onLoad,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Entry is ... +// This object represents an array with all exported HTTP requests. Sorting entries by startedDateTime (starting from the oldest) is preferred way how to export data since it can make importing faster. However the reader application should always make sure the array is sorted (if required for the import). +type Entry struct { + // Reference to the parent page. Leave out this field if the application does not support grouping by pages. + Pageref string `json:"pageref,omitempty"` + // Date and time stamp of the request start (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD). + StartedDateTime Time `json:"startedDateTime"` + // Total elapsed time of the request in milliseconds. This is the sum of all timings available in the timings object (i.e. not including -1 values) . + Time Duration `json:"time"` + // Detailed info about the request. + Request *Request `json:"request"` + // Detailed info about the response. + Response *Response `json:"response"` + // Info about cache usage. + Cache *Cache `json:"cache"` + // Detailed timing info about request/response round trip. + Timings *Timings `json:"timings"` + // IP address of the server that was connected (result of DNS resolution). + ServerIPAddress string `json:"serverIPAddress,omitempty"` + // Unique ID of the parent TCP/IP connection, can be the client port number. Note that a port number doesn't have to be unique identifier in cases where the port is shared for more connections. If the port isn't available for the application, any other unique connection ID can be used instead (e.g. connection index). Leave out this field if the application doesn't support this info. + Connection string `json:"connection,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Request is ... +// This object contains detailed info about performed request. +type Request struct { + // Request method (GET, POST, ...). + Method string `json:"method"` + // Absolute URL of the request (fragments are not included). + URL string `json:"url"` + // Request HTTP Version. + HTTPVersion string `json:"httpVersion"` + // List of cookie objects. + Cookies []*Cookie `json:"cookies"` + // List of header objects. + Headers []*NVP `json:"headers"` + // List of query parameter objects. + QueryString []*NVP `json:"queryString"` + // Posted data info. + PostData *PostData `json:"postData,omitempty"` + // Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body. Set to -1 if the info is not available. + HeadersSize int `json:"headersSize"` + // Size of the request body (POST data payload) in bytes. Set to -1 if the info is not available. + BodySize int `json:"bodySize"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Response is ... +// This object contains detailed info about the response. +type Response struct { + // Response status. + Status int `json:"status"` + // Response status description. + StatusText string `json:"statusText"` + // Response HTTP Version. + HTTPVersion string `json:"httpVersion"` + // List of cookie objects. + Cookies []*Cookie `json:"cookies"` + // List of header objects. + Headers []*NVP `json:"headers"` + // Details about the response body. + Content *Content `json:"content"` + // Redirection target URL from the Location response header. + RedirectURL string `json:"redirectURL"` + // Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body. Set to -1 if the info is not available. + HeadersSize int `json:"headersSize"` + // Size of the received response body in bytes. Set to zero in case of responses coming from the cache (304). Set to -1 if the info is not available. + BodySize int64 `json:"bodySize"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Cookie is ... +// This object contains list of all cookies (used in and objects). +type Cookie struct { + // The name of the cookie. + Name string `json:"name"` + // The cookie value. + Value string `json:"value"` + // The path pertaining to the cookie. + Path string `json:"path,omitempty"` + // The host of the cookie. + Domain string `json:"domain,omitempty"` + // Cookie expiration time. (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00). + Expires Time `json:"expires,omitempty"` + // Set to true if the cookie is HTTP only, false otherwise. + HTTPOnly bool `json:"httpOnly,omitempty"` + // True if the cookie was transmitted over ssl, false otherwise. + Secure bool `json:"secure,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// NVP is name-value pairs. +type NVP struct { + Name string `json:"name"` + Value string `json:"value"` + Comment string `json:"comment,omitempty"` +} + +// PostData is ... +// This object describes posted data, if any (embedded in object). +type PostData struct { + // Mime type of posted data. + MimeType string `json:"mimeType"` + // List of posted parameters (in case of URL encoded parameters). + Params []*Param `json:"params"` + // Plain text posted data + Text string `json:"text"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Param is ... +// List of posted parameters, if any (embedded in object). +type Param struct { + // name of a posted parameter. + Name string `json:"name"` + // value of a posted parameter or content of a posted file. + Value string `json:"value,omitempty"` + // name of a posted file. + FileName string `json:"fileName,omitempty"` + // content type of a posted file. + ContentType string `json:"contentType,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Content is ... +// This object describes details about response content (embedded in object). +type Content struct { + // Length of the returned content in bytes. Should be equal to response.bodySize if there is no compression and bigger when the content has been compressed. + Size int64 `json:"size"` + // Number of bytes saved. Leave out this field if the information is not available. + Compression int64 `json:"compression,omitempty"` + // MIME type of the response text (value of the Content-Type response header). The charset attribute of the MIME type is included (if available). + MimeType string `json:"mimeType"` + // Response body sent from the server or loaded from the browser cache. This field is populated with textual content only. The text field is either HTTP decoded text or a encoded (e.g. "base64") representation of the response body. Leave out this field if the information is not available. + Text string `json:"text,omitempty"` + // Encoding used for response text field e.g "base64". Leave out this field if the text field is HTTP decoded (decompressed & unchunked), than trans-coded from its original character set into UTF-8. + Encoding string `json:"encoding,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Cache is ... +// This objects contains info about a request coming from browser cache. +type Cache struct { + // State of a cache entry before the request. Leave out this field if the information is not available. + BeforeRequest *CacheInfo `json:"beforeRequest,omitempty"` + // State of a cache entry after the request. Leave out this field if the information is not available. + AfterRequest *CacheInfo `json:"afterRequest,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// CacheInfo is ... +// Both beforeRequest and afterRequest object share the following structure. +type CacheInfo struct { + // Expiration time of the cache entry. + Expires string `json:"expires,omitempty"` + // The last time the cache entry was opened. + LastAccess string `json:"lastAccess"` + // Etag + ETag string `json:"etag"` + // The number of times the cache entry has been opened. + HitCount int `json:"hitCount"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Timings is ... +// This object describes various phases within request-response round trip. All times are specified in milliseconds. +type Timings struct { + // Time spent in a queue waiting for a network connection. Use -1 if the timing does not apply to the current request. + Blocked Duration `json:"blocked,omitempty"` + // DNS resolution time. The time required to resolve a host name. Use -1 if the timing does not apply to the current request. + DNS Duration `json:"dns,omitempty"` + // Time required to create TCP connection. Use -1 if the timing does not apply to the current request. + Connect Duration `json:"connect,omitempty"` + // Time required to send HTTP request to the server. + Send Duration `json:"send"` + // Waiting for a response from the server. + Wait Duration `json:"wait"` + // Time required to read entire response from the server (or cache). + Receive Duration `json:"receive"` + // Time required for SSL/TLS negotiation. If this field is defined then the time is also included in the connect field (to ensure backward compatibility with HAR 1.1). Use -1 if the timing does not apply to the current request. + SSL Duration `json:"ssl,omitempty"` + // A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} diff --git a/pkg/harlog/types_test.go b/pkg/harlog/types_test.go new file mode 100644 index 0000000..c8e9761 --- /dev/null +++ b/pkg/harlog/types_test.go @@ -0,0 +1,163 @@ +package harlog + +import ( + "reflect" + "testing" + "time" +) + +func TestTime_MarshalJSON(t *testing.T) { + + tz, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + t Time + want string + wantErr bool + }{ + { + name: "plain", + t: Time(time.Date(2019, 10, 2, 12, 16, 30, 50, tz)), + want: `"2019-10-02T12:16:30+09:00"`, + wantErr: false, + }, + { + name: "zero value", + t: Time(time.Time{}), + want: `null`, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.t.MarshalJSON() + if (err != nil) != tt.wantErr { + t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("MarshalJSON() got = %v, want %v", string(got), tt.want) + } + }) + } +} + +func TestTime_UnmarshalJSON(t *testing.T) { + + // NOTE float64 - int64 の変換が生じるのでnsecレベルで誤差がでるのはしょうがない + + tz, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Fatal(err) + } + + type args struct { + data string + } + tests := []struct { + name string + t args + want Time + wantErr bool + }{ + { + name: "plain", + t: args{ + data: `"2019-10-02T12:16:31+09:00"`, + }, + want: Time(time.Date(2019, 10, 2, 12, 16, 31, 0, tz)), + wantErr: false, + }, + { + name: "null", + t: args{ + data: `null`, + }, + want: Time(time.Time{}), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v Time + if err := v.UnmarshalJSON([]byte(tt.t.data)); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if !time.Time(v).Equal(time.Time(tt.want)) { + t.Errorf("UnmarshalJSON() got = %v, want %v", v, tt.want) + } + }) + } +} + +func TestDuration_MarshalJSON(t *testing.T) { + tests := []struct { + name string + d Duration + want string + wantErr bool + }{ + { + name: "plain", + d: Duration(10 * time.Millisecond), + want: "10", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.d.MarshalJSON() + if (err != nil) != tt.wantErr { + t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("MarshalJSON() got = %v, want %v", string(got), tt.want) + } + }) + } +} + +func TestDuration_UnmarshalJSON(t *testing.T) { + type args struct { + data string + } + tests := []struct { + name string + args args + want Duration + wantErr bool + }{ + { + name: "plain", + args: args{ + data: "10", + }, + want: Duration(10 * time.Millisecond), + wantErr: false, + }, + { + name: "null", + args: args{ + data: `null`, + }, + want: Duration(0), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v Duration + if err := v.UnmarshalJSON([]byte(tt.args.data)); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if v != tt.want { + t.Errorf("UnmarshalJSON() got = %v, want %v", v, tt.want) + } + }) + } +} diff --git a/pkg/harlog/update.go b/pkg/harlog/update.go new file mode 100644 index 0000000..f83fd32 --- /dev/null +++ b/pkg/harlog/update.go @@ -0,0 +1,184 @@ +package harlog + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime" + "mime/multipart" + "net/http" + "net/url" + "strings" +) + +// UpdateEntryWithTimings populates a HAR entry with timings from an HTTP round trip +func UpdateEntryWithTimings(entry *Entry, trace *TimingTrace) { + entry.StartedDateTime = Time(trace.startAt) + entry.Time = Duration(trace.endAt.Sub(trace.startAt)) + entry.Timings = &Timings{ + Blocked: Duration(trace.startAt.Sub(trace.connStart)), + DNS: -1, + Connect: -1, + Send: Duration(trace.writeRequest.Sub(trace.connObtained)), + Wait: Duration(trace.firstResponseByte.Sub(trace.writeRequest)), + Receive: Duration(trace.endAt.Sub(trace.firstResponseByte)), + SSL: -1, + } + if !trace.dnsStart.IsZero() { + entry.Timings.DNS = Duration(trace.dnsEnd.Sub(trace.dnsStart)) + } + if !trace.connStart.IsZero() { + entry.Timings.Connect = Duration(trace.connObtained.Sub(trace.connStart)) + } + if !trace.tlsHandshakeStart.IsZero() { + entry.Timings.SSL = Duration(trace.tlsHandshakeEnd.Sub(trace.tlsHandshakeStart)) + } +} + +// UpdateEntryWithRequest populates a HAR entry with values from an HTTP request. It treats the provided +// body as the body of the HTTP request and does not read or modify r.Body. +func UpdateEntryWithRequest(entry *Entry, r *http.Request, body []byte) error { + bodySize := -1 + var postData *PostData + + if body != nil { + bodySize = len(body) + + mimeType := r.Header.Get("Content-Type") + postData = &PostData{ + MimeType: mimeType, + Params: []*Param{}, + Text: string(body), + } + + // ignore missing or malformed mime type here + mediaType, mediaParams, _ := mime.ParseMediaType(mimeType) + + switch mediaType { + case "application/x-www-form-urlencoded": + formdata, err := url.ParseQuery(string(body)) + if err == nil { + for k, v := range formdata { + for _, s := range v { + postData.Params = append(postData.Params, &Param{ + Name: k, + Value: s, + }) + } + } + } + + case "multipart/form-data": // consider allowing "multipart/mixed" here too + boundary, ok := mediaParams["boundary"] + if !ok { + return fmt.Errorf("got a multipart/form-data request with no boundary in the media type") + } + + mr := multipart.NewReader(bytes.NewReader(body), boundary) + formdata, err := mr.ReadForm(10 * 1024 * 1024) + if err == nil { + for k, v := range formdata.Value { + for _, s := range v { + postData.Params = append(postData.Params, &Param{ + Name: k, + Value: s, + }) + } + } + for k, v := range formdata.File { + for _, s := range v { + postData.Params = append(postData.Params, &Param{ + Name: k, + FileName: s.Filename, + ContentType: s.Header.Get("Content-Type"), + }) + } + } + } + } + } + + entry.Request = &Request{ + Method: r.Method, + URL: r.URL.String(), + HTTPVersion: r.Proto, + Cookies: toHARCookies(r.Cookies()), + Headers: toHARNVP(r.Header), + QueryString: toHARNVP(r.URL.Query()), + PostData: postData, + HeadersSize: -1, // TODO + BodySize: bodySize, + } + + return nil +} + +// UpdateEntryWithResponse populates a HAR entry with data from an HTTP response. It treats the provided +// body bytes as the content of the response. It does not read from or modify resp.Body. +func UpdateEntryWithResponse(entry *Entry, resp *http.Response, body []byte) { + mimeType := resp.Header.Get("Content-Type") + + // parse the mime type, and ignore parse errors + mediaType, _, _ := mime.ParseMediaType(mimeType) + + var text string + var encoding string + switch { + case strings.HasPrefix(mediaType, "text/"): + text = string(body) + default: + text = base64.StdEncoding.EncodeToString(body) + encoding = "base64" + } + + entry.Response = &Response{ + Status: resp.StatusCode, + StatusText: "", + HTTPVersion: resp.Proto, + Cookies: toHARCookies(resp.Cookies()), + Headers: toHARNVP(resp.Header), + Content: &Content{ + Size: resp.ContentLength, // TODO 圧縮されている場合のフォロー + Compression: 0, + MimeType: mimeType, + Text: text, + Encoding: encoding, + }, + RedirectURL: resp.Header.Get("Location"), + HeadersSize: -1, + BodySize: resp.ContentLength, + } +} + +func toHARCookies(cookies []*http.Cookie) []*Cookie { + harCookies := make([]*Cookie, 0, len(cookies)) + + for _, cookie := range cookies { + harCookies = append(harCookies, &Cookie{ + Name: cookie.Name, + Value: cookie.Value, + Path: cookie.Path, + Domain: cookie.Domain, + Expires: Time(cookie.Expires), + HTTPOnly: cookie.HttpOnly, + Secure: cookie.Secure, + }) + } + + return harCookies +} + +func toHARNVP(vs map[string][]string) []*NVP { + nvps := make([]*NVP, 0, len(vs)) + + for k, v := range vs { + for _, s := range v { + nvps = append(nvps, &NVP{ + Name: k, + Value: s, + }) + } + } + + return nvps +}