add --dump-har to write HAR files as output

Makefile

@@ -83,6 +83,11 @@ test-with-js: clean
 test-with-self: clean
 	go run . go run . curl https://www.example.com
 
+# Test HAR output
+
+test-with-har:
+	go run . --dump-har out.har -- curl -Lso /dev/null https://monasticacademy.org
+
 # These tests are currently broken
 
 broken-test-with-nonroot-user: clean

README.md

@@ -149,7 +149,21 @@ buddhismforaisutraco�
 ...
 ```
 
-Here the `--head` option tells httptap to print the HTTP headers, and `--body` tells it to print the raw HTTP payloads. To keep it short I'm showing just the first request/response pair above.
+Here the `--head` option tells httptap to print the HTTP headers, and `--body` tells it to print the raw HTTP payloads. To keep it short I'm showing just the first request/response pair.
+
+# HAR output
+
+You can dump the HTTP requests and responses to a HAR file like this:
+
+```
+$ httptap --dump-har out.har -- curl -Lso /dev/null https://monasticacademy.org
+```
+
+There are many HAR viewers out there that can visualize this dump file. For example here is how the above looks in the Google HAR Analyzer](https://toolbox.googleapps.com/apps/har_analyzer/):
+
+![HAR Analyzer Screenshot](docs/har-screenshot.png)
+
+Again, what you're looking at here is one HTTP request to https://monasticacademy.org that returns a 308 Redirect, followed by a second HTTP request to https://www.monasticacademy.org that return a 200 OK.
 
 # How it works
 

context.go

@@ -0,0 +1,9 @@
+package main
+
+// the type for context keys created by this package
+type contextKey string
+
+// a value for this context key is set on all HTTP requests intercepted by httptap, and is used
+// in the DialContext function associated with http transports to dial the same hostname that
+// the subprocess was dialing, regardless of what hostname is in HTTP request.
+var dialToContextKey contextKey = "httptap.dialTo"

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/google/gopacket v1.1.19
 	github.com/joemiller/certin v0.3.5
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
+	golang.org/x/tools v0.22.0
 	software.sslmate.com/src/go-pkcs12 v0.5.0
 )
 
@@ -19,7 +21,6 @@ require (
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/tools v0.22.0 // indirect
 )
 
 require (

go.sum

@@ -53,18 +53,15 @@ github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1Y
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
 golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
 golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
 golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -76,8 +73,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
 golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

http.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/joemiller/certin"
 )
@@ -140,9 +139,8 @@ func ipFromAddr(addr net.Addr) net.IP {
 	}
 }
 
-// listen for incomming connections on l and proxy each one to the outside world, while sending
-// information about the request/response pairs to all HTTP listeners
-func proxyHTTPS(conn net.Conn, root *certin.KeyAndCert) {
+// service an incoming HTTPS connection on conn by sending a request out to the world through dst.
+func proxyHTTPS(dst http.RoundTripper, conn net.Conn, root *certin.KeyAndCert) {
 	defer handlePanic()
 	defer conn.Close()
 
@@ -174,12 +172,12 @@ func proxyHTTPS(conn net.Conn, root *certin.KeyAndCert) {
 
 	verbosef("reading request sent to %v (%v) ...", conn.LocalAddr(), serverName)
 
-	proxyHTTP(tlsconn)
+	proxyHTTP(dst, tlsconn)
 }
 
-// listen for incomming connections on l and proxy each one to the outside world, while sending
-// information about the request/response pairs to all HTTP listeners
-func proxyHTTP(conn net.Conn) {
+// Service an incoming HTTP connection on conn by sending a request out to the world through dst.
+// All HTTP requests sent to dst will have a context containing a value for the key dialToContextKey.
+func proxyHTTP(dst http.RoundTripper, conn net.Conn) {
 	defer handlePanic()
 	defer conn.Close()
 
@@ -199,8 +197,7 @@ func proxyHTTP(conn net.Conn) {
 	}
 	defer req.Body.Close()
 
-	// the request may contain a relative URL but we need an absolute URL for RoundTrip to know
-	// where to dial
+	// the request may contain a relative URL but we need an absolute URL for call to RoundTrip
 	if req.URL.Host == "" {
 		req.URL.Host = req.Host
 		if req.URL.Host == "" {
@@ -211,37 +208,23 @@ func proxyHTTP(conn net.Conn) {
 		req.URL.Scheme = "https"
 	}
 
-	// create a RoundTripper that always dials the IP we intercepted packets to
+	// add the IP to which we intercepted packets as a context variable
 	dialTo := req.URL.Host
 	if !strings.Contains(dialTo, ":") {
 		dialTo += ":https"
 	}
-
-	// these parameters copied from http.DefaultTransport
-	roundTripper := http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
-			if network != "tcp" {
-				return nil, fmt.Errorf("network %q was requested of dialer pinned to tcp (%v)", network, dialTo)
-			}
-			verbosef("pinned dialer ignoring %q and dialing %v", address, dialTo)
-			return net.Dial("tcp", dialTo)
-		},
-		ForceAttemptHTTP2:     true,
-		MaxIdleConns:          5,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
-	}
+	req = req.WithContext(context.WithValue(req.Context(), dialToContextKey, dialTo))
 
 	// capture the request body into memory for inspection later
 	var reqbody bytes.Buffer
 	req.Body = TeeReadCloser(req.Body, &reqbody)
 
+	// it seems that harlog assumes that request.GetBody will be non-nil whenever request.Body is non-nil
+	req.GetBody = func() (io.ReadCloser, error) { return req.Body, nil }
+
 	// do roundtrip to the actual server in the world -- we use RoundTrip here because
 	// we do not want to follow redirects or accumulate our own cookies
-	resp, err := roundTripper.RoundTrip(req)
+	resp, err := dst.RoundTrip(req)
 	if err != nil {
 		// error here means the server hostname could not be resolved, or a TCP connection could not be made,
 		// or TLS could not be negotiated, or something like that

httptap.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -16,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/alexflint/go-arg"
 	"github.com/fatih/color"
@@ -24,6 +26,7 @@ import (
 	"github.com/joemiller/certin"
 	"github.com/mdlayher/packet"
 	"github.com/monasticacademy/httptap/pkg/certfile"
+	"github.com/monasticacademy/httptap/pkg/harlog"
 	"github.com/monasticacademy/httptap/pkg/opensslpaths"
 	"github.com/monasticacademy/httptap/pkg/overlay"
 	"github.com/songgao/water"
@@ -160,17 +163,18 @@ func Main() error {
 	var args struct {
 		Verbose            bool     `arg:"-v,--verbose,env:HTTPTAP_VERBOSE"`
 		NoNewUserNamespace bool     `arg:"--no-new-user-namespace,env:HTTPTAP_NO_NEW_USER_NAMESPACE" help:"do not create a new user namespace (must be run as root)"`
-		Stderr             bool     `arg:"env:HTTPTAP_LOG_TO_STDERR" help:"log to stderr (default is stdout)"`
-		Tun                string   `default:"httptap" help:"name of the network device to create"`
+		Stderr             bool     `arg:"env:HTTPTAP_LOG_TO_STDERR" help:"log to standard error (default is standard out)"`
+		Tun                string   `default:"httptap" help:"name of the TUN device that will be created"`
 		Subnet             string   `default:"10.1.1.100/24" help:"IP address of the network interface that the subprocess will see"`
 		Gateway            string   `default:"10.1.1.1" help:"IP address of the gateway that intercepts and proxies network packets"`
 		WebUI              string   `arg:"env:HTTPTAP_WEB_UI" help:"address and port to serve API on"`
 		User               string   `help:"run command as this user (username or id)"`
 		NoOverlay          bool     `arg:"--no-overlay,env:HTTPTAP_NO_OVERLAY" help:"do not mount any overlay filesystems"`
-		Stack              string   `arg:"env:HTTPTAP_STACK" default:"gvisor" help:"'gvisor' or 'homegrown'"`
-		Dump               bool     `arg:"env:HTTPTAP_DUMP" help:"dump all packets sent and received"`
-		HTTPPorts          []int    `arg:"--http"`
-		HTTPSPorts         []int    `arg:"--https"`
+		Stack              string   `arg:"env:HTTPTAP_STACK" default:"gvisor" help:"which tcp implementation to use: 'gvisor' or 'homegrown'"`
+		DumpTCP            bool     `arg:"--dump-tcp,env:HTTPTAP_DUMP_TCP" help:"dump all TCP packets sent and received to standard out"`
+		DumpHAR            string   `arg:"--dump-har,env:HTTPTAP_DUMP_HAR" help:"path to dump HAR capture to"`
+		HTTPPorts          []int    `arg:"--http" help:"list of TCP ports to intercept HTTPS traffic on"`
+		HTTPSPorts         []int    `arg:"--https" help:"list of TCP ports to intercept HTTP traffic on"`
 		Head               bool     `help:"whether to include HTTP headers in terminal output"`
 		Body               bool     `help:"whether to include HTTP payloads in terminal output"`
 		Command            []string `arg:"positional"`
@@ -193,15 +197,16 @@ func Main() error {
 		verbosef("re-execing in a new user namespace...")
 
 		// Here we move to a new user namespace, which is an unpriveleged operation, and which
-		// allows us to do everything else we need to do in unpriveleged mode.
+		// allows us to do everything else without being root.
 		//
 		// In a C program, we could run unshare(CLONE_NEWUSER) and directly be in a new user
 		// namespace. In a Go program that is not possible because all Go programs are multithreaded
 		// (even with GOMAXPROCS=1), and unshare(CLONE_NEWUSER) is only available to single-threaded
 		// programs.
 		//
-		// Our best option is then to launch ourselves in a subprocess that is in a new user namespace,
-		// using /proc/self/exe, which contains the executable code for the current process.
+		// Our best option is to launch ourselves in a subprocess that is in a new user namespace,
+		// using /proc/self/exe, which contains the executable code for the current process. This
+		// is the same approach taken by docker's reexec package.
 
 		cmd := exec.Command("/proc/self/exe")
 		cmd.Args = append([]string{"/proc/self/exe"}, os.Args[1:]...)
@@ -354,7 +359,7 @@ func Main() error {
 	}
 
 	// if --dump was provided then start watching everything
-	if args.Dump {
+	if args.DumpTCP {
 		iface, err := net.InterfaceByName(args.Tun)
 		if err != nil {
 			return err
@@ -587,38 +592,72 @@ func Main() error {
 		handleDNS(context.Background(), w, p.payload)
 	})
 
-	// set up a test TCP interceptor
-	mux.HandleTCP(":11223", func(conn net.Conn) {
-		fmt.Fprint(conn, "hello 11223\n")
-		conn.Close()
-	})
-
-	// set up a test UDP interceptor
-	mux.HandleUDP(":11223", func(w udpResponder, p *udpPacket) {
-		verbosef("got udp packet: %q, replying", string(p.payload))
-		_, err = w.Write([]byte("hello udp 11223!\n"))
+	// create the transport that will proxy intercepted connections out to the world
+	var roundTripper http.RoundTripper = &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, fmt.Errorf("network %q was requested of dialer pinned to tcp", network)
+			}
+			dialTo, ok := ctx.Value(dialToContextKey).(string)
+			if !ok {
+				return nil, fmt.Errorf("context on proxied request was missing dialTo key")
+			}
+			verbosef("pinned dialer ignoring %q and dialing %v", address, dialTo)
+			return net.Dial("tcp", dialTo)
+		},
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          5,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+	}
+
+	// set up middlewares for HAR file logging if requested
+	if args.DumpHAR != "" {
+		// open the file right away so that filesystem errors get surfaced as soon as possible
+		f, err := os.Create(args.DumpHAR)
 		if err != nil {
-			errorf("error writing udp packet back to sender: %v", err)
-			return
+			log.Printf("error opening HAR file for writing: %w", err)
+		}
+		defer f.Close()
+
+		// add the HAR middleware
+		harlogger := harlog.Transport{
+			Transport: roundTripper,
+			UnusualError: func(err error) error {
+				verbosef("error in HAR log capture: %v, ignoring", err)
+				return nil
+			},
 		}
-	})
 
-	// TODO: proxy all other UDP packets to the public internet
-	// go proxyUDP(udppstack.Listen("*"))
+		roundTripper = &harlogger
 
-	// intercept all TCP connections on port 443 and treat as HTTPS
+		// write the HAR log at program termination
+		defer func() {
+			err := json.NewEncoder(f).Encode(harlogger.HAR())
+			if err != nil {
+				verbosef("error serializing HAR output: %v, ignoring", err)
+			}
+		}()
+	}
+
+	// intercept TCP connections on requested HTTP ports and treat as HTTP
 	for _, port := range args.HTTPPorts {
-		mux.HandleTCP(fmt.Sprintf(":%d", port), proxyHTTP)
+		mux.HandleTCP(fmt.Sprintf(":%d", port), func(conn net.Conn) {
+			proxyHTTP(roundTripper, conn)
+		})
 	}
 
-	// intercept all TCP connections on port 443 and treat as HTTPS
+	// intercept TCP connections on requested HTTPS ports and treat as HTTPS
 	for _, port := range args.HTTPSPorts {
 		mux.HandleTCP(fmt.Sprintf(":%d", port), func(conn net.Conn) {
-			proxyHTTPS(conn, ca)
+			proxyHTTPS(roundTripper, conn, ca)
 		})
 	}
 
-	// listen for TCP connections and proxy each one to the world
+	// listen for other TCP connections and proxy to the world
 	mux.HandleTCP("*", func(conn net.Conn) {
 		proxyTCP(conn)
 	})

pkg/harlog/.gitignore

@@ -0,0 +1,3 @@
+.idea/
+
+bin/

pkg/harlog/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Masahiro Wakame
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.