support upload callback(#104)

Author: imroc

README.md

@@ -781,6 +781,29 @@ client.R().SetFileReader("avatar", "avatar.png", avatarImgFile).Post(url)
 */
 ```
 
+**Upload Callback**
+
+You can set `UploadCallback` if you want to show upload progress:
+
+```go
+client := req.C()
+client.R().
+	SetFile("excel", "test.xlsx").
+	SetUploadCallback(func(info req.UploadInfo) {
+        fmt.Printf("%q uploaded %.2f%%\n", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0)
+    }).Post("https://exmaple.com/upload")
+/* Output
+"test.xlsx" uploaded 7.44%
+"test.xlsx" uploaded 29.78%
+"test.xlsx" uploaded 52.08%
+"test.xlsx" uploaded 74.47%
+"test.xlsx" uploaded 96.87%
+"test.xlsx" uploaded 100.00%
+*/
+```
+
+> `UploadCallback` will be invoked at least every 200ms by default, you can customize the minimal invoke interval using `SetUploadCallbackWithInterval`.
+
 ## <a name="AutoDecode">Auto-Decode</a>
 
 `Req` detect the charset of response body and decode it to utf-8 automatically to avoid garbled characters by default.

docs/api.md

@@ -203,6 +203,8 @@ Basically, you can know the meaning of most settings directly from the method na
 * [SetFileBytes(paramName, filename string, content []byte)](https://pkg.go.dev/github.com/imroc/req/v3#Request.SetFileBytes)
 * [SetFileReader(paramName, filePath string, reader io.Reader)](https://pkg.go.dev/github.com/imroc/req/v3#Request.SetFileReader)
 * [SetFileUpload(uploads ...FileUpload)](https://pkg.go.dev/github.com/imroc/req/v3#Request.SetFileUpload) - Set the fully custimized multipart file upload options.
+* [SetUploadCallback(callback UploadCallback)](https://pkg.go.dev/github.com/imroc/req/v3#Request.SetUploadCallback)
+* [SetUploadCallbackWithInterval(callback UploadCallback, minInterval time.Duration)](https://pkg.go.dev/github.com/imroc/req/v3#Request.SetUploadCallbackWithInterval)
 
 ### <a name="Download">Download</a>
 

examples/uploadcallback/README.md

@@ -0,0 +1,19 @@
+# uploadcallback
+
+This is a upload callback exmaple for `req`
+
+## How to Run
+
+Run `uploadserver`:
+
+```go
+cd uploadserver
+go run .
+```
+
+Run `uploadclient`:
+
+```go
+cd uploadclient
+go run .
+```

examples/uploadcallback/uploadclient/go.mod

@@ -0,0 +1,7 @@
+module uploadclient
+
+go 1.13
+
+replace github.com/imroc/req/v3 => ../../../
+
+require github.com/imroc/req/v3 v3.0.0

examples/uploadcallback/uploadclient/go.sum

@@ -0,0 +1,13 @@
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+golang.org/x/net v0.0.0-20220111093109-d55c255bac03 h1:0FB83qp0AzVJm+0wcIlauAjJ+tNdh7jLuacRYCIVv7s=
+golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

examples/uploadcallback/uploadclient/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"time"
+	"github.com/imroc/req/v3"
+)
+
+type SlowReader struct {
+	Size int
+	n    int
+}
+
+func (r *SlowReader) Close() error {
+	return nil
+}
+
+func (r *SlowReader) Read(p []byte) (int, error) {
+	if r.n >= r.Size {
+		return 0, io.EOF
+	}
+	time.Sleep(1 * time.Millisecond)
+	n := len(p)
+	if r.n+n >= r.Size {
+		n = r.Size - r.n
+	}
+	for i := 0; i < n; i++ {
+		p[i] = 'h'
+	}
+	r.n += n
+	return n, nil
+}
+
+func main() {
+	size := 10 * 1024 * 1024
+	req.SetFileUpload(req.FileUpload{
+		ParamName: "file",
+		FileName:  "test.txt",
+		GetFileContent: func() (io.ReadCloser, error) {
+			return &SlowReader{Size: size}, nil
+		},
+		FileSize: int64(size),
+	}).SetUploadCallbackWithInterval(func(info req.UploadInfo) {
+		fmt.Printf("%s: %.2f%%\n", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0)
+	}, 1*time.Second).Post("http://127.0.0.1:8888/upload")
+}

examples/uploadcallback/uploadserver/go.mod

@@ -0,0 +1,5 @@
+module uploadserver
+
+go 1.13
+
+require github.com/gin-gonic/gin v1.7.7

examples/uploadcallback/uploadserver/go.sum

@@ -0,0 +1,54 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
+github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

examples/uploadcallback/uploadserver/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"github.com/gin-gonic/gin"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+func main() {
+	router := gin.Default()
+	router.POST("/upload", func(c *gin.Context) {
+		body := c.Request.Body
+		io.Copy(ioutil.Discard, body)
+		c.String(http.StatusOK, "ok")
+	})
+	router.Run(":8888")
+}

middleware.go

@@ -2,6 +2,7 @@ package req
 
 import (
 	"bytes"
+	"errors"
 	"github.com/imroc/req/v3/internal/util"
 	"io"
 	"io/ioutil"
@@ -12,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 type (
@@ -55,30 +57,90 @@ func closeq(v interface{}) {
 	}
 }
 
-func writeMultipartFormFile(w *multipart.Writer, file *FileUpload) error {
+func writeMultipartFormFile(w *multipart.Writer, file *FileUpload, r *Request) error {
 	content, err := file.GetFileContent()
 	if err != nil {
 		return err
 	}
 	defer content.Close()
 	// Auto detect actual multipart content type
 	cbuf := make([]byte, 512)
+	seeEOF := false
+	lastTime := time.Now()
 	size, err := content.Read(cbuf)
-	if err != nil && err != io.EOF {
-		return err
+	if err != nil {
+		if err == io.EOF {
+			seeEOF = true
+		} else {
+			return err
+		}
 	}
 
 	pw, err := w.CreatePart(createMultipartHeader(file, http.DetectContentType(cbuf)))
 	if err != nil {
 		return err
 	}
-
 	if _, err = pw.Write(cbuf[:size]); err != nil {
 		return err
 	}
+	if seeEOF {
+		return nil
+	}
+	if r.uploadCallback == nil {
+		_, err = io.Copy(pw, content)
+		return err
+	}
 
-	_, err = io.Copy(pw, content)
-	return err
+	uploadedBytes := int64(size)
+	progressCallback := func() {
+		r.uploadCallback(UploadInfo{
+			ParamName:    file.ParamName,
+			FileName:     file.FileName,
+			FileSize:     file.FileSize,
+			UploadedSize: uploadedBytes,
+		})
+	}
+	if now := time.Now(); now.Sub(lastTime) >= r.uploadCallbackInterval {
+		lastTime = now
+		progressCallback()
+	}
+	buf := make([]byte, 1024)
+	for {
+		callback := false
+		nr, er := content.Read(buf)
+		if nr > 0 {
+			nw, ew := pw.Write(buf[:nr])
+			if nw < 0 || nr < nw {
+				nw = 0
+				if ew == nil {
+					ew = errors.New("invalid write result")
+				}
+			}
+			uploadedBytes += int64(nw)
+			if ew != nil {
+				return ew
+			}
+			if nr != nw {
+				return io.ErrShortWrite
+			}
+			if now := time.Now(); now.Sub(lastTime) >= r.uploadCallbackInterval {
+				lastTime = now
+				progressCallback()
+				callback = true
+			}
+		}
+		if er != nil {
+			if er == io.EOF {
+				if !callback {
+					progressCallback()
+				}
+				break
+			} else {
+				return er
+			}
+		}
+	}
+	return nil
 }
 
 func writeMultiPart(r *Request, w *multipart.Writer, pw *io.PipeWriter) {
@@ -88,7 +150,7 @@ func writeMultiPart(r *Request, w *multipart.Writer, pw *io.PipeWriter) {
 		}
 	}
 	for _, file := range r.uploadFiles {
-		writeMultipartFormFile(w, file)
+		writeMultipartFormFile(w, file, r)
 	}
 	w.Close()  // close multipart to write tailer boundary
 	pw.Close() // close pipe writer so that pipe reader could get EOF, and stop upload

req.go

@@ -53,14 +53,33 @@ type FileUpload struct {
 	FileName string
 	// The file to be uploaded.
 	GetFileContent GetContentFunc
+	// Optional file length in bytes.
+	FileSize int64
 
+	// Optional extra ContentDisposition parameters.
 	// According to the HTTP specification, this should be nil,
 	// but some servers may not follow the specification and
 	// requires `Content-Disposition` parameters more than just
 	// "name" and "filename".
 	ExtraContentDisposition *ContentDisposition
 }
 
+// UploadInfo is the information for each UploadCallback call.
+type UploadInfo struct {
+	// parameter name in multipart upload
+	ParamName string
+	// filename in multipart upload
+	FileName string
+	// total file length in bytes.
+	FileSize int64
+	// uploaded file length in bytes.
+	UploadedSize int64
+}
+
+// UploadCallback is the callback which will be invoked during
+// multipart upload.
+type UploadCallback func(info UploadInfo)
+
 func cloneCookies(cookies []*http.Cookie) []*http.Cookie {
 	if len(cookies) == 0 {
 		return nil