Kadai3-2: lfcd85

kadai3-2/lfcd85/.gitignore

@@ -0,0 +1 @@
+bin/mypget

kadai3-2/lfcd85/Makefile

@@ -0,0 +1,14 @@
+build:
+	GO111MODULE=on go build -o bin/mypget cmd/main.go
+
+PHONY: fmt
+fmt:
+	go fmt ./...
+
+PHONY: check
+check:
+	GO111MODULE=on go test ./... -v
+
+PHONY: coverage
+coverage:
+	GO111MODULE=on go test ./... -cover

kadai3-2/lfcd85/README.md

@@ -0,0 +1,30 @@
+# Split Downloader (my pget)
+
+A CLI for split downloading, kadai3-2 of Gopherdojo #5.
+
+Gopher道場 #5 課題3-2 `分割ダウンローダを作ろう` の実装です。
+
+## Installation
+
+```bash
+$ make build
+```
+
+## Usage
+
+URL to download is required as argument.
+
+ダウンロードするURLを引数として渡してください。
+
+
+```bash
+$ bin/mypget https://domain.name/path/to/file
+```
+
+With `-n` option, you can specify the number of split ranges. Default number is `8` . The number must be less than the length of file to download.
+
+`-n` オプションで分割の数を指定できます。デフォルトの数は `8` です。分割数はダウンロード対象のファイルサイズよりも小さい必要があります。
+
+```bash
+$ bin/mypget -n 16 https://domain.name/path/to/file
+```

kadai3-2/lfcd85/cmd/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+
+	"github.com/gopherdojo/dojo5/kadai3-2/lfcd85/mypget"
+)
+
+func main() {
+	splitNum := flag.Int("n", 8, "Number of splitted downloads")
+	flag.Parse()
+	urlStr := flag.Arg(0)
+	if urlStr == "" {
+		err := errors.New("URL is not inputted")
+		fmt.Fprintln(os.Stderr, "error: ", err)
+		os.Exit(1)
+	}
+
+	url, err := url.Parse(urlStr)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error: ", err)
+		os.Exit(1)
+	}
+
+	if err := mypget.New(url, *splitNum).Execute(nil); err != nil {
+		fmt.Fprintln(os.Stderr, "error: ", err)
+		os.Exit(1)
+	}
+}

kadai3-2/lfcd85/go.mod

@@ -0,0 +1,5 @@
+module github.com/gopherdojo/dojo5/kadai3-2/lfcd85
+
+go 1.12
+
+require golang.org/x/sync v0.0.0-20190423024810-112230192c58

kadai3-2/lfcd85/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

kadai3-2/lfcd85/mypget/export_test.go

@@ -0,0 +1,5 @@
+package mypget
+
+func (d *Downloader) ExportOutputPath() string {
+	return d.outputPath
+}

kadai3-2/lfcd85/mypget/mypget.go

@@ -0,0 +1,200 @@
+package mypget
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	tempDirName    = "partials"
+	tempFilePrefix = "partial"
+)
+
+// Downloader stores the information used for split downloading.
+type Downloader struct {
+	url        *url.URL
+	splitNum   int
+	ranges     []string
+	outputPath string
+}
+
+// New creates a Downloader struct.
+func New(url *url.URL, splitNum int) *Downloader {
+	return &Downloader{
+		url:      url,
+		splitNum: splitNum,
+	}
+}
+
+// Execute do the split download.
+func (d *Downloader) Execute(ctx context.Context) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	req, err := http.NewRequest(http.MethodGet, d.url.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if !acceptBytesRanges(resp) {
+		return errors.New("split download is not supported in this response")
+	}
+
+	length := int(resp.ContentLength)
+	if length < d.splitNum {
+		return errors.New("the number of split ranges is larger than file length")
+	}
+	d.splitToRanges(length)
+
+	tempDir, err := ioutil.TempDir("", tempDirName)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	if err := d.downloadByRanges(ctx, tempDir); err != nil {
+		return err
+	}
+
+	if err := d.combine(tempDir); err != nil {
+		return err
+	}
+
+	fmt.Printf("Download completed! saved at: %v\n", d.outputPath)
+
+	return nil
+}
+
+func acceptBytesRanges(resp *http.Response) bool {
+	return resp.Header.Get("Accept-Ranges") == "bytes"
+}
+
+func (d *Downloader) splitToRanges(length int) {
+	var ranges []string
+	var rangeStart, rangeEnd int
+	rangeLength := length / d.splitNum
+
+	for i := 0; i < d.splitNum; i++ {
+		if i != 0 {
+			rangeStart = rangeEnd + 1
+		}
+		rangeEnd = rangeStart + rangeLength
+
+		if i == d.splitNum-1 && rangeEnd != length {
+			rangeEnd = length
+		}
+
+		ranges = append(ranges, fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd))
+	}
+	d.ranges = ranges
+}
+
+func (d *Downloader) downloadByRanges(ctx context.Context, tempDir string) error {
+	eg, ctx := errgroup.WithContext(ctx)
+
+	for i, r := range d.ranges {
+		i, r := i, r
+		eg.Go(func() error {
+			req, err := http.NewRequest("GET", d.url.String(), nil)
+			if err != nil {
+				return err
+			}
+			req = req.WithContext(ctx)
+			req.Header.Set("Range", r)
+
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+
+			if err := validateStatusPartialContent(resp); err != nil {
+				return err
+			}
+
+			partialPath := generatePartialPath(tempDir, i)
+			fmt.Printf("Downloading range %v / %v (%v) ...\n", i+1, len(d.ranges), r)
+
+			f, err := os.Create(partialPath)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			if _, err = io.Copy(f, resp.Body); err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+func validateStatusPartialContent(resp *http.Response) error {
+	validStatusCode := http.StatusPartialContent
+	if resp.StatusCode != validStatusCode {
+		return fmt.Errorf("status code must be %d: actually was %d", validStatusCode, resp.StatusCode)
+	}
+	return nil
+}
+
+func generatePartialPath(tempDir string, i int) string {
+	base := strings.Join([]string{tempFilePrefix, strconv.Itoa(i)}, "_")
+	return strings.Join([]string{tempDir, base}, "/")
+}
+
+func (d *Downloader) combine(tempDir string) error {
+	d.outputPath = d.getOutputFileName()
+	f, err := os.Create(d.outputPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	fmt.Printf("Combining partials to %v ...\n", d.outputPath)
+
+	for i, _ := range d.ranges {
+		partialPath := generatePartialPath(tempDir, i)
+		partial, err := os.Open(partialPath)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(f, partial)
+		partial.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *Downloader) getOutputFileName() string {
+	base := filepath.Base(d.url.EscapedPath())
+	switch base {
+	case "/", ".", "":
+		return "output"
+	default:
+		return base
+	}
+}