Kadai3 2 misonog

kadai3-2/misonog/.gitignore

@@ -0,0 +1,16 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+/pdownload
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/

kadai3-2/misonog/Makefile

@@ -0,0 +1,9 @@
+BINARY_NAME=pdownload
+
+all: test build
+
+build:
+	go build -o $(BINARY_NAME)
+
+test:
+	go test -v ./...

kadai3-2/misonog/README.md

@@ -0,0 +1,61 @@
+# 分割ダウンローダ
+
+## 仕様
+
+- 分割ダウンロードを行う
+  - Range アクセスを用いる
+  - いくつかのゴルーチンでダウンロードしてマージする
+  - エラー処理を工夫する
+    - golang.org/x/sync/errgourp パッケージなどを使ってみる
+  - キャンセルが発生した場合の実装を行う
+
+## オプション
+
+| オプション | 内容                                   | デフォルト |
+| ---------- | -------------------------------------- | ---------- |
+| -d         | ファイルをダウンロードするディレクトリ | $PWD       |
+| -t         | タイムアウトするまでの時間（秒）       | 10         |
+
+## 利用方法
+
+### setup
+
+```shell
+$ make  # テスト & ビルド
+```
+
+### ダウンロードコマンドの例
+
+```shell
+$ ./pdownload https://blog.golang.org/gopher/header.jpg
+$ # ディレクトリとタイムアウトまでの時間の指定
+$ ./pdownload -d testdata/ -t 30 https://blog.golang.org/gopher/header.jpg
+```
+
+## ディレクトリ構造
+
+```
+.
+├── Makefile
+├── README.md
+├── go.mod
+├── go.sum
+├── main.go
+├── pdownload
+├── pdownload.go
+├── pdownload_test.go
+├── requests.go
+├── requests_test.go
+├── termination
+│   ├── termination.go
+│   └── termination_test.go
+├── testdata
+│   ├── header.jpg
+│   └── test_download
+│       └── header.jpg
+└── util.go
+```
+
+## 参考
+
+[Code-Hex/pget](https://github.com/Code-Hex/pget)と[gopherdojo/dojo3#50](https://github.com/gopherdojo/dojo3/pull/50)を参考にさせていただきました。

kadai3-2/misonog/go.mod

@@ -0,0 +1,8 @@
+module github.com/misonog/gopherdojo-studyroom/kadai3-2/misonog
+
+go 1.16
+
+require (
+	golang.org/x/net v0.0.0-20210502030024-e5908800b52b
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+)

kadai3-2/misonog/go.sum

@@ -0,0 +1,9 @@
+golang.org/x/net v0.0.0-20210502030024-e5908800b52b h1:jCRjgm6WJHzM8VQrm/es2wXYqqbq0NZ1yXFHHgzkiVQ=
+golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

kadai3-2/misonog/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"os"
+	"time"
+)
+
+const timeout = 10 * time.Second
+
+func main() {
+	ctx := context.Background()
+
+	var targetDir string
+	var timeout time.Duration
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	flag.StringVar(&targetDir, "d", pwd, "path to the directory to save the downloaded file, filename will be taken from url")
+	flag.DurationVar(&timeout, "t", timeout, "timeout of checking request in seconds")
+	flag.Parse()
+
+	cli := New()
+	if err := cli.Run(ctx, flag.Args(), targetDir, timeout); err != nil {
+		log.Fatal(err)
+	}
+}

kadai3-2/misonog/pdownload.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/misonog/gopherdojo-studyroom/kadai3-2/misonog/termination"
+)
+
+// Pdownload structs
+type Pdownload struct {
+	URL          string
+	TargetDir    string
+	Procs        int
+	timeout      time.Duration
+	useragent    string
+	referer      string
+	filename     string
+	filesize     uint
+	dirname      string
+	fullfilename string
+}
+
+func New() *Pdownload {
+	return &Pdownload{
+		Procs:   runtime.NumCPU(), // default
+		timeout: timeout,
+	}
+}
+
+func (pdownload *Pdownload) Run(ctx context.Context, args []string, targetDir string, timeout time.Duration) error {
+	var cancel context.CancelFunc
+
+	ctx, clean := termination.Listen(ctx, os.Stdout)
+	defer clean()
+
+	if err := pdownload.Ready(args, targetDir, timeout); err != nil {
+		return err
+	}
+
+	dir, err := os.MkdirTemp(pdownload.TargetDir, "")
+	if err != nil {
+		return err
+	}
+	clean = func() { os.RemoveAll(dir) }
+	defer clean()
+	termination.CleanFunc(clean)
+
+	ctx, cancel = context.WithTimeout(ctx, pdownload.timeout)
+	defer cancel()
+
+	err = pdownload.Check(ctx, dir)
+	if err != nil {
+		return err
+	}
+
+	if err := pdownload.Download(ctx); err != nil {
+		return err
+	}
+
+	if err := mergeFiles(pdownload.Procs, pdownload.filename, pdownload.dirname, pdownload.fullfilename); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (pdownload *Pdownload) Ready(args []string, targetDir string, timeout time.Duration) error {
+	if err := pdownload.parseURL(args); err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(targetDir); os.IsNotExist(err) {
+		return fmt.Errorf("target directory is not exist: %w", err)
+	}
+	pdownload.TargetDir = targetDir
+	pdownload.timeout = timeout
+
+	return nil
+}
+
+func (pdownload *Pdownload) parseURL(args []string) error {
+	if len(args) > 1 {
+		return errors.New("URL must be a single")
+	}
+	if len(args) < 1 {
+		return errors.New("urls not found in the arguments passed")
+	}
+
+	for _, arg := range args {
+		_, err := url.ParseRequestURI(arg)
+		if err != nil {
+			return err
+		}
+		pdownload.URL = arg
+	}
+
+	return nil
+}
+
+func (pdownload *Pdownload) setFullFileName(dir, filename string) {
+	if dir == "" {
+		pdownload.fullfilename = filename
+	} else {
+		pdownload.fullfilename = fmt.Sprintf("%s/%s", dir, filename)
+	}
+}
+
+// makeRange will return Range struct to download function
+func (pdownload *Pdownload) makeRange(i, split, procs uint) Range {
+	low := split * i
+	high := low + split - 1
+	if i == procs-1 {
+		high = pdownload.filesize
+	}
+
+	return Range{
+		low:   low,
+		high:  high,
+		woker: i,
+	}
+}

kadai3-2/misonog/pdownload_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+)
+
+// requests_test.goで作成しているテストサーバを利用してテストを行う
+func TestRun(t *testing.T) {
+	ctx := context.Background()
+
+	url := ts.URL
+	args := []string{fmt.Sprintf("%s/%s", url, "header.jpg")}
+	targetDir := "testdata/test_download"
+	timeout := 30 * time.Second
+
+	p := New()
+	if err := p.Run(ctx, args, targetDir, timeout); err != nil {
+		t.Errorf("failed to Run: %s", err)
+	}
+
+	if err := os.Remove(p.fullfilename); err != nil {
+		t.Errorf("failed to remove of result file: %s", err)
+	}
+}
+
+func TestParseURL(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    []string
+		expected string
+	}{
+		{name: "an URL", input: []string{"https://www.google.com/"}, expected: "https://www.google.com/"},
+		{name: "URLs", input: []string{"https://www.google.com/", "https://golang.org/"}, expected: ""},
+		{name: "invalid URL", input: []string{"invalid_url"}, expected: ""},
+	}
+
+	for _, c := range cases {
+		c := c
+		p := New()
+		t.Run(c.name, func(t *testing.T) {
+			_ = p.parseURL(c.input)
+			if actual := p.URL; c.expected != actual {
+				t.Errorf("want p.URL = %v, got %v",
+					c.expected, actual)
+			}
+		})
+	}
+}