diff --git a/kadai3-2/kimuson13/.gitignore b/kadai3-2/kimuson13/.gitignore new file mode 100644 index 00000000..f7b34922 --- /dev/null +++ b/kadai3-2/kimuson13/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ \ No newline at end of file diff --git a/kadai3-2/kimuson13/README.md b/kadai3-2/kimuson13/README.md new file mode 100644 index 00000000..b7ca24ee --- /dev/null +++ b/kadai3-2/kimuson13/README.md @@ -0,0 +1,17 @@ +# 分割ダウンローダ +Rangeアクセスを用いて、ダウンロードする。 +## 使用方法 +まずはkimuson13のディレクトリに移動する。 +その後、 +```go run main.go`` +もしくは、 +```go build main.go``` +```./main``` +で実行可能。 +## オプション +```-p ``` +分割数を指定できる。 +```-t ``` +タイムアウトが起きる時間を指定できる。 +```-f ``` +ダウンロード後のファイル名を指定できる。 diff --git a/kadai3-2/kimuson13/TODO.md b/kadai3-2/kimuson13/TODO.md new file mode 100644 index 00000000..4899fb58 --- /dev/null +++ b/kadai3-2/kimuson13/TODO.md @@ -0,0 +1,20 @@ +TODO +=== +- [x] GET requestを送ってみる +- [x] ダウンロードしたものをファイルにする +- [x] HEADでコンテンツのサイズを確認する +- [x] range requestの実装 +- [x] 分割ダウンロードの実装 +- [x] timeoutの実装 + +- [x] エラー処理を工夫する。(道場の資料39を参考にする。) +- [x] キャンセルが発生した場合の実装(道場の資料48を参考にする。) +- [x] 型とメソッドにまとめる +- [x] 入力された情報が正しいか判断する(govalidateのIsURLが使えそう。) + +- [x] オプションのstructを作る(分割数、拡張子、タイムアウト、ダウンロードしたファイルの名前) +- [x] オプションに入力された引数が正しいか判断する。 + +- [x] 各関数の説明を書く +- [] レビューで指摘してもらった箇所の修正 +- [] テスト書く \ No newline at end of file diff --git a/kadai3-2/kimuson13/download/_testdata/empty.txt b/kadai3-2/kimuson13/download/_testdata/empty.txt new file mode 100644 index 00000000..e69de29b diff --git a/kadai3-2/kimuson13/download/_testdata/foo.png b/kadai3-2/kimuson13/download/_testdata/foo.png new file mode 100644 index 00000000..8aae4662 Binary files /dev/null and b/kadai3-2/kimuson13/download/_testdata/foo.png differ diff --git a/kadai3-2/kimuson13/download/download.go b/kadai3-2/kimuson13/download/download.go new file mode 100644 index 00000000..191693df --- /dev/null +++ b/kadai3-2/kimuson13/download/download.go @@ -0,0 +1,297 @@ +package download + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "time" + + "golang.org/x/sync/errgroup" +) + +var ( + ErrValidateParallelism = errors.New("the parallel number needs to be bigger than 1") + ErrValidateTimeout = errors.New("the timeout needs to be bigeer than 0") + ErrNotIncludeRangeAccess = errors.New("the response does not include Accept-Range header") + ErrNotContent = errors.New("it is not content") + ErrNoContentLength = errors.New("it does not have Content-Length") +) + +// Donwnloader struct +type Downloader struct { + parallel int + timeout int + filename string + url string +} + +// Rnage struct +type Range struct { + low int + high int + number int +} + +// New for download package +func New(opts *Options) *Downloader { + return &Downloader{ + parallel: opts.Parallel, + timeout: opts.Timeout, + filename: opts.Filename, + url: opts.URL, + } +} + +func CreateTempdir() error { + if err := os.Mkdir("tempdir", 0755); err != nil { + return err + } + + return nil +} + +func DeleteTempdir() { + err := os.RemoveAll("tempdir") + if err != nil { + log.Println("can't remove the tempdir", err) + } +} + +// Run excecute method in download package +func (d *Downloader) Run(ctx context.Context) error { + if err := d.Validation(); err != nil { + return err + } + + if err := CreateTempdir(); err != nil { + return err + } + defer DeleteTempdir() + + contentLength, err := d.CheckContentLength(ctx) + if err != nil { + return err + } + + if err := d.Download(contentLength, ctx); err != nil { + return err + } + + if err := d.MergeFile(d.parallel, contentLength); err != nil { + return err + } + + return nil +} + +//Preparate method define the variables to Donwload +func (d *Downloader) Validation() error { + if d.parallel <= 1 { + return ErrValidateParallelism + } + + if d.timeout < 1 { + return ErrValidateTimeout + } + + return nil +} + +// CheckContentLength method gets the Content-Length want to download +func (d *Downloader) CheckContentLength(ctx context.Context) (int, error) { + if _, err := fmt.Fprintf(os.Stdout, "Start HEAD request to check Content-Length\n"); err != nil { + return 0, err + } + + req, err := http.NewRequest("HEAD", d.url, nil) + if err != nil { + return 0, err + } + + req = req.WithContext(ctx) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return 0, err + } + + acceptRange := res.Header.Get("Accept-Ranges") + if _, err := fmt.Fprintf(os.Stdout, "got: Accept-Ranges: %s\n", acceptRange); err != nil { + return 0, err + } + + if acceptRange == "" { + return 0, ErrNotIncludeRangeAccess + } + + if acceptRange != "bytes" { + return 0, ErrNotContent + } + + contentLength := int(res.ContentLength) + if _, err := fmt.Fprintf(os.Stdout, "got: Content-Length: %v\n", contentLength); err != nil { + return 0, err + } + + if contentLength < 1 { + return 0, ErrNoContentLength + } + + return contentLength, nil +} + +// Download method does split-download with goroutine +func (d *Downloader) Download(contentLength int, ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, time.Duration(d.timeout)*time.Second) + defer cancel() + + parallel := d.parallel + split := contentLength / parallel + grp, ctx := errgroup.WithContext(ctx) + for i := 0; i < parallel; i++ { + r := MakeRange(i, split, parallel, contentLength) + tempfile := fmt.Sprintf("tempdir/tempfile.%d.%d", parallel, r.number) + filename, err := CreateTempfile(tempfile) + if err != nil { + return err + } + + grp.Go(func() error { + err := Requests(r, d.url, filename) + return err + }) + } + + if err := grp.Wait(); err != nil { + return err + } + + return nil +} + +func CreateTempfile(name string) (string, error) { + file, err := os.Create(name) + if err != nil { + return "", err + } + + defer func() { + err := file.Close() + if err != nil { + log.Println("can't close the "+name, err) + } + }() + + return file.Name(), nil +} + +// MakeRange function distributes Content-Length for split-download +func MakeRange(i, split, parallel, contentLength int) Range { + low := split * i + high := low + split - 1 + if i == parallel-1 { + high = contentLength + } + + return Range{ + low: low, + high: high, + number: i, + } +} + +// Requests function sends GET request +func Requests(r Range, url, filename string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.low, r.high)) + if _, err := fmt.Fprintf(os.Stdout, "start GET request: bytes=%d-%d\n", r.low, r.high); err != nil { + return err + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer func() { + err := res.Body.Close() + if err != nil { + log.Println("the response body can't close", err) + } + }() + + if res.StatusCode != http.StatusPartialContent { + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + + output, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return err + } + defer func() { + err := output.Close() + if err != nil { + log.Println("can't close the tempfile", err) + } + }() + + _, err = io.Copy(output, res.Body) + if err != nil { + return err + } + return nil +} + +// MergeFile method merges tempfiles to new file +func (d *Downloader) MergeFile(parallel, contentLength int) error { + fmt.Println("\nmerging files...") + filename := d.filename + fh, err := os.Create(filename) + if err != nil { + return err + } + + defer func() { + err := fh.Close() + if err != nil { + log.Println("can't close the download file!", err) + } + }() + + for i := 0; i < parallel; i++ { + if err := Merger(parallel, i, fh); err != nil { + return err + } + } + + fmt.Println("complete parallel donwload") + return nil +} + +func Merger(parallel, i int, fh *os.File) error { + f := fmt.Sprintf("tempdir/tempfile.%d.%d", parallel, i) + sub, err := os.Open(f) + if err != nil { + return err + } + + _, err = io.Copy(fh, sub) + if err != nil { + return err + } + + defer func() { + err := sub.Close() + if err != nil { + log.Println("can't close the "+f, err) + } + }() + return nil +} diff --git a/kadai3-2/kimuson13/download/download_test.go b/kadai3-2/kimuson13/download/download_test.go new file mode 100644 index 00000000..b1b46ac8 --- /dev/null +++ b/kadai3-2/kimuson13/download/download_test.go @@ -0,0 +1,273 @@ +package download_test + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/kimuson13/gopherdojo-studyroom/kimuson13/download" +) + +var currentTestdataName string +var registeredTestdatum = map[string][]byte{ + "foo.png": readTestdata("foo.png"), + "empty.txt": readTestdata("empty.txt"), +} + +func TestDownloader_Run_Success(t *testing.T) { + cases := map[string]struct { + parallelism int + timeout int + output string + currentTestDataName string + }{ + "normal": { + parallelism: 3, + timeout: 30, + output: "output.txt", + currentTestDataName: "foo.png", + }, + "highparallelism": { + parallelism: 100, + timeout: 30, + output: "output.txt", + currentTestDataName: "foo.png", + }, + "lowparallelism": { + parallelism: 2, + timeout: 30, + output: "output.txt", + currentTestDataName: "foo.png", + }, + "shortname": { + parallelism: 3, + timeout: 30, + output: "a", + currentTestDataName: "foo.png", + }, + } + + for n, c := range cases { + c := c + t.Run(n, func(t *testing.T) { + currentTestdataName = c.currentTestDataName + + output, clean := createTempOutput(t, c.output) + defer clean() + + ts, closefunc := newTestServer(t, normalHandler) + defer closefunc() + + opt := &download.Options{ + Parallel: c.parallelism, + Timeout: c.timeout, + Filename: output, + URL: ts.URL, + } + + downloader := download.New(opt) + + err := downloader.Run(context.Background()) + if err != nil { + t.Fatalf("err: %s", err) + } + + before, err := os.Stat(path.Join("_testdata", currentTestdataName)) + if err != nil { + panic(err) + } + + after, err := os.Stat(output) + if err != nil { + panic(err) + } + + if after.Name() != c.output { + t.Fatalf("downloading file name is %v, but expected is %v", after.Name(), c.output) + } + + if before.Size() != after.Size()-1 { + t.Fatalf("it is not same %d and %d", before.Size(), after.Size()-1) + } + }) + } +} + +func TestDownloader_With_Errors(t *testing.T) { + cases := map[string]struct { + parallelism int + timeout int + expected error + handler func(t *testing.T, w http.ResponseWriter, r *http.Request) + currentTestDataName string + }{ + "parallelValidationError0": { + parallelism: 0, + timeout: 30, + expected: download.ErrValidateParallelism, + handler: normalHandler, + currentTestDataName: "foo.png", + }, + "parallelValidationError1": { + parallelism: 1, + timeout: 30, + expected: download.ErrValidateParallelism, + handler: normalHandler, + currentTestDataName: "foo.png", + }, + "timeoutValidationError": { + parallelism: 3, + timeout: 0, + expected: download.ErrValidateTimeout, + handler: normalHandler, + currentTestDataName: "foo.png", + }, + "noContent-LengthError": { + parallelism: 3, + timeout: 30, + expected: download.ErrNoContentLength, + handler: normalHandler, + currentTestDataName: "empty.txt", + }, + "notIncludeRange-AccessError": { + parallelism: 3, + timeout: 30, + expected: download.ErrNotIncludeRangeAccess, + handler: notIncludeAcceptRangeHandler, + currentTestDataName: "foo.png", + }, + "notContentInAccpetRange": { + parallelism: 3, + timeout: 30, + expected: download.ErrNotContent, + handler: notcontentAcceptRangeHeaderHandler, + currentTestDataName: "foo.png", + }, + } + + for n, c := range cases { + c := c + currentTestdataName = c.currentTestDataName + t.Run(n, func(t *testing.T) { + testForErrors(t, c.parallelism, c.timeout, c.expected, c.handler) + }) + } +} + +func testForErrors(t *testing.T, parallel, timeout int, expected error, handler func(t *testing.T, w http.ResponseWriter, r *http.Request)) { + t.Helper() + + output, clean := createTempOutput(t, "output.txt") + defer clean() + + ts, closefunc := newTestServer(t, handler) + defer closefunc() + + opt := &download.Options{ + Parallel: parallel, + Timeout: timeout, + Filename: output, + URL: ts.URL, + } + + downloader := download.New(opt) + + actual := downloader.Run(context.Background()) + if actual != expected { + t.Errorf("expected: %v, but got: %v", expected, actual) + } +} + +func newTestServer(t *testing.T, handler func(t *testing.T, w http.ResponseWriter, r *http.Request)) (*httptest.Server, func()) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r) + }, + )) + + return ts, func() { ts.Close() } +} + +func normalHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + w.Header().Set("Accept-Ranges", "bytes") + + rangeHeader := r.Header.Get("Range") + + body := func() []byte { + if rangeHeader == "" { + return registeredTestdatum[currentTestdataName] + } + + eqlSplitVals := strings.Split(rangeHeader, "=") + if eqlSplitVals[0] != "bytes" { + t.Fatalf("err: %s", eqlSplitVals[1]) + } + + c := strings.Split(eqlSplitVals[1], "-") + + min, err := strconv.Atoi(c[0]) + if err != nil { + t.Fatalf("err: %s", err) + } + + max, err := strconv.Atoi(c[1]) + if err != nil { + t.Fatalf("err: %s", err) + } + + return registeredTestdatum[currentTestdataName][min : max+1] + }() + + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + + w.WriteHeader(http.StatusPartialContent) + w.Write(body) +} + +func notIncludeAcceptRangeHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + body := registeredTestdatum[currentTestdataName] + w.Write(body) +} + +func notcontentAcceptRangeHeaderHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + w.Header().Set("Accept-Ranges", "none") + + body := registeredTestdatum[currentTestdataName] + w.Write(body) +} + +func readTestdata(filename string) []byte { + b, err := ioutil.ReadFile(path.Join("_testdata", filename)) + if err != nil { + panic(err) + } + + return b +} + +func createTempOutput(t *testing.T, name string) (string, func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "parallel-download") + if err != nil { + panic(err) + } + + return filepath.Join(dir, name), func() { os.RemoveAll(dir) } +} diff --git a/kadai3-2/kimuson13/download/option.go b/kadai3-2/kimuson13/download/option.go new file mode 100644 index 00000000..96791db4 --- /dev/null +++ b/kadai3-2/kimuson13/download/option.go @@ -0,0 +1,37 @@ +package download + +import ( + "flag" + "net/url" + "runtime" +) + +// Options struct +type Options struct { + Parallel int + Timeout int + Filename string + URL string +} + +// Parse method parses options +func (opts *Options) Parse(args ...string) (*Options, error) { + flg := flag.NewFlagSet("parallelDownload", flag.ExitOnError) + parallel := flg.Int("p", runtime.NumCPU(), "separate Content-Length with this argument") + timeout := flg.Int("t", 30, "timeout for this second") + filename := flg.String("f", "paralleldownload", "save the file as this name") + if err := flg.Parse(args); err != nil { + return nil, err + } + u, err := url.Parse(flg.Arg(0)) + if err != nil { + return nil, err + } + + return &Options{ + Parallel: *parallel, + Timeout: *timeout, + Filename: *filename, + URL: u.String(), + }, nil +} diff --git a/kadai3-2/kimuson13/download/option_test.go b/kadai3-2/kimuson13/download/option_test.go new file mode 100644 index 00000000..6336b619 --- /dev/null +++ b/kadai3-2/kimuson13/download/option_test.go @@ -0,0 +1,59 @@ +package download_test + +import ( + "runtime" + "testing" + + "github.com/kimuson13/gopherdojo-studyroom/kimuson13/download" +) + +var testurl string = "https://www.naoshima.net/wp-content/uploads/2019/07/393d0895747d5a947ad3acc35eb09688.pdf" + +var options download.Options + +var fileName string = "paralleldownload" + +func TestParse(t *testing.T) { + cases := []struct { + name string + args []string + eParallel int + eTimeout int + eFilename string + }{ + {name: "noOption", args: []string{testurl}, eParallel: runtime.NumCPU(), eTimeout: 30, eFilename: fileName}, + {name: "parallelOption", args: []string{"-p=6", testurl}, eParallel: 6, eTimeout: 30, eFilename: fileName}, + {name: "timeoutOption", args: []string{"-t=10", testurl}, eParallel: runtime.NumCPU(), eTimeout: 10, eFilename: fileName}, + {name: "filenameOption", args: []string{"-f=test", testurl}, eParallel: runtime.NumCPU(), eTimeout: 30, eFilename: "test"}, + {name: "PandT", args: []string{"-p=6", "-t=20", testurl}, eParallel: 6, eTimeout: 20, eFilename: fileName}, + {name: "PandF", args: []string{"-p=6", "-f=test", testurl}, eParallel: 6, eTimeout: 30, eFilename: "test"}, + {name: "TandF", args: []string{"-t=20", "-f=test", testurl}, eParallel: runtime.NumCPU(), eTimeout: 20, eFilename: "test"}, + {name: "AllOption", args: []string{"-p=6", "-t=20", "-f=test", testurl}, eParallel: 6, eTimeout: 20, eFilename: "test"}, + } + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + testParse(t, c.args, c.eParallel, c.eTimeout, c.eFilename) + }) + } +} + +func testParse(t *testing.T, args []string, parallel, timeout int, filename string) { + t.Helper() + opt, err := options.Parse(args...) + if err != nil { + t.Fatal(err) + } + + if opt.Parallel != parallel { + t.Errorf("want %v, got %v", parallel, opt.Parallel) + } + + if opt.Timeout != timeout { + t.Errorf("want %v, got %v", timeout, opt.Timeout) + } + + if opt.Filename != filename { + t.Errorf("want %v, got %v", filename, opt.Filename) + } +} diff --git a/kadai3-2/kimuson13/go.mod b/kadai3-2/kimuson13/go.mod new file mode 100644 index 00000000..b8f03b2f --- /dev/null +++ b/kadai3-2/kimuson13/go.mod @@ -0,0 +1,5 @@ +module github.com/kimuson13/gopherdojo-studyroom/kimuson13 + +go 1.16 + +require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c diff --git a/kadai3-2/kimuson13/go.sum b/kadai3-2/kimuson13/go.sum new file mode 100644 index 00000000..5c00efd3 --- /dev/null +++ b/kadai3-2/kimuson13/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/kimuson13/interrupt/interrupt.go b/kadai3-2/kimuson13/interrupt/interrupt.go new file mode 100644 index 00000000..86cdd1cb --- /dev/null +++ b/kadai3-2/kimuson13/interrupt/interrupt.go @@ -0,0 +1,31 @@ +package interrupt + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" +) + +// Listen function check canceled or not +func Listen(ctx context.Context) (context.Context, func()) { + ctx, cancel := context.WithCancel(ctx) + ch := make(chan os.Signal) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + select { + case <-ch: + fmt.Println("interrupt") + if f, err := os.Stat("tempdir"); os.IsExist(err) || f.IsDir() { + if err := os.RemoveAll("tempdir"); err != nil { + log.Fatal("err:", err) + } + } + cancel() + } + }() + + return ctx, cancel +} diff --git a/kadai3-2/kimuson13/interrupt/interrupt_test.go b/kadai3-2/kimuson13/interrupt/interrupt_test.go new file mode 100644 index 00000000..21e0ac28 --- /dev/null +++ b/kadai3-2/kimuson13/interrupt/interrupt_test.go @@ -0,0 +1,32 @@ +package interrupt_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/kimuson13/gopherdojo-studyroom/kimuson13/interrupt" +) + +func TestInterrupt(t *testing.T) { + ctx, cancel := interrupt.Listen(context.Background()) + defer cancel() + + proc, err := os.FindProcess(os.Getpid()) + if err != nil { + t.Fatalf("err: %d", err) + } + + err = proc.Signal(os.Interrupt) + if err != nil { + t.Fatalf("err: %d", err) + } + + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + t.Fatal("timeout") + } +} diff --git a/kadai3-2/kimuson13/main.go b/kadai3-2/kimuson13/main.go new file mode 100644 index 00000000..33a87715 --- /dev/null +++ b/kadai3-2/kimuson13/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/kimuson13/gopherdojo-studyroom/kimuson13/download" + "github.com/kimuson13/gopherdojo-studyroom/kimuson13/interrupt" +) + +func main() { + err := setUp(os.Args[1:]) + if err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } +} + +//setUp function preparate for running in main package +func setUp(args []string) error { + var options download.Options + ctx := context.Background() + ctx, cancel := interrupt.Listen(ctx) + defer cancel() + + opts, err := options.Parse(args...) + if err != nil { + return err + } + + return download.New(opts).Run(ctx) +}