diff --git a/kadai3-2/misonog/.gitignore b/kadai3-2/misonog/.gitignore new file mode 100644 index 00000000..9e03bbfa --- /dev/null +++ b/kadai3-2/misonog/.gitignore @@ -0,0 +1,16 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +/pdownload + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/kadai3-2/misonog/Makefile b/kadai3-2/misonog/Makefile new file mode 100644 index 00000000..28cb0153 --- /dev/null +++ b/kadai3-2/misonog/Makefile @@ -0,0 +1,9 @@ +BINARY_NAME=pdownload + +all: test build + +build: + go build -o $(BINARY_NAME) + +test: + go test -v ./... diff --git a/kadai3-2/misonog/README.md b/kadai3-2/misonog/README.md new file mode 100644 index 00000000..80e18aa8 --- /dev/null +++ b/kadai3-2/misonog/README.md @@ -0,0 +1,61 @@ +# 分割ダウンローダ + +## 仕様 + +- 分割ダウンロードを行う + - Range アクセスを用いる + - いくつかのゴルーチンでダウンロードしてマージする + - エラー処理を工夫する + - golang.org/x/sync/errgourp パッケージなどを使ってみる + - キャンセルが発生した場合の実装を行う + +## オプション + +| オプション | 内容 | デフォルト | +| ---------- | -------------------------------------- | ---------- | +| -d | ファイルをダウンロードするディレクトリ | $PWD | +| -t | タイムアウトするまでの時間(秒) | 10 | + +## 利用方法 + +### setup + +```shell +$ make # テスト & ビルド +``` + +### ダウンロードコマンドの例 + +```shell +$ ./pdownload https://blog.golang.org/gopher/header.jpg +$ # ディレクトリとタイムアウトまでの時間の指定 +$ ./pdownload -d testdata/ -t 30 https://blog.golang.org/gopher/header.jpg +``` + +## ディレクトリ構造 + +``` +. +├── Makefile +├── README.md +├── go.mod +├── go.sum +├── main.go +├── pdownload +├── pdownload.go +├── pdownload_test.go +├── requests.go +├── requests_test.go +├── termination +│ ├── termination.go +│ └── termination_test.go +├── testdata +│ ├── header.jpg +│ └── test_download +│ └── header.jpg +└── util.go +``` + +## 参考 + +[Code-Hex/pget](https://github.com/Code-Hex/pget)と[gopherdojo/dojo3#50](https://github.com/gopherdojo/dojo3/pull/50)を参考にさせていただきました。 diff --git a/kadai3-2/misonog/go.mod b/kadai3-2/misonog/go.mod new file mode 100644 index 00000000..9b04e335 --- /dev/null +++ b/kadai3-2/misonog/go.mod @@ -0,0 +1,8 @@ +module github.com/misonog/gopherdojo-studyroom/kadai3-2/misonog + +go 1.16 + +require ( + golang.org/x/net v0.0.0-20210502030024-e5908800b52b + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c +) diff --git a/kadai3-2/misonog/go.sum b/kadai3-2/misonog/go.sum new file mode 100644 index 00000000..76b3e047 --- /dev/null +++ b/kadai3-2/misonog/go.sum @@ -0,0 +1,9 @@ +golang.org/x/net v0.0.0-20210502030024-e5908800b52b h1:jCRjgm6WJHzM8VQrm/es2wXYqqbq0NZ1yXFHHgzkiVQ= +golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/kadai3-2/misonog/main.go b/kadai3-2/misonog/main.go new file mode 100644 index 00000000..158ae0ad --- /dev/null +++ b/kadai3-2/misonog/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "flag" + "log" + "os" + "time" +) + +const timeout = 10 * time.Second + +func main() { + ctx := context.Background() + + var targetDir string + var timeout time.Duration + + pwd, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + + flag.StringVar(&targetDir, "d", pwd, "path to the directory to save the downloaded file, filename will be taken from url") + flag.DurationVar(&timeout, "t", timeout, "timeout of checking request in seconds") + flag.Parse() + + cli := New() + if err := cli.Run(ctx, flag.Args(), targetDir, timeout); err != nil { + log.Fatal(err) + } +} diff --git a/kadai3-2/misonog/pdownload.go b/kadai3-2/misonog/pdownload.go new file mode 100644 index 00000000..ad305e07 --- /dev/null +++ b/kadai3-2/misonog/pdownload.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "runtime" + "time" + + "github.com/misonog/gopherdojo-studyroom/kadai3-2/misonog/termination" +) + +// Pdownload structs +type Pdownload struct { + URL string + TargetDir string + Procs int + timeout time.Duration + useragent string + referer string + filename string + filesize uint + dirname string + fullfilename string +} + +func New() *Pdownload { + return &Pdownload{ + Procs: runtime.NumCPU(), // default + timeout: timeout, + } +} + +func (pdownload *Pdownload) Run(ctx context.Context, args []string, targetDir string, timeout time.Duration) error { + var cancel context.CancelFunc + + ctx, clean := termination.Listen(ctx, os.Stdout) + defer clean() + + if err := pdownload.Ready(args, targetDir, timeout); err != nil { + return err + } + + dir, err := os.MkdirTemp(pdownload.TargetDir, "") + if err != nil { + return err + } + clean = func() { os.RemoveAll(dir) } + defer clean() + termination.CleanFunc(clean) + + ctx, cancel = context.WithTimeout(ctx, pdownload.timeout) + defer cancel() + + err = pdownload.Check(ctx, dir) + if err != nil { + return err + } + + if err := pdownload.Download(ctx); err != nil { + return err + } + + if err := mergeFiles(pdownload.Procs, pdownload.filename, pdownload.dirname, pdownload.fullfilename); err != nil { + return err + } + + return nil +} + +func (pdownload *Pdownload) Ready(args []string, targetDir string, timeout time.Duration) error { + if err := pdownload.parseURL(args); err != nil { + return err + } + + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + return fmt.Errorf("target directory is not exist: %w", err) + } + pdownload.TargetDir = targetDir + pdownload.timeout = timeout + + return nil +} + +func (pdownload *Pdownload) parseURL(args []string) error { + if len(args) > 1 { + return errors.New("URL must be a single") + } + if len(args) < 1 { + return errors.New("urls not found in the arguments passed") + } + + for _, arg := range args { + _, err := url.ParseRequestURI(arg) + if err != nil { + return err + } + pdownload.URL = arg + } + + return nil +} + +func (pdownload *Pdownload) setFullFileName(dir, filename string) { + if dir == "" { + pdownload.fullfilename = filename + } else { + pdownload.fullfilename = fmt.Sprintf("%s/%s", dir, filename) + } +} + +// makeRange will return Range struct to download function +func (pdownload *Pdownload) makeRange(i, split, procs uint) Range { + low := split * i + high := low + split - 1 + if i == procs-1 { + high = pdownload.filesize + } + + return Range{ + low: low, + high: high, + woker: i, + } +} diff --git a/kadai3-2/misonog/pdownload_test.go b/kadai3-2/misonog/pdownload_test.go new file mode 100644 index 00000000..c32011ba --- /dev/null +++ b/kadai3-2/misonog/pdownload_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + "os" + "testing" + "time" +) + +// requests_test.goで作成しているテストサーバを利用してテストを行う +func TestRun(t *testing.T) { + ctx := context.Background() + + url := ts.URL + args := []string{fmt.Sprintf("%s/%s", url, "header.jpg")} + targetDir := "testdata/test_download" + timeout := 30 * time.Second + + p := New() + if err := p.Run(ctx, args, targetDir, timeout); err != nil { + t.Errorf("failed to Run: %s", err) + } + + if err := os.Remove(p.fullfilename); err != nil { + t.Errorf("failed to remove of result file: %s", err) + } +} + +func TestParseURL(t *testing.T) { + cases := []struct { + name string + input []string + expected string + }{ + {name: "an URL", input: []string{"https://www.google.com/"}, expected: "https://www.google.com/"}, + {name: "URLs", input: []string{"https://www.google.com/", "https://golang.org/"}, expected: ""}, + {name: "invalid URL", input: []string{"invalid_url"}, expected: ""}, + } + + for _, c := range cases { + c := c + p := New() + t.Run(c.name, func(t *testing.T) { + _ = p.parseURL(c.input) + if actual := p.URL; c.expected != actual { + t.Errorf("want p.URL = %v, got %v", + c.expected, actual) + } + }) + } +} diff --git a/kadai3-2/misonog/requests.go b/kadai3-2/misonog/requests.go new file mode 100644 index 00000000..881cdccc --- /dev/null +++ b/kadai3-2/misonog/requests.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + + "golang.org/x/net/context/ctxhttp" + "golang.org/x/sync/errgroup" +) + +// Range struct for range access +type Range struct { + low uint + high uint + woker uint +} + +func isLastProc(i, procs uint) bool { + return i == procs-1 +} + +// Check method check be able to range access. +func (p *Pdownload) Check(ctx context.Context, dir string) error { + res, err := ctxhttp.Head(ctx, http.DefaultClient, p.URL) + if err != nil { + return err + } + + if res.Header.Get("Accept-Ranges") != "bytes" { + return fmt.Errorf("not supported range access: %s", p.URL) + } + + if res.ContentLength <= 0 { + return errors.New("invalid content length") + } + + filename := p.filename + if filename == "" { + filename = path.Base(p.URL) + } + p.filename = filename + p.setFullFileName(p.TargetDir, filename) + p.dirname = dir + p.filesize = uint(res.ContentLength) + + return nil +} + +// Download method distributes the task to each goroutine +func (p *Pdownload) Download(ctx context.Context) error { + procs := uint(p.Procs) + filesize := p.filesize + + // calculate split file size + split := filesize / procs + + grp, ctx := errgroup.WithContext(ctx) + + p.Assignment(grp, ctx, procs, split) + + // wait for Assignment method + if err := grp.Wait(); err != nil { + return err + } + + return nil +} + +func (p Pdownload) Assignment(grp *errgroup.Group, ctx context.Context, procs, split uint) { + filename := p.filename + dirname := p.dirname + + for i := uint(0); i < procs; i++ { + partName := fmt.Sprintf("%s/%s.%d.%d", dirname, filename, procs, i) + + // make range + r := p.makeRange(i, split, procs) + if info, err := os.Stat(partName); err == nil { + infosize := uint(info.Size()) + // check if the part is fully downloaded + if isLastProc(i, procs) { + if infosize == r.high-r.low { + continue + } + } else if infosize == split { + continue + } + + // make low range from this next byte + r.low += infosize + } + + // execute get request + grp.Go(func() error { + return p.Requests(ctx, r, filename, dirname, p.URL) + }) + } + +} + +// Requests method will download the file +func (p Pdownload) Requests(ctx context.Context, r Range, filename, dirname, url string) error { + res, err := p.MakeResponse(ctx, r, url) + if err != nil { + return fmt.Errorf("failed to split get requests: %d", r.woker) + } + defer res.Body.Close() + + partName := fmt.Sprintf("%s/%s.%d.%d", dirname, filename, p.Procs, r.woker) + + output, err := os.OpenFile(partName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 06666) + if err != nil { + return fmt.Errorf("failed to create %s in %s", filename, dirname) + } + defer output.Close() + + if _, err := io.Copy(output, res.Body); err != nil { + return err + } + + return nil +} + +// MakeResponse return *http.Respnse include context and range header +func (p Pdownload) MakeResponse(ctx context.Context, r Range, url string) (*http.Response, error) { + // create get request + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to split NewRequest for get: %d", r.woker) + } + + // set download ranges + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.low, r.high)) + + // set useragent + if p.useragent != "" { + req.Header.Set("User-Agent", p.useragent) + } + + // set referer + if p.referer != "" { + req.Header.Set("Referer", p.referer) + } + + return http.DefaultClient.Do(req) +} diff --git a/kadai3-2/misonog/requests_test.go b/kadai3-2/misonog/requests_test.go new file mode 100644 index 00000000..1046757d --- /dev/null +++ b/kadai3-2/misonog/requests_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" +) + +const testDir = "testdata/test_download" + +var ( + dir string + mkdirErr error + ts *httptest.Server +) + +func TestMain(m *testing.M) { + setUp() + code := m.Run() + tearDown() + os.Exit(code) +} + +func setUp() { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/header.jpg", http.StatusFound) + }) + + mux.HandleFunc("/header.jpg", func(w http.ResponseWriter, r *http.Request) { + fp := "testdata/header.jpg" + data, err := os.ReadFile(fp) + if err != nil { + panic(err) + } + http.ServeContent(w, r, fp, time.Now(), bytes.NewReader(data)) + }) + + ts = httptest.NewServer(mux) + + dir, mkdirErr = os.MkdirTemp(testDir, "") + if mkdirErr != nil { + panic(mkdirErr) + } +} + +func tearDown() { + ts.Close() + os.RemoveAll(dir) +} + +func TestCheck(t *testing.T) { + p := New() + p.URL = ts.URL + + if err := p.Check(context.Background(), dir); err != nil { + t.Errorf("failed to check header: %s", err) + } +} + +func TestDownload(t *testing.T) { + p := New() + p.URL = ts.URL + p.TargetDir = testDir + p.filename = "header.jpg" + + err := p.Check(context.Background(), dir) + if err != nil { + t.Errorf("failed to check header: %s", err) + } + + if err := p.Download(context.Background()); err != nil { + t.Errorf("failed to download: %s", err) + } + + for i := 0; i < p.Procs; i++ { + filename := fmt.Sprintf(p.dirname+"/header.jpg.%d.%d", p.Procs, i) + _, err := os.Stat(filename) + if err != nil { + t.Errorf("file not exist: %s", err) + } + } +} + +// utils.goにあるメソッドをテストするのは違和感があるがこのファイルの中でテストを行う +func TestMergeFiles(t *testing.T) { + p := New() + p.URL = ts.URL + p.TargetDir = testDir + p.filename = "header.jpg" + p.fullfilename = "testdata/test_download/header.jpg" + + err := p.Check(context.Background(), dir) + if err != nil { + t.Errorf("failed to check header: %s", err) + } + + if err := p.Download(context.Background()); err != nil { + t.Errorf("failed to download: %s", err) + } + + if err := mergeFiles(p.Procs, p.filename, p.dirname, p.fullfilename); err != nil { + t.Errorf("failed to MergeFiles: %s", err) + } +} diff --git a/kadai3-2/misonog/termination/termination.go b/kadai3-2/misonog/termination/termination.go new file mode 100644 index 00000000..7ca037a8 --- /dev/null +++ b/kadai3-2/misonog/termination/termination.go @@ -0,0 +1,37 @@ +package termination + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "syscall" +) + +var cleanFns []func() +var osExit = os.Exit + +// Listen listens signal +func Listen(ctx context.Context, w io.Writer) (context.Context, func()) { + ctx, cancel := context.WithCancel(ctx) + + ch := make(chan os.Signal, 2) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + <-ch + fmt.Fprintln(w, "Ctrl+C pressed in Terminal") + cancel() + for _, f := range cleanFns { + f() + } + osExit(0) + }() + + return ctx, cancel +} + +// CleanFunc registers clean function +func CleanFunc(f func()) { + cleanFns = append(cleanFns, f) +} diff --git a/kadai3-2/misonog/termination/termination_test.go b/kadai3-2/misonog/termination/termination_test.go new file mode 100644 index 00000000..c9eeeadb --- /dev/null +++ b/kadai3-2/misonog/termination/termination_test.go @@ -0,0 +1,36 @@ +package termination + +import ( + "context" + "io" + "os" + "testing" + "time" +) + +func TestListen(t *testing.T) { + CleanFunc(func() {}) + + doneCh := make(chan struct{}) + osExit = func(code int) { doneCh <- struct{}{} } + + _, clean := Listen(context.Background(), io.Discard) + defer clean() + + proc, err := os.FindProcess(os.Getpid()) + if err != nil { + t.Fatal(err) + } + + err = proc.Signal(os.Interrupt) + if err != nil { + t.Fatal(err) + } + + select { + case <-doneCh: + return + case <-time.After(100 * time.Millisecond): + t.Fatal("timeout") + } +} diff --git a/kadai3-2/misonog/testdata/header.jpg b/kadai3-2/misonog/testdata/header.jpg new file mode 100644 index 00000000..bcf63e98 Binary files /dev/null and b/kadai3-2/misonog/testdata/header.jpg differ diff --git a/kadai3-2/misonog/testdata/test_download/header.jpg b/kadai3-2/misonog/testdata/test_download/header.jpg new file mode 100644 index 00000000..bcf63e98 Binary files /dev/null and b/kadai3-2/misonog/testdata/test_download/header.jpg differ diff --git a/kadai3-2/misonog/util.go b/kadai3-2/misonog/util.go new file mode 100644 index 00000000..dbaa0e3f --- /dev/null +++ b/kadai3-2/misonog/util.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "io" + "os" +) + +// mergeFiles function merege file after split download +func mergeFiles(procs int, filename, dirname, fullfilename string) error { + mergefile, err := os.Create(fullfilename) + if err != nil { + return err + } + defer mergefile.Close() + + var f string + for i := 0; i < procs; i++ { + f = fmt.Sprintf("%s/%s.%d.%d", dirname, filename, procs, i) + subfp, err := os.Open(f) + if err != nil { + return err + } + + if _, err := io.Copy(mergefile, subfp); err != nil { + return err + } + subfp.Close() + + if err := os.Remove(f); err != nil { + return err + } + } + + return nil +}