Kadai3 2 n kumaya

kadai3-2/nKumaya/README.md

@@ -0,0 +1,10 @@
+# 課題3-2 分割ダウンロードを行う
+
+## 分割ダウンロードの説明
+
+```
+go run main.go [ターゲットURL]
+```
+対象URLをCPU数に応じて分割ダウンロードを行う。
+対象URLのステータスコードが200以外のときはエラーを返す。
+

kadai3-2/nKumaya/kget/kget.go

@@ -0,0 +1,158 @@
+package kget
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+)
+
+type Client struct {
+	URL                      *url.URL
+	HTTPClient               *http.Client
+	ContentLength, RangeSize int
+	Filename                 string
+}
+
+func NewClient(urlString string) (*Client, error) {
+	var err error
+	client := &Client{}
+	client.URL, err = url.Parse(urlString)
+	splitedPath := strings.Split(client.URL.Path, "/")
+	client.Filename = splitedPath[len(splitedPath)-1]
+	if err != nil {
+		return nil, err
+	}
+	client.HTTPClient = &http.Client{}
+
+	if err = client.setHeadContent(); err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// レスポンスヘッダに Accept-Ranges が含まれている場合は分割, ない場合は分割しない
+func (c *Client) setHeadContent() error {
+	req, err := http.NewRequest("HEAD", c.URL.String(), nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	c.ContentLength, err = strconv.Atoi(res.Header["Content-Length"][0])
+	if err != nil {
+		return err
+	}
+	if strings.Contains("bytes", res.Header["Accept-Ranges"][0]) {
+		// CPU数に応じて並行処理を行う
+		c.RangeSize = runtime.NumCPU()
+	} else {
+		c.RangeSize = 1
+	}
+	return nil
+}
+
+func (c *Client) newRequest(ctx context.Context, index int) (*http.Request, error) {
+	req, err := http.NewRequest("GET", c.URL.String(), nil)
+	if err != nil {
+		return req, nil
+	}
+
+	req = req.WithContext(ctx)
+	req.Header.Add("User-Agent", "kget")
+	var chank int
+	chank = c.ContentLength / c.RangeSize
+	low := chank * index
+	high := chank*(index+1) - 1
+	if index+1 == c.RangeSize && high+1 != c.ContentLength {
+		high = c.ContentLength - 1
+	}
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high))
+	return req, nil
+}
+
+func (c *Client) save(ctx context.Context, index int) error {
+	req, err := c.newRequest(ctx, index)
+	if err != nil {
+		return err
+	}
+	file, err := os.Create(c.Filename + "_" + strconv.Itoa(index))
+	if err != nil {
+		return err
+	}
+	res, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(file, res.Body)
+	if err != nil {
+		return err
+	}
+	file.Close()
+	return nil
+}
+
+func (c *Client) merge(ctx context.Context) error {
+	file, err := os.Create(c.Filename)
+	if err != nil {
+		return err
+	}
+	for i := 0; i < c.RangeSize; i++ {
+		subFileName := c.Filename + "_" + strconv.Itoa(i)
+		subFile, err := os.Open(subFileName)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(file, subFile)
+		subFile.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) Download(ctx context.Context) error {
+	eg := errgroup.Group{}
+	for i := 0; i < c.RangeSize; i++ {
+		i := i
+		eg.Go(func() error {
+			return c.save(ctx, i)
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		c.DeleteFiles()
+		return err
+	}
+	err := c.merge(ctx)
+	c.DeleteFiles()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) DeleteFiles() error {
+	for i := 0; i < c.RangeSize; i++ {
+		subFileName := c.Filename + "_" + strconv.Itoa(i)
+		if fi, _ := os.Stat(subFileName); fi != nil {
+			if err := os.Remove(subFileName); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

kadai3-2/nKumaya/main.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+
+	"github.com/dojo3/kadai3-2/nKumaya/kget"
+)
+
+const (
+	ExitCodeOK = iota
+	ExitCodeParseFlagError
+	ExitCodeInvalidUrlError
+	ExitCodeCreateHTTPClient
+	ExitCodeErrorDownload
+	ExitCodeErrorCansel
+)
+
+type CLI struct {
+	outStream, errStream io.Writer
+}
+
+func (c *CLI) Run(args []string) int {
+	flags := flag.NewFlagSet("kget", flag.ContinueOnError)
+	if err := flags.Parse(args[1:]); err != nil {
+		return ExitCodeParseFlagError
+	}
+	url := args[1]
+	fmt.Fprintln(c.outStream, "Checking now", url)
+	response, err := http.Get(url)
+	if response.StatusCode != 200 {
+		return ExitCodeInvalidUrlError
+	}
+	client, err := kget.NewClient(url)
+	if err != nil {
+		return ExitCodeCreateHTTPClient
+	}
+	bc := context.Background()
+	ctx, cancel := context.WithCancel(bc)
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt)
+	defer func() {
+		signal.Stop(ch)
+		cancel()
+	}()
+	fmt.Fprintln(c.outStream, "Download start", url)
+	go func() error {
+		select {
+		case <-ch:
+			cancel()
+			return nil
+		case <-ctx.Done():
+			if err = client.DeleteFiles(); err != nil {
+				return err
+			}
+			return nil
+		}
+	}()
+	err = client.Download(ctx)
+	if err != nil {
+		return ExitCodeErrorDownload
+	}
+	fmt.Fprintln(c.outStream, "Complite")
+	return ExitCodeOK
+}
+
+func main() {
+	cli := &CLI{os.Stdout, os.Stderr}
+	os.Exit(cli.Run(os.Args))
+}