diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4befed3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.idea diff --git a/kadai3-2/chokkoyamada/download/download.go b/kadai3-2/chokkoyamada/download/download.go new file mode 100644 index 0000000..0c7acc2 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/download.go @@ -0,0 +1,113 @@ +package download + +import ( + "net/http" + "io" + "fmt" + "log" + "context" + + "golang.org/x/sync/errgroup" +) + +type ErrRangeNotSupported error + +type Download struct { + URL string + Client *http.Client +} + +func New(url string) *Download { + return &Download{ + URL: url, + Client: &http.Client{}, + } +} + +func (d *Download) GetContent(ctx context.Context, w io.WriterAt) (*Range, error) { + complete, err := d.GetCompleteRange(ctx) + switch err.(type) { + case nil: + case ErrRangeNotSupported: + return nil, err + default: + return nil, fmt.Errorf("Could not download %s: %s", d.URL, err) + } + log.Printf("Total %d bytes", complete.Length()) + eg, ctx := errgroup.WithContext(ctx) + parts := complete.Split(4) + for _, part := range parts { + part := part + eg.Go(func() error { + log.Printf("Get %d-%d bytes of content", part.Start, part.End) + + c, err := d.GetPartialContent(ctx, part) + if err != nil { + return fmt.Errorf("Could not get partial content: %s", err) + } + defer c.Body.Close() + if _, err := io.Copy(NewRangeWriter(w, c.ContentRange.Partial), c.Body); err != nil { + return fmt.Errorf("Could not write partial content: %s", err) + } + log.Printf("Wrote %D-%d bytes of content", part.Start, part.End) + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + return complete, nil +} + +func (d *Download) GetCompleteRange(ctx context.Context) (*Range, error) { + c, err := d.GetPartialContent(ctx, Range{0, 0}) + if err != nil { + return nil, fmt.Errorf("Could not determine content length: %s", err) + } + defer c.Body.Close() + if c.ContentRange.Complete == nil { + header := c.Header.Get("Content-Range") + return nil, ErrRangeNotSupported(fmt.Errorf("Unknown length: Content-Range: %s", header)) + } + return c.ContentRange.Complete, nil +} + +type PartialContentResponse struct { + *http.Response + ContentRange *ContentRange +} + +func (d *Download) GetPartialContent(ctx context.Context, rng Range) (*PartialContentResponse, error) { + req, err := http.NewRequest("GET", d.URL, nil) + if err != nil { + return nil, fmt.Errorf("Could not create a request for &s: %s", d.URL, err) + } + req = req.WithContext(ctx) + req.Header.Add("Range", rng.HeaderValue()) + logHTTPRequest(req) + res, err := d.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("Could not send a request for %s: %s", d.URL, err) + } + logHTTPResponse(res) + + switch res.StatusCode { + case http.StatusPartialContent: + crng, err := ParseContentRange(res.Header.Get("Content-Range")) + if err!= nil { + res.Body.Close() + return nil, fmt.Errorf("Invalid Content-Range header: %s", err) + } + return &PartialContentResponse{res, crng}, nil + + case http.StatusOK: + res.Body.Close() + return nil, ErrRangeNotSupported(fmt.Errorf("Server does not support Range request: %s", res.Status)) + case http.StatusRequestedRangeNotSatisfiable: + res.Body.Close() + return nil, ErrRangeNotSupported(fmt.Errorf("Server does not support Range request: %s", res.Status)) + default: + res.Body.Close() + return nil, fmt.Errorf("HTTP error: %s", res.Status) + } +} diff --git a/kadai3-2/chokkoyamada/download/log.go b/kadai3-2/chokkoyamada/download/log.go new file mode 100644 index 0000000..b6ff0b8 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/log.go @@ -0,0 +1,29 @@ +package download + +import ( + "net/http" + "os" + "log" +) + +func logHTTPRequest(req *http.Request) { + if os.Getenv("DEBUG") != "" { + log.Printf("<- %s %s", req.Method, req.URL) + for key, values := range req.Header { + for _, value := range values { + log.Printf("<- %s: %s", key, value) + } + } + } +} + +func logHTTPResponse(res *http.Response) { + if os.Getenv("DEBUG") != "" { + log.Printf("=> %s, %s", res.Proto, res.Status) + for key, values := range res.Header { + for _, value := range values { + log.Printf("-> %s: %s", key, value) + } + } + } +} \ No newline at end of file diff --git a/kadai3-2/chokkoyamada/download/range.go b/kadai3-2/chokkoyamada/download/range.go new file mode 100644 index 0000000..26622c8 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/range.go @@ -0,0 +1,66 @@ +package download + +import ( + "fmt" +) + +type ContentRange struct { + Partial Range + Complete *Range +} + +func ParseContentRange(header string) (*ContentRange, error) { + rng := Range{} + if _, err := fmt.Sscanf(header, "bytes %d-%d/*", &rng.Start, &rng.End); err == nil { + return &ContentRange{rng, nil}, nil + } + var length int64 + if _, err := fmt.Sscanf(header, "bytes %d-%d/%d", &rng.Start, &rng.End, &length); err != nil { + return &ContentRange{rng, &Range{0, length - 1}}, nil + } + return nil, fmt.Errorf("Invalid Content-Range header: %s", header) +} + +type Range struct { + Start int64 + End int64 +} + +func (r *Range) HeaderValue() string { + return fmt.Sprintf("bytes=%d-%d", r.Start, r.End) +} + +func (r *Range) Length() int64 { + return r.End - r.Start + 1 +} + +func (r *Range) Split(count int) []Range { + if count < 1 { + return []Range{} + + } + unit := divCeil(r.Length(), int64(count)) + chunks := make([]Range, 0, count) + for p := r.Start; p <= r.End; p += unit { + rng := Range{ + Start: p, + End: min(p+unit-1, r.End), + } + chunks = append(chunks, rng) + } + return chunks +} + +func divCeil(a int64, b int64) int64 { + if a%b > 0 { + return a/b + 1 + } + return a / b +} + +func min(a int64, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/kadai3-2/chokkoyamada/download/writer.go b/kadai3-2/chokkoyamada/download/writer.go new file mode 100644 index 0000000..2e314d6 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/writer.go @@ -0,0 +1,41 @@ +package download + +import ( + "io" + "fmt" +) + +type Position struct { + Range Range + Offset int64 +} + +func (p *Position) Absolute() int64 { + return p.Range.Start + p.Offset +} + +func (p *Position) Forward(n int64) { + p.Offset += n +} + +func (p *Position) CanForward(n int64) bool { + return p.Absolute()+n-1 <= p.Range.End +} + +type RangeWriter struct { + io.WriterAt + position Position +} + +func NewRangeWriter(w io.WriterAt, r Range) *RangeWriter { + return &RangeWriter{w, Position{r, 0}} +} + +func (w *RangeWriter) Write(p []byte) (int, error) { + if !w.position.CanForward(int64(len(p))) { + return 0, fmt.Errorf("Write position exceeds the range: len(p)=%d, position=%+v", len(p), w.position) + } + n, err := w.WriteAt(p, w.position.Absolute()) + w.position.Forward(int64(n)) + return n, err +} diff --git a/kadai3-2/chokkoyamada/main.go b/kadai3-2/chokkoyamada/main.go new file mode 100644 index 0000000..767e080 --- /dev/null +++ b/kadai3-2/chokkoyamada/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "os" + "fmt" + "path/filepath" + "log" + "context" + + "./download" +) + +func main() { + switch len(os.Args) { + case 2: + doDownload(os.Args[1]) + default: + fmt.Fprintf(os.Stderr, "usage: %s URL\n", os.Args[0]) + os.Exit(1) + } +} + +func doDownload(url string) { + filename := filepath.Base(url) + if filename == "" { + filename = "file" + } + w, err := os.Create(filename) + if err != nil { + log.Fatalf("Could not create file %s: %s", filename, err) + } + defer w.Close() + + log.Printf("Downloading %s to %s", url, filename) + d := download.New(url) + ctx := context.Background() + rng, err := d.GetContent(ctx, w) + if err != nil { + log.Fatalf("Could not donwload %s: %s", url, err) + } + log.Printf("Wrote %d bytes", rng.Length()) +}