From 01644a76a93199208854a4083d257239cfaf102a Mon Sep 17 00:00:00 2001 From: Vilsol Date: Sun, 8 Nov 2020 08:00:22 +0200 Subject: [PATCH] Fix paths, warmup and index support --- .dockerignore | 4 ++- .goreleaser.yml | 5 --- Dockerfile | 1 + README.md | 34 ++++++++++++++++++++ cmd/root.go | 7 ++++ go.mod | 1 + server/cache.go | 69 +++++++++++++++++++++++++++++++++------- server/utils.go | 16 ++++++++++ server/webserver.go | 11 +++++-- server/webserver_test.go | 8 ++++- 10 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 server/utils.go diff --git a/.dockerignore b/.dockerignore index b2b64ca..15bed5c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,6 @@ !cmd/ !server/ -!main.go \ No newline at end of file +!main.go + +!yeet \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 432e8cb..873364d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -19,11 +19,6 @@ builds: goarm: - 6 - 7 - - 8 - ignore: - - goos: linux - goarch: arm - goarm: 8 archives: - replacements: diff --git a/Dockerfile b/Dockerfile index 69fd8f5..3191d1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ FROM scratch COPY yeet / +EXPOSE 8080 ENTRYPOINT ["/yeet"] \ No newline at end of file diff --git a/README.md b/README.md index 2b1539b..303faea 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Yeet +![GitHub Workflow Status](https://img.shields.io/github/workflow/status/vilsol/yeet/build) +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/vilsol/yeet) +![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/vilsol/yeet) + ``` Usage: yeet [command] @@ -17,9 +21,11 @@ Flags: --expiry-time duration Lifetime of a cache entry (default 1h0m0s) -h, --help help for yeet --host string Hostname to bind the webserver (default "0.0.0.0") + --index-file string The directory default index file (default "index.html") --log string The log level to output (default "info") --paths strings Paths to serve on the webserver (default [./www]) --port int Port to run the webserver on (default 8080) + --warmup Load all files into memory on startup Use "yeet [command] --help" for more information about a command. ``` @@ -28,4 +34,32 @@ Use "yeet [command] --help" for more information about a command. ``` docker run -v /path/to/data:/www -p 8080:8080 ghcr.io/vilsol/yeet:latest +``` + +## Benchmarks (GOMAXPROCS=1) + +### Without cache expiry + +``` +BenchmarkServerGet1ReqPerConn 8684911 1345 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn 11419969 1046 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn 15521960 763 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10KReqPerConn 17596045 690 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet1ReqPerConn10KClients 8395502 1386 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn10KClients 11061466 1053 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn10KClients 15229926 771 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet100ReqPerConn10KClients 16757032 699 ns/op 0 B/op 0 allocs/op +``` + +### With cache expiry + +``` +BenchmarkServerGet1ReqPerConnExpiry 2419442 4780 ns/op 6176 B/op 2 allocs/op +BenchmarkServerGet2ReqPerConnExpiry 2912630 4074 ns/op 6176 B/op 2 allocs/op +BenchmarkServerGet10ReqPerConnExpiry 3479211 3468 ns/op 6176 B/op 2 allocs/op +BenchmarkServerGet10KReqPerConnExpiry 4027263 2952 ns/op 6176 B/op 2 allocs/op +BenchmarkServerGet1ReqPerConn10KClientsExpiry 1645986 7244 ns/op 6411 B/op 2 allocs/op +BenchmarkServerGet2ReqPerConn10KClientsExpiry 1818912 7026 ns/op 6177 B/op 2 allocs/op +BenchmarkServerGet10ReqPerConn10KClientsExpiry 1960964 6217 ns/op 6177 B/op 2 allocs/op +BenchmarkServerGet100ReqPerConn10KClientsExpiry 2766102 4114 ns/op 6223 B/op 2 allocs/op ``` \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index e002891..1d51e32 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -14,6 +14,7 @@ var rootCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { viper.SetConfigName("config") viper.AddConfigPath(".") + viper.SetEnvPrefix("yeet") viper.AutomaticEnv() _ = viper.ReadInConfig() @@ -56,6 +57,7 @@ func init() { rootCmd.PersistentFlags().Int("port", 8080, "Port to run the webserver on") rootCmd.PersistentFlags().StringSlice("paths", []string{"./www"}, "Paths to serve on the webserver") + rootCmd.PersistentFlags().Bool("warmup", false, "Load all files into memory on startup") rootCmd.PersistentFlags().Bool("expiry", false, "Use cache expiry") rootCmd.PersistentFlags().Duration("expiry-time", time.Minute*60, "Lifetime of a cache entry") @@ -63,6 +65,8 @@ func init() { rootCmd.PersistentFlags().Int("expiry-memory", 128, "Max memory usage in MB") rootCmd.PersistentFlags().Int("expiry-shards", 64, "Cache shard count") + rootCmd.PersistentFlags().String("index-file", "index.html", "The directory default index file") + _ = viper.BindPFlag("log", rootCmd.PersistentFlags().Lookup("log")) _ = viper.BindPFlag("colors", rootCmd.PersistentFlags().Lookup("colors")) @@ -70,10 +74,13 @@ func init() { _ = viper.BindPFlag("port", rootCmd.PersistentFlags().Lookup("port")) _ = viper.BindPFlag("paths", rootCmd.PersistentFlags().Lookup("paths")) + _ = viper.BindPFlag("warmup", rootCmd.PersistentFlags().Lookup("warmup")) _ = viper.BindPFlag("expiry", rootCmd.PersistentFlags().Lookup("expiry")) _ = viper.BindPFlag("expiry.time", rootCmd.PersistentFlags().Lookup("expiry-time")) _ = viper.BindPFlag("expiry.interval", rootCmd.PersistentFlags().Lookup("expiry-interval")) _ = viper.BindPFlag("expiry.memory", rootCmd.PersistentFlags().Lookup("expiry-memory")) _ = viper.BindPFlag("expiry.shards", rootCmd.PersistentFlags().Lookup("expiry-shards")) + + _ = viper.BindPFlag("index.file", rootCmd.PersistentFlags().Lookup("index-file")) } diff --git a/go.mod b/go.mod index ba02b58..56d4ab1 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/VictoriaMetrics/fastcache v1.5.7 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 + github.com/davecgh/go-spew v1.1.1 github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.7.0 github.com/spf13/cobra v1.1.1 diff --git a/server/cache.go b/server/cache.go index 6462983..c507855 100644 --- a/server/cache.go +++ b/server/cache.go @@ -28,44 +28,89 @@ func (cache *CachedInstance) Reset() { cache.GetExpiry = LoadCacheExpiry } -func IndexCache(cache map[string]*CachedInstance) error { +func IndexCache(cache map[string]*CachedInstance) (int64, error) { + totalSize := int64(0) for _, dirPath := range viper.GetStringSlice("paths") { cleanPath := path.Clean(dirPath) - if err := indexPath(cache, cleanPath); err != nil { - return errors.Wrap(err, "error indexing path "+cleanPath) + pathSize, err := indexPath(cache, cleanPath) + if err != nil { + return 0, errors.Wrap(err, "error indexing path "+cleanPath) } + totalSize += pathSize } - return nil + return totalSize, nil } -func indexPath(cache map[string]*CachedInstance, dirPath string) error { - return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { +func indexPath(cache map[string]*CachedInstance, dirPath string) (int64, error) { + trimmed := strings.Trim(dirPath, "/") + toRemove := len(strings.Split(trimmed, "/")) + + if trimmed == "." || trimmed == "" { + toRemove = 0 + } + + totalSize := int64(0) + + if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } + filePath := path + if info.IsDir() { - return nil + indexFile := viper.GetString("index.file") + if indexFile != "" { + joined := filepath.Join(path, indexFile) + _, err := os.Stat(joined) + if err != nil && !os.IsNotExist(err) { + return err + } else if err != nil { + return nil + } + filePath = joined + } else { + return nil + } } + absPath, _ := filepath.Abs(filePath) + cleanedPath := strings.ReplaceAll(filepath.Clean(path), "\\", "/") - cleanedPath = cleanedPath[len(dirPath)-1:] + + // Remove the initial path + cleanedPath = strings.Join(strings.Split(cleanedPath, "/")[toRemove:], "/") if !strings.HasPrefix(cleanedPath, "/") { cleanedPath = "/" + cleanedPath } - // TODO Optional storing in memory on index - absPath, _ := filepath.Abs(path) - cache[cleanedPath] = &CachedInstance{ + log.Tracef("Indexed: %s -> %s", cleanedPath, absPath) + + instance := &CachedInstance{ RelativePath: cleanedPath, AbsolutePath: absPath, Get: LoadCache, GetExpiry: LoadCacheExpiry, } + cache[cleanedPath] = instance + + if viper.GetBool("warmup") { + if viper.GetBool("expiry") { + panic("expiry not supported if warmup is enabled") + } + + instance.Get(instance) + totalSize += int64(len(instance.Data)) + } + return nil - }) + }); err != nil { + return 0, err + } + + return totalSize, nil } // TODO Streaming diff --git a/server/utils.go b/server/utils.go new file mode 100644 index 0000000..d736289 --- /dev/null +++ b/server/utils.go @@ -0,0 +1,16 @@ +package server + +import "fmt" + +func ByteCountToHuman(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) +} diff --git a/server/webserver.go b/server/webserver.go index b4fc756..9b1c73a 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -14,13 +14,18 @@ type Webserver interface { func Run() error { cache := make(map[string]*CachedInstance) - if err := IndexCache(cache); err != nil { + totalSize, err := IndexCache(cache) + + if err != nil { return err } - log.Infof("Indexed %d files", len(cache)) + if viper.GetBool("warmup") { + log.Infof("Indexed %d files with %s of memory usage", len(cache), ByteCountToHuman(totalSize)) + } else { + log.Infof("Indexed %d files", len(cache)) + } - var err error var ws Webserver if viper.GetBool("expiry.enabled") { ws, err = GetExpiryWebserver(cache) diff --git a/server/webserver_test.go b/server/webserver_test.go index da49372..2d6fad4 100644 --- a/server/webserver_test.go +++ b/server/webserver_test.go @@ -21,7 +21,7 @@ func init() { cache = make(map[string]*CachedInstance) - if err := IndexCache(cache); err != nil { + if _, err := IndexCache(cache); err != nil { panic(err) } } @@ -30,6 +30,12 @@ func benchmarkServerGet(b *testing.B, clientsCount, requestsPerConn int, expiry var handler fasthttp.RequestHandler if expiry { + viper.Set("expiry", true) + viper.Set("expiry.shards", 64) + viper.Set("expiry.time", time.Minute*10) + viper.Set("expiry.interval", time.Minute*60) + viper.Set("expiry.memory", 128) + ws, err := GetExpiryWebserver(cache) if err != nil {