Fix paths, warmup and index support

Vilsol · Nov 8, 2020 · 01644a7 · 01644a7
1 parent 20994d0
commit 01644a7
Show file tree

Hide file tree

Showing 10 changed files with 134 additions and 22 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -5,4 +5,6 @@
 
 !cmd/
 !server/
-!main.go
+!main.go
+
+!yeet
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -19,11 +19,6 @@ builds:
     goarm:
       - 6
       - 7
-      - 8
-    ignore:
-      - goos: linux
-        goarch: arm
-        goarm: 8
 
 archives:
   - replacements:

diff --git a/Dockerfile b/Dockerfile
@@ -1,3 +1,4 @@
 FROM scratch
 COPY yeet /
+EXPOSE 8080
 ENTRYPOINT ["/yeet"]
diff --git a/README.md b/README.md
@@ -1,5 +1,9 @@
 # Yeet
 
+![GitHub Workflow Status](https://img.shields.io/github/workflow/status/vilsol/yeet/build)
+![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/vilsol/yeet)
+![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/vilsol/yeet)
+
 ```
 Usage:
   yeet [command]
@@ -17,9 +21,11 @@ Flags:
       --expiry-time duration       Lifetime of a cache entry (default 1h0m0s)
   -h, --help                       help for yeet
       --host string                Hostname to bind the webserver (default "0.0.0.0")
+      --index-file string          The directory default index file (default "index.html")
       --log string                 The log level to output (default "info")
       --paths strings              Paths to serve on the webserver (default [./www])
       --port int                   Port to run the webserver on (default 8080)
+      --warmup                     Load all files into memory on startup
 
 Use "yeet [command] --help" for more information about a command.
 ```
@@ -28,4 +34,32 @@ Use "yeet [command] --help" for more information about a command.
 
 ```
 docker run -v /path/to/data:/www -p 8080:8080 ghcr.io/vilsol/yeet:latest
+```
+
+## Benchmarks (GOMAXPROCS=1)
+
+### Without cache expiry
+
+```
+BenchmarkServerGet1ReqPerConn                            8684911              1345 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet2ReqPerConn                           11419969              1046 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet10ReqPerConn                          15521960               763 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet10KReqPerConn                         17596045               690 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet1ReqPerConn10KClients                  8395502              1386 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet2ReqPerConn10KClients                 11061466              1053 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet10ReqPerConn10KClients                15229926               771 ns/op               0 B/op          0 allocs/op
+BenchmarkServerGet100ReqPerConn10KClients               16757032               699 ns/op               0 B/op          0 allocs/op
+```
+
+### With cache expiry
+
+```
+BenchmarkServerGet1ReqPerConnExpiry                      2419442              4780 ns/op            6176 B/op          2 allocs/op
+BenchmarkServerGet2ReqPerConnExpiry                      2912630              4074 ns/op            6176 B/op          2 allocs/op
+BenchmarkServerGet10ReqPerConnExpiry                     3479211              3468 ns/op            6176 B/op          2 allocs/op
+BenchmarkServerGet10KReqPerConnExpiry                    4027263              2952 ns/op            6176 B/op          2 allocs/op
+BenchmarkServerGet1ReqPerConn10KClientsExpiry            1645986              7244 ns/op            6411 B/op          2 allocs/op
+BenchmarkServerGet2ReqPerConn10KClientsExpiry            1818912              7026 ns/op            6177 B/op          2 allocs/op
+BenchmarkServerGet10ReqPerConn10KClientsExpiry           1960964              6217 ns/op            6177 B/op          2 allocs/op
+BenchmarkServerGet100ReqPerConn10KClientsExpiry          2766102              4114 ns/op            6223 B/op          2 allocs/op
 ```
diff --git a/cmd/root.go b/cmd/root.go
@@ -14,6 +14,7 @@ var rootCmd = &cobra.Command{
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		viper.SetConfigName("config")
 		viper.AddConfigPath(".")
+		viper.SetEnvPrefix("yeet")
 		viper.AutomaticEnv()
 
 		_ = viper.ReadInConfig()
@@ -56,24 +57,30 @@ func init() {
 	rootCmd.PersistentFlags().Int("port", 8080, "Port to run the webserver on")
 
 	rootCmd.PersistentFlags().StringSlice("paths", []string{"./www"}, "Paths to serve on the webserver")
+	rootCmd.PersistentFlags().Bool("warmup", false, "Load all files into memory on startup")
 
 	rootCmd.PersistentFlags().Bool("expiry", false, "Use cache expiry")
 	rootCmd.PersistentFlags().Duration("expiry-time", time.Minute*60, "Lifetime of a cache entry")
 	rootCmd.PersistentFlags().Duration("expiry-interval", time.Minute*10, "Port to run the webserver on")
 	rootCmd.PersistentFlags().Int("expiry-memory", 128, "Max memory usage in MB")
 	rootCmd.PersistentFlags().Int("expiry-shards", 64, "Cache shard count")
 
+	rootCmd.PersistentFlags().String("index-file", "index.html", "The directory default index file")
+
 	_ = viper.BindPFlag("log", rootCmd.PersistentFlags().Lookup("log"))
 	_ = viper.BindPFlag("colors", rootCmd.PersistentFlags().Lookup("colors"))
 
 	_ = viper.BindPFlag("host", rootCmd.PersistentFlags().Lookup("host"))
 	_ = viper.BindPFlag("port", rootCmd.PersistentFlags().Lookup("port"))
 
 	_ = viper.BindPFlag("paths", rootCmd.PersistentFlags().Lookup("paths"))
+	_ = viper.BindPFlag("warmup", rootCmd.PersistentFlags().Lookup("warmup"))
 
 	_ = viper.BindPFlag("expiry", rootCmd.PersistentFlags().Lookup("expiry"))
 	_ = viper.BindPFlag("expiry.time", rootCmd.PersistentFlags().Lookup("expiry-time"))
 	_ = viper.BindPFlag("expiry.interval", rootCmd.PersistentFlags().Lookup("expiry-interval"))
 	_ = viper.BindPFlag("expiry.memory", rootCmd.PersistentFlags().Lookup("expiry-memory"))
 	_ = viper.BindPFlag("expiry.shards", rootCmd.PersistentFlags().Lookup("expiry-shards"))
+
+	_ = viper.BindPFlag("index.file", rootCmd.PersistentFlags().Lookup("index-file"))
 }
diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/VictoriaMetrics/fastcache v1.5.7
 	github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156
+	github.com/davecgh/go-spew v1.1.1
 	github.com/pkg/errors v0.8.1
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1

diff --git a/server/cache.go b/server/cache.go
@@ -28,44 +28,89 @@ func (cache *CachedInstance) Reset() {
 	cache.GetExpiry = LoadCacheExpiry
 }
 
-func IndexCache(cache map[string]*CachedInstance) error {
+func IndexCache(cache map[string]*CachedInstance) (int64, error) {
+	totalSize := int64(0)
 	for _, dirPath := range viper.GetStringSlice("paths") {
 		cleanPath := path.Clean(dirPath)
-		if err := indexPath(cache, cleanPath); err != nil {
-			return errors.Wrap(err, "error indexing path "+cleanPath)
+		pathSize, err := indexPath(cache, cleanPath)
+		if err != nil {
+			return 0, errors.Wrap(err, "error indexing path "+cleanPath)
 		}
+		totalSize += pathSize
 	}
-	return nil
+	return totalSize, nil
 }
 
-func indexPath(cache map[string]*CachedInstance, dirPath string) error {
-	return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+func indexPath(cache map[string]*CachedInstance, dirPath string) (int64, error) {
+	trimmed := strings.Trim(dirPath, "/")
+	toRemove := len(strings.Split(trimmed, "/"))
+
+	if trimmed == "." || trimmed == "" {
+		toRemove = 0
+	}
+
+	totalSize := int64(0)
+
+	if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
+		filePath := path
+
 		if info.IsDir() {
-			return nil
+			indexFile := viper.GetString("index.file")
+			if indexFile != "" {
+				joined := filepath.Join(path, indexFile)
+				_, err := os.Stat(joined)
+				if err != nil && !os.IsNotExist(err) {
+					return err
+				} else if err != nil {
+					return nil
+				}
+				filePath = joined
+			} else {
+				return nil
+			}
 		}
 
+		absPath, _ := filepath.Abs(filePath)
+
 		cleanedPath := strings.ReplaceAll(filepath.Clean(path), "\\", "/")
-		cleanedPath = cleanedPath[len(dirPath)-1:]
+
+		// Remove the initial path
+		cleanedPath = strings.Join(strings.Split(cleanedPath, "/")[toRemove:], "/")
 
 		if !strings.HasPrefix(cleanedPath, "/") {
 			cleanedPath = "/" + cleanedPath
 		}
 
-		// TODO Optional storing in memory on index
-		absPath, _ := filepath.Abs(path)
-		cache[cleanedPath] = &CachedInstance{
+		log.Tracef("Indexed: %s -> %s", cleanedPath, absPath)
+
+		instance := &CachedInstance{
 			RelativePath: cleanedPath,
 			AbsolutePath: absPath,
 			Get:          LoadCache,
 			GetExpiry:    LoadCacheExpiry,
 		}
 
+		cache[cleanedPath] = instance
+
+		if viper.GetBool("warmup") {
+			if viper.GetBool("expiry") {
+				panic("expiry not supported if warmup is enabled")
+			}
+
+			instance.Get(instance)
+			totalSize += int64(len(instance.Data))
+		}
+
 		return nil
-	})
+	}); err != nil {
+		return 0, err
+	}
+
+	return totalSize, nil
 }
 
 // TODO Streaming

diff --git a/server/utils.go b/server/utils.go
@@ -0,0 +1,16 @@
+package server
+
+import "fmt"
+
+func ByteCountToHuman(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
+}
diff --git a/server/webserver.go b/server/webserver.go
@@ -14,13 +14,18 @@ type Webserver interface {
 func Run() error {
 	cache := make(map[string]*CachedInstance)
 
-	if err := IndexCache(cache); err != nil {
+	totalSize, err := IndexCache(cache)
+
+	if err != nil {
 		return err
 	}
 
-	log.Infof("Indexed %d files", len(cache))
+	if viper.GetBool("warmup") {
+		log.Infof("Indexed %d files with %s of memory usage", len(cache), ByteCountToHuman(totalSize))
+	} else {
+		log.Infof("Indexed %d files", len(cache))
+	}
 
-	var err error
 	var ws Webserver
 	if viper.GetBool("expiry.enabled") {
 		ws, err = GetExpiryWebserver(cache)

diff --git a/server/webserver_test.go b/server/webserver_test.go
@@ -21,7 +21,7 @@ func init() {
 
 	cache = make(map[string]*CachedInstance)
 
-	if err := IndexCache(cache); err != nil {
+	if _, err := IndexCache(cache); err != nil {
 		panic(err)
 	}
 }
@@ -30,6 +30,12 @@ func benchmarkServerGet(b *testing.B, clientsCount, requestsPerConn int, expiry
 	var handler fasthttp.RequestHandler
 
 	if expiry {
+		viper.Set("expiry", true)
+		viper.Set("expiry.shards", 64)
+		viper.Set("expiry.time", time.Minute*10)
+		viper.Set("expiry.interval", time.Minute*60)
+		viper.Set("expiry.memory", 128)
+
 		ws, err := GetExpiryWebserver(cache)
 
 		if err != nil {