Add state pruning to new CLI

Author: cffls

docs/cli/README.md

@@ -44,6 +44,10 @@
 
 - [```server```](./server.md)
 
+- [```snapshot```](./snapshot.md)
+
+- [```snapshot prune-state```](./snapshot_prune-state.md)
+
 - [```status```](./status.md)
 
 - [```version```](./version.md)

docs/cli/account_import.md

@@ -6,4 +6,4 @@ The ```account import``` command imports an account in Json format to the Bor da
 
 - ```datadir```: Path of the data directory to store information
 
-- ```keystore```: Path of the data directory to store information
+- ```keystore```: Path of the data directory to store keys

docs/cli/account_list.md

@@ -6,4 +6,4 @@ The `account list` command lists all the accounts in the Bor data directory.
 
 - ```datadir```: Path of the data directory to store information
 
-- ```keystore```: Path of the data directory to store information
+- ```keystore```: Path of the data directory to store keys

docs/cli/account_new.md

@@ -6,4 +6,4 @@ The `account new` command creates a new local account file on the Bor data direc
 
 - ```datadir```: Path of the data directory to store information
 
-- ```keystore```: Path of the data directory to store information
+- ```keystore```: Path of the data directory to store keys

docs/cli/server.md

@@ -32,6 +32,8 @@ The ```bor server``` command runs the Bor client.
 
 - ```bor.withoutheimdall```: Run without Heimdall service (for testing purpose) (default: false)
 
+- ```bor.heimdallgRPC```: Address of Heimdall gRPC service
+
 - ```ethstats```: Reporting URL of a ethstats service (nodename:secret@host:port)
 
 - ```gpo.blocks```: Number of recent blocks to check for gas prices (default: 20)
@@ -80,6 +82,8 @@ The ```bor server``` command runs the Bor client.
 
 - ```cache.preimages```: Enable recording the SHA3/keccak preimages of trie keys (default: false)
 
+- ```cache.triesinmemory```: Number of block states (tries) to keep in memory (default = 128) (default: 128)
+
 - ```txlookuplimit```: Number of recent blocks to maintain transactions index for (default: 2350000)
 
 ### JsonRPC Options

docs/cli/snapshot.md

@@ -0,0 +1,5 @@
+# snapshot
+
+The ```snapshot``` command groups snapshot related actions:
+
+- [```snapshot prune-state```](./snapshot_prune-state.md): Joins the local client to another remote peer.

docs/cli/snapshot_prune-state.md

@@ -0,0 +1,21 @@
+# Prune state
+
+The ```bor snapshot prune-state``` command will prune historical state data with the help of the state snapshot. All trie nodes and contract codes that do not belong to the specified	version state will be deleted from the database. After pruning, only two version states are available: genesis and the specific one.
+
+## Options
+
+- ```datadir```: Path of the data directory to store information
+
+- ```keystore```: Path of the data directory to store keys
+
+- ```datadir.ancient```: Path of the ancient data directory to store information
+
+- ```bloomfilter.size```: Size of the bloom filter (default: 2048)
+
+### Cache Options
+
+- ```cache```: Megabytes of memory allocated to internal caching (default: 1024)
+
+- ```cache.trie```: Percentage of cache memory allowance to use for trie caching (default: 25)
+
+- ```cache.trie.journal```: Path of the trie journal directory to store information (default: triecache)

internal/cli/command.go

@@ -189,6 +189,16 @@ func Commands() map[string]MarkDownCommandFactory {
 				Meta2: meta2,
 			}, nil
 		},
+		"snapshot": func() (MarkDownCommand, error) {
+			return &SnapshotCommand{
+				UI: ui,
+			}, nil
+		},
+		"snapshot prune-state": func() (MarkDownCommand, error) {
+			return &PruneStateCommand{
+				Meta: meta,
+			}, nil
+		},
 	}
 }
 
@@ -248,7 +258,7 @@ func (m *Meta) NewFlagSet(n string) *flagset.Flagset {
 	f.StringFlag(&flagset.StringFlag{
 		Name:  "keystore",
 		Value: &m.keyStoreDir,
-		Usage: "Path of the data directory to store information",
+		Usage: "Path of the data directory to store keys",
 	})
 
 	return f

internal/cli/removedb.go

@@ -24,9 +24,10 @@ type RemoveDBCommand struct {
 }
 
 const (
-	chaindataPath      string = "chaindata"
-	ancientPath        string = "ancient"
-	lightchaindataPath string = "lightchaindata"
+	chaindataPath        string = "chaindata"
+	ancientPath          string = "ancient"
+	trieCacheJournalPath string = "triecache"
+	lightchaindataPath   string = "lightchaindata"
 )
 
 // MarkDown implements cli.MarkDown interface

internal/cli/server/config.go

@@ -686,7 +686,7 @@ func (c *Config) loadChain() error {
 
 //nolint:gocognit
 func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (*ethconfig.Config, error) {
-	dbHandles, err := makeDatabaseHandles()
+	dbHandles, err := MakeDatabaseHandles()
 	if err != nil {
 		return nil, err
 	}
@@ -1075,7 +1075,7 @@ func (c *Config) Merge(cc ...*Config) error {
 	return nil
 }
 
-func makeDatabaseHandles() (int, error) {
+func MakeDatabaseHandles() (int, error) {
 	limit, err := fdlimit.Maximum()
 	if err != nil {
 		return -1, err

internal/cli/snapshot.go

@@ -0,0 +1,183 @@
+// Snapshot related commands
+
+package cli
+
+import (
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state/pruner"
+	"github.com/ethereum/go-ethereum/internal/cli/flagset"
+	"github.com/ethereum/go-ethereum/internal/cli/server"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+
+	"github.com/mitchellh/cli"
+)
+
+// SnapshotCommand is the command to group the snapshot commands
+type SnapshotCommand struct {
+	UI cli.Ui
+}
+
+// MarkDown implements cli.MarkDown interface
+func (a *SnapshotCommand) MarkDown() string {
+	items := []string{
+		"# snapshot",
+		"The ```snapshot``` command groups snapshot related actions:",
+		"- [```snapshot prune-state```](./snapshot_prune-state.md): Joins the local client to another remote peer.",
+	}
+
+	return strings.Join(items, "\n\n")
+}
+
+// Help implements the cli.Command interface
+func (c *SnapshotCommand) Help() string {
+	return `Usage: bor snapshot <subcommand>
+
+  This command groups snapshot related actions.
+
+  Prune the state trie:
+
+    $ bor snapshot prune-state`
+}
+
+// Synopsis implements the cli.Command interface
+func (c *SnapshotCommand) Synopsis() string {
+	return "Snapshot related commands"
+}
+
+// Run implements the cli.Command interface
+func (c *SnapshotCommand) Run(args []string) int {
+	return cli.RunResultHelp
+}
+
+type PruneStateCommand struct {
+	*Meta
+
+	datadirAncient   string
+	cache            uint64
+	cacheTrie        uint64
+	cacheTrieJournal string
+	bloomfilterSize  uint64
+}
+
+// MarkDown implements cli.MarkDown interface
+func (c *PruneStateCommand) MarkDown() string {
+	items := []string{
+		"# Prune state",
+		"The ```bor snapshot prune-state``` command will prune historical state data with the help of the state snapshot. All trie nodes and contract codes that do not belong to the specified	version state will be deleted from the database. After pruning, only two version states are available: genesis and the specific one.",
+		c.Flags().MarkDown(),
+	}
+
+	return strings.Join(items, "\n\n")
+}
+
+// Help implements the cli.Command interface
+func (c *PruneStateCommand) Help() string {
+	return `Usage: bor snapshot prune-state <datadir>
+
+  This command will prune state databases at the given datadir location` + c.Flags().Help()
+}
+
+// Synopsis implements the cli.Command interface
+func (c *PruneStateCommand) Synopsis() string {
+	return "Prune state databases"
+}
+
+// Flags: datadir, datadir.ancient, cache.trie.journal, bloomfilter.size
+func (c *PruneStateCommand) Flags() *flagset.Flagset {
+	flags := c.NewFlagSet("prune-state")
+
+	flags.StringFlag(&flagset.StringFlag{
+		Name:    "datadir.ancient",
+		Value:   &c.datadirAncient,
+		Usage:   "Path of the ancient data directory to store information",
+		Default: "",
+	})
+
+	flags.Uint64Flag(&flagset.Uint64Flag{
+		Name:    "cache",
+		Usage:   "Megabytes of memory allocated to internal caching",
+		Value:   &c.cache,
+		Default: 1024.0,
+		Group:   "Cache",
+	})
+
+	flags.Uint64Flag(&flagset.Uint64Flag{
+		Name:    "cache.trie",
+		Usage:   "Percentage of cache memory allowance to use for trie caching",
+		Value:   &c.cacheTrie,
+		Default: 25,
+		Group:   "Cache",
+	})
+
+	flags.StringFlag(&flagset.StringFlag{
+		Name:    "cache.trie.journal",
+		Value:   &c.cacheTrieJournal,
+		Usage:   "Path of the trie journal directory to store information",
+		Default: trieCacheJournalPath,
+		Group:   "Cache",
+	})
+
+	flags.Uint64Flag(&flagset.Uint64Flag{
+		Name:    "bloomfilter.size",
+		Value:   &c.bloomfilterSize,
+		Usage:   "Size of the bloom filter",
+		Default: 2048,
+	})
+
+	return flags
+}
+
+// Run implements the cli.Command interface
+func (c *PruneStateCommand) Run(args []string) int {
+	flags := c.Flags()
+
+	if err := flags.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	datadir := c.dataDir
+	if datadir == "" {
+		c.UI.Error("datadir is required")
+		return 1
+	}
+
+	// Create the node
+	node, err := node.New(&node.Config{
+		DataDir: datadir,
+	})
+
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	dbHandles, err := server.MakeDatabaseHandles()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false)
+
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	pruner, err := pruner.NewPruner(chaindb, node.ResolvePath(""), node.ResolvePath(c.cacheTrieJournal), c.bloomfilterSize)
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "err", err)
+		return 1
+	}
+
+	if err = pruner.Prune(common.Hash{}); err != nil {
+		log.Error("Failed to prune state", "err", err)
+		return 1
+	}
+
+	return 0
+}