add --sync-from-height command-line option

This causes lightwalletd to discard cached blocks at the given height
and beyond. This in turn causes it to re-fetch those blocks from zcashd.
It's similar to --redownload, except that option discards all blocks
(equivalent to --sync-from-height 0, but the existing --redownload is
retained for compatibility).

This is mostly intended for testing. It's sometimes useful to force the
node to (re)sync some recent blocks, but redownloading all of them takes
around an hour.

Author: Larry Ruane

cmd/root.go

@@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{
 			GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"),
 			DataDir:             viper.GetString("data-dir"),
 			Redownload:          viper.GetBool("redownload"),
+			SyncFromHeight:      viper.GetInt("sync-from-height"),
 			PingEnable:          viper.GetBool("ping-very-insecure"),
 			Darkside:            viper.GetBool("darkside-very-insecure"),
 			DarksideTimeout:     viper.GetUint64("darkside-timeout"),
@@ -239,7 +240,11 @@ func startServer(opts *common.Options) error {
 		os.Stderr.WriteString(fmt.Sprintf("\n  ** Can't create db directory: %s\n\n", dbPath))
 		os.Exit(1)
 	}
-	cache := common.NewBlockCache(dbPath, chainName, saplingHeight, opts.Redownload)
+	syncFromHeight := opts.SyncFromHeight
+	if opts.Redownload {
+		syncFromHeight = 0
+	}
+	cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight)
 	if !opts.Darkside {
 		go common.BlockIngestor(cache, 0 /*loop forever*/)
 	} else {
@@ -325,6 +330,7 @@ func init() {
 	rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
 	rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production")
 	rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files")
+	rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height")
 	rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)")
 	rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing")
 	rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)")
@@ -356,6 +362,8 @@ func init() {
 	viper.SetDefault("gen-cert-very-insecure", false)
 	viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload"))
 	viper.SetDefault("redownload", false)
+	viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height"))
+	viper.SetDefault("sync-from-height", -1)
 	viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir"))
 	viper.SetDefault("data-dir", "/var/lib/lightwalletd")
 	viper.BindPFlag("ping-very-insecure", rootCmd.Flags().Lookup("ping-very-insecure"))

common/cache.go

@@ -191,7 +191,8 @@ func (c *BlockCache) Reset(startHeight int) {
 
 // NewBlockCache returns an instance of a block cache object.
 // (No locking here, we assume this is single-threaded.)
-func NewBlockCache(dbPath string, chainName string, startHeight int, redownload bool) *BlockCache {
+// syncFromHeight < 0 means latest (tip) height.
+func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHeight int) *BlockCache {
 	c := &BlockCache{}
 	c.firstBlock = startHeight
 	c.nextBlock = startHeight
@@ -208,18 +209,20 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, redownload
 	if err != nil {
 		Log.Fatal("open ", c.lengthsName, " failed: ", err)
 	}
-	if redownload {
-		if err := c.lengthsFile.Truncate(0); err != nil {
-			Log.Fatal("truncate lengths file failed: ", err)
-		}
-		if err := c.blocksFile.Truncate(0); err != nil {
-			Log.Fatal("truncate blocks file failed: ", err)
-		}
-	}
 	lengths, err := ioutil.ReadFile(c.lengthsName)
 	if err != nil {
 		Log.Fatal("read ", c.lengthsName, " failed: ", err)
 	}
+	// 4 bytes per lengths[] value (block length)
+	if syncFromHeight >= 0 {
+		if syncFromHeight < startHeight {
+			syncFromHeight = startHeight
+		}
+		if (syncFromHeight-startHeight)*4 < len(lengths) {
+			// discard the entries at and beyond (newer than) the specified height
+			lengths = lengths[:(syncFromHeight-startHeight)*4]
+		}
+	}
 
 	// The last entry in starts[] is where to write the next block.
 	var offset int64

common/cache_test.go

@@ -58,7 +58,7 @@ func TestCache(t *testing.T) {
 
 	// Pretend Sapling starts at 289460.
 	os.RemoveAll(unitTestPath)
-	cache = NewBlockCache(unitTestPath, unitTestChain, 289460, true)
+	cache = NewBlockCache(unitTestPath, unitTestChain, 289460, 0)
 
 	// Initially cache is empty.
 	if cache.GetLatestHeight() != -1 {
@@ -75,7 +75,7 @@ func TestCache(t *testing.T) {
 	fillCache(t)
 
 	// Simulate a restart to ensure the db files are read correctly.
-	cache = NewBlockCache(unitTestPath, unitTestChain, 289460, false)
+	cache = NewBlockCache(unitTestPath, unitTestChain, 289460, -1)
 
 	// Should still be 6 blocks.
 	if cache.nextBlock != 289466 {

common/common.go

@@ -42,6 +42,7 @@ type Options struct {
 	NoTLSVeryInsecure   bool   `json:"no_tls_very_insecure,omitempty"`
 	GenCertVeryInsecure bool   `json:"gen_cert_very_insecure,omitempty"`
 	Redownload          bool   `json:"redownload"`
+	SyncFromHeight      int    `json:"sync_from_height"`
 	DataDir             string `json:"data_dir"`
 	PingEnable          bool   `json:"ping_enable"`
 	Darkside            bool   `json:"darkside"`

common/common_test.go

@@ -62,7 +62,7 @@ func TestMain(m *testing.M) {
 		blockJSON, _ := json.Marshal(scan.Text())
 		blocks = append(blocks, blockJSON)
 	}
-	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true)
+	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
 
 	// Setup is done; run all tests.
 	exitcode := m.Run()
@@ -355,7 +355,7 @@ func TestBlockIngestor(t *testing.T) {
 	Time.Sleep = sleepStub
 	Time.Now = nowStub
 	os.RemoveAll(unitTestPath)
-	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, false)
+	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, -1)
 	BlockIngestor(testcache, 11)
 	if step != 19 {
 		t.Error("unexpected final step", step)
@@ -488,7 +488,7 @@ func TestGetBlockRange(t *testing.T) {
 	testT = t
 	RawRequest = getblockStub
 	os.RemoveAll(unitTestPath)
-	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true)
+	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
 	blockChan := make(chan *walletrpc.CompactBlock)
 	errChan := make(chan error)
 	go GetBlockRange(testcache, blockChan, errChan, 380640, 380642)
@@ -567,7 +567,7 @@ func TestGetBlockRangeReverse(t *testing.T) {
 	testT = t
 	RawRequest = getblockStubReverse
 	os.RemoveAll(unitTestPath)
-	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true)
+	testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
 	blockChan := make(chan *walletrpc.CompactBlock)
 	errChan := make(chan error)
 

frontend/frontend_test.go

@@ -36,7 +36,7 @@ const (
 
 func testsetup() (walletrpc.CompactTxStreamerServer, *common.BlockCache) {
 	os.RemoveAll(unitTestPath)
-	cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, true)
+	cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
 	lwd, err := NewLwdStreamer(cache, "main", false /* enablePing */)
 	if err != nil {
 		os.Stderr.WriteString(fmt.Sprint("NewLwdStreamer failed:", err))