Flatter TTL data without levelDB (#279)

Replaces levelDB for TTL values with a flat file.  
Also changes the behavior of the TTL data: very high (>2 billion) TTL values indicate a skipped output.  TTL 0 still means that it hasn't yet been spent.

accumulator/batchproof.go

@@ -222,7 +222,7 @@ func (bp *BatchProof) ToString() string {
 	}
 	s += fmt.Sprintf("\n%d proofs: ", len(bp.Proof))
 	for _, p := range bp.Proof {
-		s += fmt.Sprintf("%04x\t", p[:4])
+		s += fmt.Sprintf("%04x\t", p[:8])
 	}
 	s += "\n"
 	return s

accumulator/forest.go

@@ -710,7 +710,12 @@ func (f *Forest) ToString() string {
 	fh := f.rows
 	// tree rows should be 6 or less
 	if fh > 6 {
-		return "forest too big to print "
+		s := fmt.Sprintf("can't print %d leaves. roots:\n", f.numLeaves)
+		roots := f.getRoots()
+		for i, r := range roots {
+			s += fmt.Sprintf("\t%d %x\n", i, r.Mini())
+		}
+		return s
 	}
 
 	output := make([]string, (fh*2)+1)

bridgenode/config.go

@@ -18,8 +18,11 @@ The bridgenode server generates proofs and serves to the CSN node.
 OPTIONS:
   -net=mainnet                 configure whether to use mainnet. Optional.
   -net=regtest                 configure whether to use regtest. Optional.
+  -forest                      select forest type to use (ram, cow, cache, disk). 
+  Defaults to disk
   -net=signet                 configure whether to use signet. Optional.
   -forest                      select forest type to use (ram, cow, cache, disk). Defaults to disk
+
   -datadir="path/to/directory" set a custom DATADIR.
                                Defaults to the Bitcoin Core DATADIR path
   -datadir="path/to/directory" set a custom DATADIR.
@@ -42,14 +45,12 @@ var (
 		`Set a custom bridgenode datadir. Usage: "-bridgedir='path/to/directory"`)
 	forestTypeCmd = argCmd.String("forest", "disk",
 		`Set a forest type to use (cow, ram, disk, cache). Usage: "-forest=cow"`)
+	quitAfterCmd = argCmd.Int("quitafter", -1,
+		`quit generating proofs after the given block height. (meant for testing)`)
 	cowMaxCache = argCmd.Int("cowmaxcache", 4000,
 		`how much memory to use in MB for the copy-on-write forest`)
-	memTTLdb = argCmd.Bool("memttldb", false,
-		`keep a cache of the ttldb in memory.`)
-	allInMemTTLdb = argCmd.Bool("allttldbinmem", false,
-		`keeps the entire ttldb in memory. Uses up a lot of memory.`)
-	quitAtCmd = argCmd.Int("quitat", -1,
-		`quit generating proofs after the given block height. (meant for testing)`)
+	memTTL = argCmd.Bool("memttl", false,
+		`keep the ttls in memory instead of on disk. Uses lots of ram.`)
 	serve = argCmd.Bool("serve", false,
 		`immediately start server without building or checking proof data`)
 	noServeCmd = argCmd.Bool("noserve", false,
@@ -105,18 +106,18 @@ type utreeDir struct {
 	OffsetDir offsetDir
 	ProofDir  proofDir
 	ForestDir forestDir
-	UndoDir   undoDir
 	TtlDir    ttlDir
-	Ttldb     string
+	UndoDir   undoDir
 }
 
 // init an utreeDir with a selected basepath. Has all the names for the forest
 func initUtreeDir(basePath string) utreeDir {
 	offBase := filepath.Join(basePath, "offsetdata")
 	off := offsetDir{
-		base:                      offBase,
-		OffsetFile:                filepath.Join(offBase, "offsetfile.dat"),
-		lastIndexOffsetHeightFile: filepath.Join(offBase, "lastindexoffsetheightfile.dat"),
+		base:       offBase,
+		OffsetFile: filepath.Join(offBase, "offsetfile.dat"),
+		lastIndexOffsetHeightFile: filepath.Join(offBase,
+			"lastindexoffsetheightfile.dat"),
 	}
 
 	proofBase := filepath.Join(basePath, "proofdata")
@@ -130,54 +131,55 @@ func initUtreeDir(basePath string) utreeDir {
 	forestBase := filepath.Join(basePath, "forestdata")
 	cowDir := filepath.Join(forestBase, "cow")
 	forest := forestDir{
-		base:                            forestBase,
-		forestFile:                      filepath.Join(forestBase, "forestfile.dat"),
-		miscForestFile:                  filepath.Join(forestBase, "miscforestfile.dat"),
-		forestLastSyncedBlockHeightFile: filepath.Join(forestBase, "forestlastsyncedheight.dat"),
-		cowForestDir:                    cowDir,
-		cowForestCurFile:                filepath.Join(cowDir, "CURRENT"),
+		base:           forestBase,
+		forestFile:     filepath.Join(forestBase, "forestfile.dat"),
+		miscForestFile: filepath.Join(forestBase, "miscforestfile.dat"),
+		forestLastSyncedBlockHeightFile: filepath.Join(forestBase,
+			"forestlastsyncedheight.dat"),
+		cowForestDir:     cowDir,
+		cowForestCurFile: filepath.Join(cowDir, "CURRENT"),
 	}
-
-	undoBase := filepath.Join(basePath, "undoblockdata")
-	undo := undoDir{
-		base:       undoBase,
-		undoFile:   filepath.Join(undoBase, "undo.dat"),
-		offsetFile: filepath.Join(undoBase, "offset.dat"),
-	}
-
 	ttlBase := filepath.Join(basePath, "ttldata")
 	ttl := ttlDir{
 		base:       ttlBase,
 		ttlsetFile: filepath.Join(ttlBase, "ttldata.dat"),
 		OffsetFile: filepath.Join(ttlBase, "offsetfile.dat"),
 	}
-	ttldb := filepath.Join(basePath, "ttldb")
+	undoBase := filepath.Join(basePath, "undoblockdata")
+	undo := undoDir{
+		base:       undoBase,
+		undoFile:   filepath.Join(undoBase, "undo.dat"),
+		offsetFile: filepath.Join(undoBase, "offset.dat"),
+	}
 
 	return utreeDir{
 		OffsetDir: off,
 		ProofDir:  proof,
 		ForestDir: forest,
-		UndoDir:   undo,
 		TtlDir:    ttl,
-		Ttldb:     ttldb,
+		UndoDir:   undo,
 	}
 }
 
 // MakePaths makes the necessary paths for all files in a given network
 func makePaths(dir utreeDir) error {
 	err := os.MkdirAll(dir.OffsetDir.base, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("init makePaths error %s")
+		return fmt.Errorf("init makePaths error %s", err.Error())
 	}
 	err = os.MkdirAll(dir.ProofDir.base, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("init makePaths error %s")
+		return fmt.Errorf("init makePaths error %s", err.Error())
 	}
 	err = os.MkdirAll(dir.ForestDir.base, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("init makePaths error %s")
+		return fmt.Errorf("init makePaths error %s", err.Error())
 	}
 	err = os.MkdirAll(dir.ForestDir.cowForestDir, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("init makePaths error %s", err.Error())
+	}
+	err = os.MkdirAll(dir.TtlDir.base, os.ModePerm)
 	if err != nil {
 		return fmt.Errorf("init makePaths error %s")
 	}
@@ -225,16 +227,13 @@ type Config struct {
 	forestType forestType
 
 	// quitAfter syncing to this block height
-	quitAt int
+	quitAfter int32
 
 	// how much cache to allow for cowforest
 	cowMaxCache int
 
-	// keep the ttldb in memory
-	memTTLdb bool
-
-	// only keep the ttldb in memory
-	allInMemTTLdb bool
+	// keep ttls in memory
+	memTTL bool
 
 	// just immidiately start serving what you have on disk
 	serve bool
@@ -318,13 +317,7 @@ func Parse(args []string) (*Config, error) {
 	cfg.MemProf = *memProfCmd
 	cfg.TraceProf = *traceCmd
 	cfg.ProfServer = *profServerCmd
-	cfg.memTTLdb = *memTTLdb
-
-	// If allInMemTTLdb flag was given, the ttldb has to be kept in ram
-	if *allInMemTTLdb {
-		cfg.memTTLdb = true
-	}
-	cfg.allInMemTTLdb = *allInMemTTLdb
+	cfg.memTTL = *memTTL
 
 	switch *forestTypeCmd {
 	case "disk":
@@ -340,7 +333,7 @@ func Parse(args []string) (*Config, error) {
 		return nil, errWrongForestType(*forestTypeCmd)
 	}
 
-	cfg.quitAt = *quitAtCmd
+	cfg.quitAfter = int32(*quitAfterCmd)
 	cfg.noServe = *noServeCmd
 	cfg.serve = *serve