Merge pull request #254 from kcalvinalvin/2025-02-06-dont-allow-0-or-…

…less-utreexoindexmaxmemory

main: put a minimum of 250mib for utreexoproofmaxmemory

blockchain/indexers/flatutreexoproofindex.go

@@ -956,7 +956,6 @@ func loadFlatFileState(dataDir, name string) (*FlatFileState, error) {
 
 // NewFlatUtreexoProofIndex returns a new instance of an indexer that is used to create a flat utreexo proof index.
 // The passed in maxMemoryUsage should be in bytes and it determines how much memory the proof index will use up.
-// A maxMemoryUsage of 0 will keep all the elements on disk and a negative maxMemoryUsage will keep all the elements in memory.
 //
 // It implements the Indexer interface which plugs into the IndexManager that in
 // turn is used by the blockchain package.  This allows the index to be

blockchain/indexers/utreexobackend.go

@@ -81,20 +81,6 @@ func (us *UtreexoState) flush(bestHash *chainhash.Hash) error {
 		return err
 	}
 
-	// If we're keeping everything in memory, make sure to remove everything from the
-	// current database before flushing as the keys deleted in memory may still exist
-	// on disk.
-	if us.config.MaxMemoryUsage < 0 {
-		iter := us.utreexoStateDB.NewIterator(nil, nil)
-		for iter.Next() {
-			err := ldbTx.Delete(iter.Key(), nil)
-			if err != nil {
-				ldbTx.Discard()
-				return err
-			}
-		}
-	}
-
 	// Write the best block hash and the numleaves for the utreexo state.
 	err = dbWriteUtreexoStateConsistency(ldbTx, bestHash, us.state.GetNumLeaves())
 	if err != nil {
@@ -682,80 +668,34 @@ func InitUtreexoState(cfg *UtreexoConfig, chain *blockchain.BlockChain,
 	}
 	p.NumLeaves = numLeaves
 
-	var flush func(ldbTx *leveldb.Transaction) error
-	var isFlushNeeded func() bool
-	if cfg.MaxMemoryUsage >= 0 {
-		p.Nodes = nodesDB
-		p.CachedLeaves = cachedLeavesDB
-		flush = func(ldbTx *leveldb.Transaction) error {
-			nodesUsed, nodesCapacity := nodesDB.UsageStats()
-			log.Debugf("Utreexo index nodesDB cache usage: %d/%d (%v%%)\n",
-				nodesUsed, nodesCapacity,
-				float64(nodesUsed)/float64(nodesCapacity))
-
-			cachedLeavesUsed, cachedLeavesCapacity := cachedLeavesDB.UsageStats()
-			log.Debugf("Utreexo index cachedLeavesDB cache usage: %d/%d (%v%%)\n",
-				cachedLeavesUsed, cachedLeavesCapacity,
-				float64(cachedLeavesUsed)/float64(cachedLeavesCapacity))
-
-			err = nodesDB.Flush(ldbTx)
-			if err != nil {
-				return err
-			}
-			err = cachedLeavesDB.Flush(ldbTx)
-			if err != nil {
-				return err
-			}
+	p.Nodes = nodesDB
+	p.CachedLeaves = cachedLeavesDB
+	flush := func(ldbTx *leveldb.Transaction) error {
+		nodesUsed, nodesCapacity := nodesDB.UsageStats()
+		log.Debugf("Utreexo index nodesDB cache usage: %d/%d (%v%%)\n",
+			nodesUsed, nodesCapacity,
+			float64(nodesUsed)/float64(nodesCapacity))
 
-			return nil
-		}
-		isFlushNeeded = func() bool {
-			nodesNeedsFlush := nodesDB.IsFlushNeeded()
-			leavesNeedsFlush := cachedLeavesDB.IsFlushNeeded()
-			return nodesNeedsFlush || leavesNeedsFlush
-		}
-	} else {
-		log.Infof("loading the utreexo state from disk...")
-		err = nodesDB.ForEach(func(k uint64, v utreexo.Leaf) error {
-			p.Nodes.Put(k, v)
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
+		cachedLeavesUsed, cachedLeavesCapacity := cachedLeavesDB.UsageStats()
+		log.Debugf("Utreexo index cachedLeavesDB cache usage: %d/%d (%v%%)\n",
+			cachedLeavesUsed, cachedLeavesCapacity,
+			float64(cachedLeavesUsed)/float64(cachedLeavesCapacity))
 
-		err = cachedLeavesDB.ForEach(func(k utreexo.Hash, v uint64) error {
-			p.CachedLeaves.Put(k, v)
-			return nil
-		})
+		err = nodesDB.Flush(ldbTx)
 		if err != nil {
-			return nil, err
+			return err
 		}
-
-		log.Infof("Finished loading the utreexo state from disk.")
-
-		flush = func(ldbTx *leveldb.Transaction) error {
-			err = p.Nodes.ForEach(func(k uint64, v utreexo.Leaf) error {
-				return blockchain.NodesBackendPut(ldbTx, k, v)
-			})
-			if err != nil {
-				return err
-			}
-
-			err = p.CachedLeaves.ForEach(func(k utreexo.Hash, v uint64) error {
-				return blockchain.CachedLeavesBackendPut(ldbTx, k, v)
-			})
-			if err != nil {
-				return err
-			}
-
-			return nil
+		err = cachedLeavesDB.Flush(ldbTx)
+		if err != nil {
+			return err
 		}
 
-		// Flush is never needed since we're keeping everything in memory.
-		isFlushNeeded = func() bool {
-			return false
-		}
+		return nil
+	}
+	isFlushNeeded := func() bool {
+		nodesNeedsFlush := nodesDB.IsFlushNeeded()
+		leavesNeedsFlush := cachedLeavesDB.IsFlushNeeded()
+		return nodesNeedsFlush || leavesNeedsFlush
 	}
 
 	uState := &UtreexoState{

blockchain/indexers/utreexoproofindex.go

@@ -610,8 +610,7 @@ func (idx *UtreexoProofIndex) PruneBlock(_ database.Tx, _ *chainhash.Hash, lastK
 
 // NewUtreexoProofIndex returns a new instance of an indexer that is used to create a utreexo
 // proof index using the database passed in. The passed in maxMemoryUsage should be in bytes and
-// it determines how much memory the proof index will use up. A maxMemoryUsage of 0 will keep
-// all the elements on disk and a negative maxMemoryUsage will keep all the elements in memory.
+// it determines how much memory the proof index will use up.
 //
 // It implements the Indexer interface which plugs into the IndexManager that in
 // turn is used by the blockchain package.  This allows the index to be

config.go

@@ -202,7 +202,7 @@ type config struct {
 	TTLIndex                   bool  `long:"ttlindex" description:"Maintain a full time to live index for all stxos available via the getttl RPC"`
 	UtreexoProofIndex          bool  `long:"utreexoproofindex" description:"Maintain a utreexo proof for all blocks"`
 	FlatUtreexoProofIndex      bool  `long:"flatutreexoproofindex" description:"Maintain a utreexo proof for all blocks in flat files"`
-	UtreexoProofIndexMaxMemory int64 `long:"utreexoproofindexmaxmemory" description:"The maxmimum memory in mebibytes (MiB) that the utreexo proof indexes will use up. Passing in 0 will make the entire proof index stay on disk. Passing in a negative value will make the entire proof index stay in memory. Default of 250MiB."`
+	UtreexoProofIndexMaxMemory int64 `long:"utreexoproofindexmaxmemory" description:"The maxmimum memory in mebibytes (MiB) that the utreexo proof indexes will use up. Default of 500MiB. Minimum of 250MiB"`
 	CFilters                   bool  `long:"cfilters" description:"Enable committed filtering (CF) support"`
 	NoPeerBloomFilters         bool  `long:"nopeerbloomfilters" description:"Disable bloom filtering support"`
 	DropAddrIndex              bool  `long:"dropaddrindex" description:"Deletes the address-based transaction index from the database on start up and then exits."`
@@ -992,6 +992,15 @@ func loadConfig() (*config, []string, error) {
 		return nil, nil, err
 	}
 
+	if cfg.UtreexoProofIndexMaxMemory < 250 {
+		err := fmt.Errorf("%s: the --utreexoproofindexmaxmemory "+
+			"option may not be less than 250",
+			funcName)
+		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, usageMessage)
+		return nil, nil, err
+	}
+
 	// Check mining addresses are valid and saved parsed versions.
 	cfg.miningAddrs = make([]btcutil.Address, 0, len(cfg.MiningAddrs))
 	for _, strAddr := range cfg.MiningAddrs {