From b89cdab027d6aa7ea2cb9aca59c6e9a61adad9c3 Mon Sep 17 00:00:00 2001
From: Dipto Chakrabarty <45638240+DiptoChakrabarty@users.noreply.github.com>
Date: Mon, 6 Sep 2021 19:56:11 +0530
Subject: [PATCH] Saving TTL data seperately (#315)

* seperate ttl data into different directory

* remove unrequired conditional

* offset value increment before writing in prooffiles

* update currentoffset outisde loop add 0s bytes for each leaf created with new channel

* add filewait for the leafblock channel

* function for writes seperation and goroutines

* add loop for each channel write operations and update offsets operation for the ttls

* incorrect append value fixed
---
 bridgenode/config.go         |  17 +++++
 bridgenode/flatfileworker.go | 133 +++++++++++++++++++++++------------
 bridgenode/genproofs.go      |   8 ++-
 3 files changed, 110 insertions(+), 48 deletions(-)

diff --git a/bridgenode/config.go b/bridgenode/config.go
index e8a984b4..63055a27 100644
--- a/bridgenode/config.go
+++ b/bridgenode/config.go
@@ -94,6 +94,11 @@ type undoDir struct {
 	undoFile   string
 	offsetFile string
 }
+type ttlDir struct {
+	base       string
+	ttlsetFile string
+	OffsetFile string
+}
 
 // All your utreexo bridgenode file paths in a nice and convinent struct
 type utreeDir struct {
@@ -101,6 +106,7 @@ type utreeDir struct {
 	ProofDir  proofDir
 	ForestDir forestDir
 	UndoDir   undoDir
+	TtlDir    ttlDir
 	Ttldb     string
 }
 
@@ -139,6 +145,12 @@ func initUtreeDir(basePath string) utreeDir {
 		offsetFile: filepath.Join(undoBase, "offset.dat"),
 	}
 
+	ttlBase := filepath.Join(basePath, "ttldata")
+	ttl := ttlDir{
+		base:       ttlBase,
+		ttlsetFile: filepath.Join(ttlBase, "ttldata.dat"),
+		OffsetFile: filepath.Join(ttlBase, "offsetfile.dat"),
+	}
 	ttldb := filepath.Join(basePath, "ttldb")
 
 	return utreeDir{
@@ -146,6 +158,7 @@ func initUtreeDir(basePath string) utreeDir {
 		ProofDir:  proof,
 		ForestDir: forest,
 		UndoDir:   undo,
+		TtlDir:    ttl,
 		Ttldb:     ttldb,
 	}
 }
@@ -172,6 +185,10 @@ func makePaths(dir utreeDir) error {
 	if err != nil {
 		return fmt.Errorf("init makePaths error %s")
 	}
+	err = os.MkdirAll(dir.TtlDir.base, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("init makePaths error %s")
+	}
 	return nil
 }
 
diff --git a/bridgenode/flatfileworker.go b/bridgenode/flatfileworker.go
index b1434003..66fb7e3e 100644
--- a/bridgenode/flatfileworker.go
+++ b/bridgenode/flatfileworker.go
@@ -58,10 +58,9 @@ type flatFileState struct {
 // pFileWorker takes in blockproof and height information from the channel
 // and writes to disk. MUST NOT have more than one worker as the proofs need to be
 // in order
-func flatFileWorker(
+
+func flatFileWorkerProofBlocks(
 	proofChan chan btcacc.UData,
-	ttlResultChan chan ttlResultBlock,
-	undoChan chan accumulator.UndoBlock,
 	utreeDir utreeDir,
 	fileWait *sync.WaitGroup) {
 
@@ -87,8 +86,23 @@ func flatFileWorker(
 		panic(err)
 	}
 
-	// for the undofiles
+	for {
+		ud := <-proofChan
+		err = ff.writeProofBlock(ud)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func flatFileWorkerUndoBlocks(
+	undoChan chan accumulator.UndoBlock,
+	utreeDir utreeDir,
+	fileWait *sync.WaitGroup) {
+
 	var uf flatFileState
+	var err error
+
 	uf.offsetFile, err = os.OpenFile(
 		utreeDir.UndoDir.offsetFile, os.O_CREATE|os.O_RDWR, 0600)
 	if err != nil {
@@ -107,48 +121,63 @@ func flatFileWorker(
 	if err != nil {
 		panic(err)
 	}
-	// Grab either proof bytes and write em to offset / proof file, OR, get a TTL result
-	// and write that.  Will this lock up if it keeps doing proofs and ignores ttls?
-	// it should keep both buffers about even.  If it keeps doing proofs and the ttl
-	// buffer fills, then eventually it'll block...?
-	// Also, is it OK to have 2 different workers here?  It probably is, with the
-	// ttl side having read access to the proof writing side's last written proof.
-	// then the TTL side can do concurrent writes.  Also the TTL writes might be
-	// slow since they're all over the place.  Also the offsets should definitely
-	// be in ram since they'll be accessed a lot.
-
-	// TODO ^^^^^^ all that stuff.
-
-	// main selector - Write block proofs whenever you get them
-	// if you get TTLs, write them only if they're not too high
-	// if they are too high, keep writing proof blocks until they're not
 	for {
-		select {
-		case ud := <-proofChan:
-			err = ff.writeProofBlock(ud)
-			if err != nil {
-				panic(err)
-			}
-		case ttlRes := <-ttlResultChan:
-			for ttlRes.Height > ff.currentHeight {
-				ud := <-proofChan
-				err = ff.writeProofBlock(ud)
-				if err != nil {
-					panic(err)
-				}
-			}
+		undo := <-undoChan
+		err = uf.writeUndoBlock(undo)
+		if err != nil {
+			panic(err)
+		}
 
-			err = ff.writeTTLs(ttlRes)
-			if err != nil {
-				panic(err)
-			}
-		case undo := <-undoChan:
-			err = uf.writeUndoBlock(undo)
-			if err != nil {
-				panic(err)
-			}
+	}
+
+}
+
+func flatFileWorkerTTlBlocks(
+	ttlResultChan chan ttlResultBlock,
+	leafblockChan chan int,
+	utreeDir utreeDir,
+	fileWait *sync.WaitGroup) {
+
+	var tf flatFileState
+	var err error
+
+	tf.offsetFile, err = os.OpenFile(
+		utreeDir.TtlDir.OffsetFile, os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		panic(err)
+	}
+
+	tf.proofFile, err = os.OpenFile(
+		utreeDir.TtlDir.ttlsetFile, os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		panic(err)
+	}
+	tf.fileWait = fileWait
+
+	err = tf.ffInit()
+	if err != nil {
+		panic(err)
+	}
+
+	for {
+		size := <-leafblockChan
+		bytesTtlWrite := make([]byte, size*4)
+		_, err = tf.proofFile.WriteAt(bytesTtlWrite, tf.currentOffset)
+		tf.fileWait.Done()
+		if err != nil {
+			panic(err)
 		}
+		ttlRes := <-ttlResultChan
+		err = tf.writeTTLs(ttlRes)
+		if err != nil {
+			panic(err)
+		}
+		// append tf offsets after writing ttl data
+		tf.offsets = append(tf.offsets, tf.currentOffset)
+		// increment currentoffset value
+		tf.currentOffset = tf.currentOffset + int64(size*4)
 	}
+
 }
 
 func (ff *flatFileState) ffInit() error {
@@ -312,6 +341,15 @@ func (ff *flatFileState) writeProofBlock(ud btcacc.UData) error {
 
 func (ff *flatFileState) writeTTLs(ttlRes ttlResultBlock) error {
 	var ttlArr [4]byte
+	var buffer [8]byte
+
+	// write ttl offset to offsetfile
+	binary.BigEndian.PutUint64(buffer[:], uint64(ff.currentOffset))
+	_, err := ff.offsetFile.WriteAt(buffer[:], int64(8*ttlRes.Height))
+	if err != nil {
+		return err
+	}
+
 	// for all the TTLs, seek and overwrite the empty values there
 	for _, c := range ttlRes.Created {
 		// seek to the location of that txo's ttl value in the proof file
@@ -321,14 +359,17 @@ func (ff *flatFileState) writeTTLs(ttlRes ttlResultBlock) error {
 
 		// write it's lifespan as a 4 byte int32 (bit of a waste as
 		// 2 or 3 bytes would work)
-		// add 16: 4 for magic, 4 for size, 4 for height, 4 numTTL, then ttls start
-		_, err := ff.proofFile.WriteAt(ttlArr[:],
-			ff.offsets[c.createHeight]+16+int64(c.indexWithinBlock*4))
+		_, err = ff.proofFile.WriteAt(ttlArr[:],
+			ff.offsets[c.createHeight]+int64(c.indexWithinBlock*4))
 		if err != nil {
 			return err
 		}
-
 	}
+
+	// increment value of offset 4 bytes of each ttlRes Created
+	ff.currentOffset = ff.currentOffset + int64(len(ttlRes.Created)*4)
+	// increment height by 1
+	ff.currentHeight = ff.currentHeight + 1
 	ff.fileWait.Done()
 	return nil
 }
diff --git a/bridgenode/genproofs.go b/bridgenode/genproofs.go
index 96b5860b..cf596c00 100644
--- a/bridgenode/genproofs.go
+++ b/bridgenode/genproofs.go
@@ -72,6 +72,7 @@ func BuildProofs(cfg *Config, sig chan bool) error {
 	ttlResultChan := make(chan ttlResultBlock, 10)   // from db worker to flat ttl writer
 	proofChan := make(chan btcacc.UData, 10)         // from proof processing to proof writer
 	undoChan := make(chan accumulator.UndoBlock, 10) // from undoblocks to undoblock writer
+	leafblockChan := make(chan int, 10)              // from blocksadd to ttl writer
 
 	// sorta ugly as in one of these will just be sitting around
 	// doing nothing
@@ -111,7 +112,9 @@ func BuildProofs(cfg *Config, sig chan bool) error {
 
 	var fileWait sync.WaitGroup
 
-	go flatFileWorker(proofChan, ttlResultChan, undoChan, cfg.UtreeDir, &fileWait)
+	go flatFileWorkerProofBlocks(proofChan, cfg.UtreeDir, &fileWait)                  // flat file worker for proof blocks
+	go flatFileWorkerUndoBlocks(undoChan, cfg.UtreeDir, &fileWait)                    // flat file worker for the undo blocks
+	go flatFileWorkerTTlBlocks(ttlResultChan, leafblockChan, cfg.UtreeDir, &fileWait) // flat file worker for the ttl blocks
 
 	fmt.Println("Building Proofs and ttldb...")
 
@@ -133,7 +136,7 @@ func BuildProofs(cfg *Config, sig chan bool) error {
 		// start waitgroups, beyond this point we have to finish all the
 		// disk writes for this iteration of the loop
 		dbwg.Add(1)     // DbWorker calls Done()
-		fileWait.Add(3) // flatFileWorker calls Done() when done writing ttls and proof and undoBlocks.
+		fileWait.Add(4) // flatFileWorker calls Done() when done writing ttls and proof and undoBlocks and also blockadds for leafs.
 
 		// Writes the new txos to leveldb,
 		// and generates TTL for txos spent in the block
@@ -146,6 +149,7 @@ func BuildProofs(cfg *Config, sig chan bool) error {
 		if err != nil {
 			return err
 		}
+		leafblockChan <- len(blockAdds) // send size of 0s needed to be added to disk
 
 		// use the accumulator to get inclusion proofs, and produce a block
 		// proof with all data needed to verify the block