From 689784d3cefc037851699d3d138c864e4ec5be26 Mon Sep 17 00:00:00 2001 From: DiptoChakrabarty Date: Sun, 8 Aug 2021 09:31:06 +0530 Subject: [PATCH] bridgenode: saveundoblocks This commit is for saving the undoblocks data returned from forest.Modify from the BuildProofs function in the genproofs.go file. A new channel of undo unBlock type has been created where the undoblocks are passed which is then sent to flatfileworker. A new parameter for undoBlocks height is added along with config for undoBlock directory. --- accumulator/undo.go | 1 + bridgenode/config.go | 19 +++++++++ bridgenode/flatfileworker.go | 76 ++++++++++++++++++++++++++++++++++++ bridgenode/genproofs.go | 18 ++++++--- 4 files changed, 108 insertions(+), 6 deletions(-) diff --git a/accumulator/undo.go b/accumulator/undo.go index 44d04e74..00cdbaf8 100644 --- a/accumulator/undo.go +++ b/accumulator/undo.go @@ -16,6 +16,7 @@ although actually it can make sense for non-bridge nodes to undo as well... // blockUndo is all the data needed to undo a block: number of adds, // and all the hashes that got deleted and where they were from type UndoBlock struct { + Height int32 // height of block numAdds uint32 // number of adds in the block positions []uint64 // position of all deletions this block hashes []Hash // hashes that were deleted diff --git a/bridgenode/config.go b/bridgenode/config.go index eda2070e..c6d63947 100644 --- a/bridgenode/config.go +++ b/bridgenode/config.go @@ -88,11 +88,18 @@ type offsetDir struct { lastIndexOffsetHeightFile string } +type undoDir struct { + base string + undoFile string + offsetFile string +} + // All your utreexo bridgenode file paths in a nice and convinent struct type utreeDir struct { OffsetDir offsetDir ProofDir proofDir ForestDir forestDir + UndoDir undoDir Ttldb string } @@ -124,12 +131,20 @@ func initUtreeDir(basePath string) utreeDir { cowForestCurFile: filepath.Join(cowDir, "CURRENT"), } + undoBase := filepath.Join(basePath, "undoblockdata") + undo := undoDir{ + base: undoBase, + undoFile: filepath.Join(undoBase, "undo.dat"), + offsetFile: filepath.Join(undoBase, "offset.dat"), + } + ttldb := filepath.Join(basePath, "ttldb") return utreeDir{ OffsetDir: off, ProofDir: proof, ForestDir: forest, + UndoDir: undo, Ttldb: ttldb, } } @@ -152,6 +167,10 @@ func makePaths(dir utreeDir) error { if err != nil { return fmt.Errorf("init makePaths error %s") } + err = os.MkdirAll(dir.UndoDir.base, os.ModePerm) + if err != nil { + return fmt.Errorf("init makePaths error %s") + } return nil } diff --git a/bridgenode/flatfileworker.go b/bridgenode/flatfileworker.go index 44a793ff..b1434003 100644 --- a/bridgenode/flatfileworker.go +++ b/bridgenode/flatfileworker.go @@ -7,6 +7,7 @@ import ( "os" "sync" + "github.com/mit-dci/utreexo/accumulator" "github.com/mit-dci/utreexo/btcacc" ) @@ -60,6 +61,7 @@ type flatFileState struct { func flatFileWorker( proofChan chan btcacc.UData, ttlResultChan chan ttlResultBlock, + undoChan chan accumulator.UndoBlock, utreeDir utreeDir, fileWait *sync.WaitGroup) { @@ -85,6 +87,26 @@ func flatFileWorker( panic(err) } + // for the undofiles + var uf flatFileState + uf.offsetFile, err = os.OpenFile( + utreeDir.UndoDir.offsetFile, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + panic(err) + } + + uf.proofFile, err = os.OpenFile( + utreeDir.UndoDir.undoFile, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + panic(err) + } + + uf.fileWait = fileWait + + err = uf.ffInit() + if err != nil { + panic(err) + } // Grab either proof bytes and write em to offset / proof file, OR, get a TTL result // and write that. Will this lock up if it keeps doing proofs and ignores ttls? // it should keep both buffers about even. If it keeps doing proofs and the ttl @@ -120,6 +142,11 @@ func flatFileWorker( if err != nil { panic(err) } + case undo := <-undoChan: + err = uf.writeUndoBlock(undo) + if err != nil { + panic(err) + } } } } @@ -175,6 +202,55 @@ func (ff *flatFileState) ffInit() error { return nil } +func (ff *flatFileState) writeUndoBlock(ub accumulator.UndoBlock) error { + undoSize := ub.SerializeSize() + buf := make([]byte, undoSize) + + // write the offset of current of undo block to offset file + buf = buf[:8] + ff.offsets = append(ff.offsets, ff.currentOffset) + + binary.BigEndian.PutUint64(buf, uint64(ff.currentOffset)) + _, err := ff.offsetFile.WriteAt(buf, int64(8*ub.Height)) + if err != nil { + return err + } + + // write to undo file + _, err = ff.proofFile.WriteAt([]byte{0xaa, 0xff, 0xaa, 0xff}, ff.currentOffset) + if err != nil { + return err + } + + //prefix with size of the undoblocks + buf = buf[:4] + binary.BigEndian.PutUint32(buf, uint32(undoSize)) + _, err = ff.proofFile.WriteAt(buf, ff.currentOffset+4) + if err != nil { + return err + } + + // Serialize UndoBlock + buf = buf[:0] + bytesBuf := bytes.NewBuffer(buf) + err = ub.Serialize(bytesBuf) + if err != nil { + return err + } + + _, err = ff.proofFile.WriteAt(bytesBuf.Bytes(), ff.currentOffset+4+4) + if err != nil { + return err + } + + ff.currentOffset = ff.currentOffset + int64(undoSize) + 8 + ff.currentHeight++ + + ff.fileWait.Done() + + return nil +} + func (ff *flatFileState) writeProofBlock(ud btcacc.UData) error { // get the new block proof // put offset in ram diff --git a/bridgenode/genproofs.go b/bridgenode/genproofs.go index 7f6ebc64..dd6228cd 100644 --- a/bridgenode/genproofs.go +++ b/bridgenode/genproofs.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/mit-dci/utreexo/accumulator" "github.com/mit-dci/utreexo/btcacc" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" @@ -67,9 +68,10 @@ func BuildProofs(cfg *Config, sig chan bool) error { // async ttldb variables var dbwg sync.WaitGroup - dbWriteChan := make(chan ttlRawBlock, 10) // from block processing to db worker - ttlResultChan := make(chan ttlResultBlock, 10) // from db worker to flat ttl writer - proofChan := make(chan btcacc.UData, 10) // from proof processing to proof writer + dbWriteChan := make(chan ttlRawBlock, 10) // from block processing to db worker + ttlResultChan := make(chan ttlResultBlock, 10) // from db worker to flat ttl writer + proofChan := make(chan btcacc.UData, 10) // from proof processing to proof writer + undoChan := make(chan accumulator.UndoBlock, 10) // from undoblocks to undoblock writer // sorta ugly as in one of these will just be sitting around // doing nothing @@ -109,7 +111,7 @@ func BuildProofs(cfg *Config, sig chan bool) error { var fileWait sync.WaitGroup - go flatFileWorker(proofChan, ttlResultChan, cfg.UtreeDir, &fileWait) + go flatFileWorker(proofChan, ttlResultChan, undoChan, cfg.UtreeDir, &fileWait) fmt.Println("Building Proofs and ttldb...") @@ -131,7 +133,7 @@ func BuildProofs(cfg *Config, sig chan bool) error { // start waitgroups, beyond this point we have to finish all the // disk writes for this iteration of the loop dbwg.Add(1) // DbWorker calls Done() - fileWait.Add(2) // flatFileWorker calls Done() when done writing ttls and proof. + fileWait.Add(3) // flatFileWorker calls Done() when done writing ttls and proof and undoBlocks. // Writes the new txos to leveldb, // and generates TTL for txos spent in the block @@ -158,10 +160,14 @@ func BuildProofs(cfg *Config, sig chan bool) error { // TODO: Don't ignore undoblock // Modifies the forest with the given TXINs and TXOUTs - _, err = forest.Modify(blockAdds, ud.AccProof.Targets) + // return the undoBlock Data + undoblock, err := forest.Modify(blockAdds, ud.AccProof.Targets) if err != nil { return err } + undoblock.Height = bnr.Height // set undoBlocks Height + // send undoBlock data to undo channel to be written to the disk + undoChan <- *undoblock if bnr.Height%100 == 0 { fmt.Println("On block :", bnr.Height+1)