From 9f7ffa2a105ceaf2f6db6333263907d235ad2bcb Mon Sep 17 00:00:00 2001 From: dp524856 <1456887752@qq.com> Date: Wed, 13 Nov 2019 23:36:24 +0800 Subject: [PATCH] hash --- playground/playground.go | 62 +++++++++++++++++++-------------- playground/stress/stress.go | 1 + playground/ytfs.config.template | 48 ++++++++++++------------- ytfs.go | 35 ++++++++++--------- 4 files changed, 78 insertions(+), 68 deletions(-) create mode 100644 playground/stress/stress.go diff --git a/playground/playground.go b/playground/playground.go index dc87348..93be2c4 100644 --- a/playground/playground.go +++ b/playground/playground.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "crypto" "flag" "fmt" "io/ioutil" @@ -19,6 +20,8 @@ import ( "github.com/yottachain/YTFS/opt" ) +var r0 = rand.New(rand.NewSource(time.Now().UnixNano())) + var ( configName string home string @@ -163,6 +166,7 @@ func printProgress(cursor, volume uint64) { } func stressWrite(ytfs *ytfs.YTFS) error { + r := rand.New(rand.NewSource(r0.Int63())) type KeyValuePair struct { hash common.Hash buf []byte @@ -173,9 +177,13 @@ func stressWrite(ytfs *ytfs.YTFS) error { for i := (uint64)(0); i < dataCaps; i++ { printProgress(i, dataCaps-1) - testHash := common.HexToHash(fmt.Sprintf("%032X", i)) data := make([]byte, ytfs.Meta().DataBlockSize, ytfs.Meta().DataBlockSize) - copy(data, testHash[:]) + r.Read(data) + var sha256 = crypto.SHA3_256.New() + sha256.Write(data) + hs := sha256.Sum(nil) + var testHash [32]byte + copy(testHash[:], hs) dataPair := KeyValuePair{ hash: testHash, buf: data, @@ -192,8 +200,8 @@ func stressWrite(ytfs *ytfs.YTFS) error { func stressBatchWrite(ytfs *ytfs.YTFS) error { type KeyValuePair struct { - hash common.Hash - buf []byte + hash common.Hash + buf []byte } dataCaps := ytfs.Cap() @@ -208,10 +216,10 @@ func stressBatchWrite(ytfs *ytfs.YTFS) error { copy(data, testHash[:]) batch[ydcommon.IndexTableKey(testHash)] = data - if (i + 1) % 17 == 0 { + if (i+1)%17 == 0 { _, err := ytfs.BatchPut(batch) if err != nil { - panic(err) + panic(err) } batch = map[ydcommon.IndexTableKey][]byte{} } @@ -257,27 +265,6 @@ func stressRead(ytfs *ytfs.YTFS) error { func stressTestReadAfterBatchWrite(ytfs *ytfs.YTFS) error { err := stressBatchWrite(ytfs) - if err != nil { - panic(err) - } - - wg := sync.WaitGroup{} - for i := 0; i < runtime.NumCPU(); i++ { - wg.Add(1) - go func(id int) { - err := stressRead(ytfs) - if err != nil { - panic(err) - } - wg.Done() - }(i) - } - wg.Wait() - return err -} - -func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error { - err := stressWrite(ytfs) if err != nil { panic(err) } @@ -297,6 +284,27 @@ func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error { return err } +func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error { + err := stressWrite(ytfs) + if err != nil { + panic(err) + } + + //wg := sync.WaitGroup{} + //for i := 0; i < runtime.NumCPU(); i++ { + // wg.Add(1) + // go func(id int) { + // err := stressRead(ytfs) + // if err != nil { + // panic(err) + // } + // wg.Done() + // }(i) + //} + //wg.Wait() + return err +} + func hybridTestReadAfterWrite(ytfs *ytfs.YTFS) error { type KeyValuePair struct { hash common.Hash diff --git a/playground/stress/stress.go b/playground/stress/stress.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/playground/stress/stress.go @@ -0,0 +1 @@ +package main diff --git a/playground/ytfs.config.template b/playground/ytfs.config.template index e2c63d2..e4ed9ff 100644 --- a/playground/ytfs.config.template +++ b/playground/ytfs.config.template @@ -1,27 +1,27 @@ { "ytfs": "ytfs default setting", "storages": [ - { - "storage": "/tmp/yotta-test-A", - "type": 0, - "dataBlockSize": 32768, - "storageSize": 1048576, - "syncPeriod": 1, - "readonly": false - }, - { - "storage": "/tmp/yotta-test-B", - "type": 0, - "storageSize": 1048576, - "dataBlockSize": 32768, - "syncPeriod": 1, - "readonly": false - } - ], - "syncPeriod": 1, - "readonly": false, - "C": 2147483648, - "N": 16384, - "D": 32768, - "M": 0 -} \ No newline at end of file + { + "storage": "/tmp/yotta-test-A", + "type": 0, + "dataBlockSize": 32768, + "storageSize": 1048576, + "syncPeriod": 1, + "readonly": false + }, + { + "storage": "/tmp/yotta-test-B", + "type": 0, + "storageSize": 1048576, + "dataBlockSize": 32768, + "syncPeriod": 1, + "readonly": false + } + ], + "syncPeriod": 1, + "readonly": false, + "C": 2147483648, + "N": 16384, + "D": 32768, + "M": 0 + } \ No newline at end of file diff --git a/ytfs.go b/ytfs.go index a06a3c3..6b38347 100644 --- a/ytfs.go +++ b/ytfs.go @@ -9,6 +9,7 @@ import ( ydcommon "github.com/yottachain/YTFS/common" "github.com/yottachain/YTFS/opt" + _ "net/http/pprof" ) type ytfsStatus struct { @@ -192,12 +193,12 @@ func (ytfs *YTFS) Put(key ydcommon.IndexTableKey, buf []byte) error { /* * Batch mode func list -*/ + */ func (ytfs *YTFS) restoreYTFS() { //TODO: save index - id := len(ytfs.savedStatus)-1 + id := len(ytfs.savedStatus) - 1 ydcommon.YottaAssert(id >= 0) - ytfs.context.restore(ytfs.savedStatus[id].ctxSP); + ytfs.context.restore(ytfs.savedStatus[id].ctxSP) ytfs.savedStatus = ytfs.savedStatus[:id] } @@ -216,41 +217,41 @@ func (ytfs *YTFS) BatchPut(batch map[ydcommon.IndexTableKey][]byte) (map[ydcommo ytfs.mutex.Lock() defer ytfs.mutex.Unlock() - if (len(batch) > 32) { + if len(batch) > 1000 { return nil, fmt.Errorf("Batch Size is too big") } // NO get check, but retore all status if error - ytfs.saveCurrentYTFS(); + ytfs.saveCurrentYTFS() batchIndexes := make([]ydcommon.IndexItem, len(batch)) - batchBuffer := []byte{}; + batchBuffer := []byte{} bufCnt := len(batch) - i:=0 + i := 0 for k, v := range batch { batchBuffer = append(batchBuffer, v...) batchIndexes[i] = ydcommon.IndexItem{ - Hash: k, + Hash: k, OffsetIdx: ydcommon.IndexTableValue(0)} i++ } - startPos, err := ytfs.context.BatchPut(bufCnt, batchBuffer); + startPos, err := ytfs.context.BatchPut(bufCnt, batchBuffer) if err != nil { - ytfs.restoreYTFS(); - return nil, err + ytfs.restoreYTFS() + return nil, err } - for i:=uint32(0); i