hash

playground/playground.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -19,6 +20,8 @@ import (
 	"github.com/yottachain/YTFS/opt"
 )
 
+var r0 = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 var (
 	configName string
 	home       string
@@ -163,6 +166,7 @@ func printProgress(cursor, volume uint64) {
 }
 
 func stressWrite(ytfs *ytfs.YTFS) error {
+	r := rand.New(rand.NewSource(r0.Int63()))
 	type KeyValuePair struct {
 		hash common.Hash
 		buf  []byte
@@ -173,9 +177,13 @@ func stressWrite(ytfs *ytfs.YTFS) error {
 
 	for i := (uint64)(0); i < dataCaps; i++ {
 		printProgress(i, dataCaps-1)
-		testHash := common.HexToHash(fmt.Sprintf("%032X", i))
 		data := make([]byte, ytfs.Meta().DataBlockSize, ytfs.Meta().DataBlockSize)
-		copy(data, testHash[:])
+		r.Read(data)
+		var sha256 = crypto.SHA3_256.New()
+		sha256.Write(data)
+		hs := sha256.Sum(nil)
+		var testHash [32]byte
+		copy(testHash[:], hs)
 		dataPair := KeyValuePair{
 			hash: testHash,
 			buf:  data,
@@ -192,8 +200,8 @@ func stressWrite(ytfs *ytfs.YTFS) error {
 
 func stressBatchWrite(ytfs *ytfs.YTFS) error {
 	type KeyValuePair struct {
-			hash common.Hash
-			buf  []byte
+		hash common.Hash
+		buf  []byte
 	}
 
 	dataCaps := ytfs.Cap()
@@ -208,10 +216,10 @@ func stressBatchWrite(ytfs *ytfs.YTFS) error {
 		copy(data, testHash[:])
 		batch[ydcommon.IndexTableKey(testHash)] = data
 
-		if (i + 1) % 17 == 0 {
+		if (i+1)%17 == 0 {
 			_, err := ytfs.BatchPut(batch)
 			if err != nil {
-					panic(err)
+				panic(err)
 			}
 			batch = map[ydcommon.IndexTableKey][]byte{}
 		}
@@ -257,27 +265,6 @@ func stressRead(ytfs *ytfs.YTFS) error {
 
 func stressTestReadAfterBatchWrite(ytfs *ytfs.YTFS) error {
 	err := stressBatchWrite(ytfs)
-	if err != nil {
-			panic(err)
-	}
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < runtime.NumCPU(); i++ {
-			wg.Add(1)
-			go func(id int) {
-					err := stressRead(ytfs)
-					if err != nil {
-							panic(err)
-					}
-					wg.Done()
-			}(i)
-	}
-	wg.Wait()
-	return err
-}
-
-func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error {
-	err := stressWrite(ytfs)
 	if err != nil {
 		panic(err)
 	}
@@ -297,6 +284,27 @@ func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error {
 	return err
 }
 
+func stressTestReadAfterWrite(ytfs *ytfs.YTFS) error {
+	err := stressWrite(ytfs)
+	if err != nil {
+		panic(err)
+	}
+
+	//wg := sync.WaitGroup{}
+	//for i := 0; i < runtime.NumCPU(); i++ {
+	//	wg.Add(1)
+	//	go func(id int) {
+	//		err := stressRead(ytfs)
+	//		if err != nil {
+	//			panic(err)
+	//		}
+	//		wg.Done()
+	//	}(i)
+	//}
+	//wg.Wait()
+	return err
+}
+
 func hybridTestReadAfterWrite(ytfs *ytfs.YTFS) error {
 	type KeyValuePair struct {
 		hash common.Hash

playground/stress/stress.go

+package main

playground/ytfs.config.template

 {
 	"ytfs": "ytfs default setting",
 	"storages": [
-	{
-        "storage": "/tmp/yotta-test-A",
-        "type": 0,
-        "dataBlockSize": 32768,
-        "storageSize": 1048576,
-	"syncPeriod": 1,
-        "readonly": false
-	},
-	{
-        "storage": "/tmp/yotta-test-B",
-        "type": 0,
-        "storageSize": 1048576,
-        "dataBlockSize": 32768,
-	"syncPeriod": 1,
-        "readonly": false
-	}
-	],
-	"syncPeriod": 1,
-	"readonly": false,
-	"C": 2147483648,
-	"N": 16384,
-	"D": 32768,
-	"M": 0
-}
\ No newline at end of file
+   	{
+           "storage": "/tmp/yotta-test-A",
+           "type": 0,
+           "dataBlockSize": 32768,
+           "storageSize": 1048576,
+   	"syncPeriod": 1,
+           "readonly": false
+   	},
+   	{
+           "storage": "/tmp/yotta-test-B",
+           "type": 0,
+           "storageSize": 1048576,
+           "dataBlockSize": 32768,
+   	"syncPeriod": 1,
+           "readonly": false
+   	}
+   	],
+   	"syncPeriod": 1,
+   	"readonly": false,
+   	"C": 2147483648,
+   	"N": 16384,
+   	"D": 32768,
+   	"M": 0
+   }
\ No newline at end of file

ytfs.go

@@ -9,6 +9,7 @@ import (
 
 	ydcommon "github.com/yottachain/YTFS/common"
 	"github.com/yottachain/YTFS/opt"
+	_ "net/http/pprof"
 )
 
 type ytfsStatus struct {
@@ -192,12 +193,12 @@ func (ytfs *YTFS) Put(key ydcommon.IndexTableKey, buf []byte) error {
 
 /*
  * Batch mode func list
-*/
+ */
 func (ytfs *YTFS) restoreYTFS() {
 	//TODO: save index
-	id := len(ytfs.savedStatus)-1
+	id := len(ytfs.savedStatus) - 1
 	ydcommon.YottaAssert(id >= 0)
-	ytfs.context.restore(ytfs.savedStatus[id].ctxSP);
+	ytfs.context.restore(ytfs.savedStatus[id].ctxSP)
 	ytfs.savedStatus = ytfs.savedStatus[:id]
 }
 
@@ -216,41 +217,41 @@ func (ytfs *YTFS) BatchPut(batch map[ydcommon.IndexTableKey][]byte) (map[ydcommo
 	ytfs.mutex.Lock()
 	defer ytfs.mutex.Unlock()
 
-	if (len(batch) > 32) {
+	if len(batch) > 1000 {
 		return nil, fmt.Errorf("Batch Size is too big")
 	}
 
 	// NO get check, but retore all status if error
-	ytfs.saveCurrentYTFS();
+	ytfs.saveCurrentYTFS()
 
 	batchIndexes := make([]ydcommon.IndexItem, len(batch))
-	batchBuffer := []byte{};
+	batchBuffer := []byte{}
 	bufCnt := len(batch)
-	i:=0
+	i := 0
 	for k, v := range batch {
 		batchBuffer = append(batchBuffer, v...)
 		batchIndexes[i] = ydcommon.IndexItem{
-			Hash: k,
+			Hash:      k,
 			OffsetIdx: ydcommon.IndexTableValue(0)}
 		i++
 	}
 
-	startPos, err := ytfs.context.BatchPut(bufCnt, batchBuffer);
+	startPos, err := ytfs.context.BatchPut(bufCnt, batchBuffer)
 	if err != nil {
-			ytfs.restoreYTFS();
-			return nil, err
+		ytfs.restoreYTFS()
+		return nil, err
 	}
 
-	for i:=uint32(0); i<uint32(bufCnt); i++ {
-			batchIndexes[i] = ydcommon.IndexItem{
-				Hash: batchIndexes[i].Hash,
-				OffsetIdx: ydcommon.IndexTableValue(startPos + i)}
+	for i := uint32(0); i < uint32(bufCnt); i++ {
+		batchIndexes[i] = ydcommon.IndexItem{
+			Hash:      batchIndexes[i].Hash,
+			OffsetIdx: ydcommon.IndexTableValue(startPos + i)}
 	}
 
 	conflicts, err := ytfs.db.BatchPut(batchIndexes)
 	if err != nil {
-			ytfs.restoreYTFS();
-			return conflicts, err
+		ytfs.restoreYTFS()
+		return conflicts, err
 	}
 	return nil, nil
 }