merge master and add del sleep time

chain33.toml

@@ -140,14 +140,6 @@ enableMavlPrefix=false
 enableMVCC=false
 enableMavlPrune=false
 pruneHeight=10000
-enableMemTree=true
-
-[store.sub.kvmvccmavl]
-enableMVCCIter=false
-enableMavlPrefix=false
-enableMVCC=false
-enableMavlPrune=false
-pruneHeight=10000
 
 [wallet]
 minFee=100000

main.go

@@ -12,18 +12,11 @@
 package main
 
 import (
-	"net/http"
-	"runtime/debug"
-
 	_ "github.com/33cn/chain33/system"
 	"github.com/33cn/chain33/util/cli"
 	_ "github.com/33cn/plugin/plugin"
 )
 
 func main() {
-	go func() {
-		http.ListenAndServe("0.0.0.0:8080", nil)
-	}()
-	debug.SetGCPercent(20)
 	cli.RunChain33("")
 }

plugin/store/kvmvccmavl/kvmvcc_mavl.go

@@ -19,6 +19,7 @@ import (
 	drivers "github.com/33cn/chain33/system/store"
 	"github.com/33cn/chain33/types"
 	"github.com/hashicorp/golang-lru"
+	"time"
 )
 
 var (
@@ -236,7 +237,7 @@ func (kvmMavls *KVmMavlStore) IterateRangeByStateHash(statehash []byte, start []
 }
 
 // ProcEvent handles supported events
-func (kvmMavls *KVmMavlStore) ProcEvent(msg queue.Message) {
+func (kvmMavls *KVmMavlStore) ProcEvent(msg *queue.Message) {
 	msg.ReplyErr("KVmMavlStore", types.ErrActionNotSupport)
 }
 
@@ -290,6 +291,7 @@ func delMavlData(db dbm.DB) bool {
 			if batch.ValueSize() > batchDataSize {
 				batch.Write()
 				batch.Reset()
+				time.Sleep(time.Millisecond * 100)
 			}
 		}
 	}

vendor/github.com/33cn/chain33/system/store/mavl/db/node.go

@@ -12,7 +12,6 @@ import (
 	"github.com/33cn/chain33/common"
 	"github.com/33cn/chain33/types"
 	"github.com/golang/protobuf/proto"
-	. "github.com/dgryski/go-farm"
 )
 
 // Node merkle avl Node
@@ -500,9 +499,6 @@ func removeOrphan(t *Tree, node *Node) {
 	if t.ndb == nil {
 		return
 	}
-	if enableMemTree && t != nil {
-		t.obsoleteNode[Hash64(node.hash)] = struct{}{}
-	}
 	t.ndb.RemoveNode(t, node)
 }
 

vendor/github.com/33cn/chain33/system/store/mavl/db/tree.go

@@ -17,7 +17,6 @@ import (
 	"github.com/33cn/chain33/types"
 	"github.com/golang/protobuf/proto"
 	"github.com/hashicorp/golang-lru"
-	. "github.com/dgryski/go-farm"
 )
 
 const (
@@ -38,9 +37,6 @@ var (
 	// 当前树的最大高度
 	maxBlockHeight int64
 	heightMtx      sync.Mutex
-	enableMemTree  bool
-
-	memTree        MemTreeOpera
 )
 
 // EnableMavlPrefix 使能mavl加前缀
@@ -53,48 +49,23 @@ func EnableMVCC(enable bool) {
 	enableMvcc = enable
 }
 
-func EnableMemTree(enable bool) {
-	enableMemTree = enable
-}
-
-type memNode struct {
-	Key                  []byte
-	Value                []byte
-	LeftHash             []byte
-	RightHash            []byte
-	Height               int32
-	Size                 int32
-}
-
 // Tree merkle avl tree
 type Tree struct {
 	root        *Node
 	ndb         *nodeDB
 	blockHeight int64
-	// 树更新之后，废弃的节点(更新缓存中节点先对旧节点进行删除然后再更新)
-	obsoleteNode map[uint64]struct{}
-	updateNode   map[uint64]*memNode
 }
 
 // NewTree 新建一个merkle avl 树
 func NewTree(db dbm.DB, sync bool) *Tree {
 	if db == nil {
 		// In-memory IAVLTree
-		return &Tree{
-			    obsoleteNode: make(map[uint64]struct{}),
-			    updateNode: make(map[uint64]*memNode),
-			}
+		return &Tree{}
 	}
 	// Persistent IAVLTree
 	ndb := newNodeDB(db, sync)
-	// 使能情况下非空创建当前整tree的缓存
-	if enableMemTree && memTree == nil  {
-		memTree = NewTreeARC(300 * 10000)
-	}
 	return &Tree{
 		ndb: ndb,
-		obsoleteNode: make(map[uint64]struct{}),
-		updateNode: make(map[uint64]*memNode),
 	}
 }
 
@@ -152,10 +123,6 @@ func (t *Tree) Set(key []byte, value []byte) (updated bool) {
 	return updated
 }
 
-func (t *Tree) GetObsoleteNode() map[uint64]struct{} {
-	return t.obsoleteNode
-}
-
 // Hash 计算tree 的roothash
 func (t *Tree) Hash() []byte {
 	if t.root == nil {
@@ -181,17 +148,6 @@ func (t *Tree) Save() []byte {
 		if enablePrune {
 			t.root.saveRootHash(t)
 		}
-		// 更新memTree
-		if enableMemTree && memTree != nil {
-			for k, _ := range t.obsoleteNode {
-				memTree.Delete(k)
-			}
-			for k, v := range t.updateNode {
-				memTree.Add(k, v)
-			}
-			treelog.Debug("Tree.Save", "memTree len", memTree.Len(), "tree height", t.blockHeight)
-		}
-
 		beg := types.Now()
 		err := t.ndb.Commit()
 		treelog.Info("tree.commit", "cost", types.Since(beg))
@@ -439,24 +395,6 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 			return elem.(*Node), nil
 		}
 	}
-	//从memtree中获取
-	if enableMemTree && memTree != nil {
-		elem, ok := memTree.Get(Hash64(hash))
-		if ok {
-			sn := elem.(*memNode)
-			node := &Node{
-				key: sn.Key,
-				value: sn.Value,
-				height: sn.Height,
-				size: sn.Size,
-				leftHash: sn.LeftHash,
-				rightHash: sn.RightHash,
-				hash: hash,
-				persisted: true,
-			}
-			return node, nil
-		}
-	}
 	// Doesn't exist, load from db.
 	var buf []byte
 	buf, err := ndb.db.Get(hash)
@@ -471,21 +409,6 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 	node.hash = hash
 	node.persisted = true
 	ndb.cacheNode(node)
-	treelog.Debug("Tree.GetNode", "height", node.height)
-	// Save node hashInt64 to mem
-	if enableMemTree && memTree != nil /*&& node.height != 0*/ {
-		memN := &memNode{
-			Key: copyBytes(node.key),
-			LeftHash: copyBytes(node.leftHash),
-			RightHash: copyBytes(node.rightHash),
-			Height: node.height,
-			Size: node.size,
-		}
-		if node.height == 0 {
-			memN.Value = copyBytes(node.value)
-		}
-		memTree.Add(Hash64(node.hash), memN)
-	}
 	return node, nil
 }
 
@@ -536,29 +459,6 @@ func (ndb *nodeDB) SaveNode(t *Tree, node *Node) {
 	ndb.cacheNode(node)
 	delete(ndb.orphans, string(node.hash))
 	//treelog.Debug("SaveNode", "hash", node.hash, "height", node.height, "value", node.value)
-	// Save node hashInt64 to localmem
-	updateLocalTreeMem(t, node)
-}
-
-// Save node hashInt64 to localmem
-func updateLocalTreeMem(t *Tree, node *Node) {
-	if t == nil || node == nil {
-		return
-	}
-	if enableMemTree && t.updateNode != nil /*&& node.height != 0*/ { // 0高度不保存
-		memN := &memNode{
-			Key: copyBytes(node.key),
-			LeftHash: copyBytes(node.leftHash),
-			RightHash: copyBytes(node.rightHash),
-			Height: node.height,
-			Size: node.size,
-		}
-		if node.height == 0 {
-			memN.Value = copyBytes(node.value)
-		}
-		t.updateNode[Hash64(node.hash)] = memN
-		//treelog.Debug("Tree.SaveNode", "store struct size", unsafe.Sizeof(store), "byte size", len(storenode), "height", node.height)
-	}
 }
 
 //cache缓存节点

vendor/github.com/33cn/chain33/system/store/mavl/db/tree_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"unsafe"
 )
 
 func init() {
@@ -1256,87 +1255,6 @@ func TestDelLeafCountKV(t *testing.T) {
 	}
 }
 
-func TestGetObsoleteNode(t *testing.T) {
-	dir, err := ioutil.TempDir("", "datastore")
-	require.NoError(t, err)
-	t.Log(dir)
-	defer os.Remove(dir)
-
-	EnableMemTree(true)
-	defer EnableMemTree(false)
-
-	db := db.NewDB("mavltree", "leveldb", dir, 100)
-	tree := NewTree(db, true)
-
-	type record struct {
-		key   string
-		value string
-	}
-	records := []record{
-		{"abc", "abc"},
-		{"low", "low"},
-		{"fan", "fan"},
-	}
-
-	for _, r := range records {
-		updated := tree.Set([]byte(r.key), []byte(r.value))
-		if updated {
-			t.Error("should have not been updated")
-		}
-	}
-	hash := tree.Save()
-	obs := tree.GetObsoleteNode()
-	require.Equal(t, 0, len(obs))
-	mp := NewTreeMap(1000)
-	LoadTree2MemDb(db, hash, mp)
-
-	tree1 := NewTree(db, true)
-	tree1.Load(hash)
-	records1 := []record{
-		{"abc", "abc1"},
-		{"low", "low1"},
-		{"fan", "fan1"},
-	}
-
-	for _, r := range records1 {
-		tree1.Set([]byte(r.key), []byte(r.value))
-	}
-	hash1 := tree1.Save()
-	obs = tree1.GetObsoleteNode()
-	mp1 := NewTreeMap(1000)
-	LoadTree2MemDb(db, hash1, mp1)
-	require.Equal(t, mp.Len(), len(obs)) //做了全部更新，因此旧节点全部删除
-	for ob,_ := range obs {
-		_, ok :=  mp.Get(ob)
-		if !ok {
-			require.Error(t, fmt.Errorf("should exist"))
-		}
-	}
-
-	tree2 := NewTree(db, true)
-	tree2.Load(hash)
-	records2 := []record{
-		{"fan", "fan1"},
-		{"foo", "foo"},
-		{"foobaz", "foobaz"},
-		{"good", "good"},
-	}
-	for _, r := range records2 {
-		tree2.Set([]byte(r.key), []byte(r.value))
-	}
-	hash2 := tree2.Save()
-	obs = tree2.GetObsoleteNode()
-	mp2 := NewTreeMap(1000)
-	LoadTree2MemDb(db, hash2, mp2)
-	//require.Equal(t, 0, len(obs))
-	for ob,_ := range obs {
-		_, ok :=  mp.Get(ob)
-		if !ok {
-			require.Error(t, fmt.Errorf("should exist"))
-		}
-	}
-}
-
 func TestPruningFirstLevelNode(t *testing.T) {
 	dir, err := ioutil.TempDir("", "datastore")
 	require.NoError(t, err)
@@ -1981,33 +1899,3 @@ func saveBlock(dbm db.DB, height int64, hash []byte, txN int64, mvcc bool) (newH
 	}
 	return newHash, nil
 }
-
-
-func TestSize1(t *testing.T) {
-	type storeNode struct {
-		Key       []byte
-		Value     []byte
-		LeftHash  []byte
-		RightHash []byte
-		Height    int32
-		Size      int32
-	}
-	type storeNode1 struct {
-		Key       [][]byte
-		Height    int32
-		Size      int32
-	}
-	a := types.StoreNode{}
-	b := storeNode{}
-	var c []byte
-	d := storeNode1{}
-
-
-	d.Key = make([][]byte, 4)
-	d.Key[0] = []byte("11111111111111111111111111111")
-	d.Key[1] = []byte("22222222222222222222222222222")
-	d.Key[2] = []byte("33333333333333333333333333333")
-
-	PrintMemStats(1)
-	fmt.Println(unsafe.Sizeof(a), unsafe.Sizeof(b), unsafe.Sizeof(c), unsafe.Sizeof(d), len(d.Key), cap(d.Key))
-}
\ No newline at end of file