for mem mavl test

chain33.toml

@@ -146,6 +146,7 @@ enableMavlPrefix=false
 enableMVCC=false
 enableMavlPrune=false
 pruneHeight=10000
+enableMemTree=true
 
 [store.sub.kvmvccmavl]
 enableMVCCIter=false

main.go

@@ -15,8 +15,14 @@ import (
 	_ "github.com/33cn/chain33/system"
 	"github.com/33cn/chain33/util/cli"
 	_ "github.com/33cn/plugin/plugin"
+	"net/http"
+	"runtime/debug"
 )
 
 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:8080", nil)
+	}()
+	debug.SetGCPercent(20)
 	cli.RunChain33("")
 }

vendor/github.com/33cn/chain33/system/store/mavl/db/memmavl.go

+// Copyright Fuzamei Corp. 2018 All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mavl
+
+import (
+	"github.com/hashicorp/golang-lru"
+	"fmt"
+	"time"
+	. "github.com/dgryski/go-farm"
+	"github.com/33cn/chain33/common"
+	dbm "github.com/33cn/chain33/common/db"
+)
+
+type MemTreeOpera interface {
+	Add(key, value interface{})
+	Get(key interface{}) (value interface{}, ok bool)
+	Delete(key interface{})
+	Contains(key interface{}) bool
+	Len() int
+}
+
+type TreeMap struct {
+	mpCache map[interface{}]interface{}
+}
+
+func NewTreeMap(size int) *TreeMap {
+	mp := &TreeMap{}
+	mp.mpCache = make(map[interface{}]interface{}, size)
+	return mp
+}
+
+func (tm *TreeMap) Add(key, value interface{}) {
+	if _, ok := tm.mpCache[key]; ok {
+		panic(fmt.Sprintln("*********for test map*******", key))
+		delete(tm.mpCache, key)
+		return
+	}
+	tm.mpCache[key] = value
+}
+
+func (tm *TreeMap) Get(key interface{}) (value interface{}, ok bool) {
+	if value, ok := tm.mpCache[key]; ok {
+		return value, ok
+	}
+	return nil, false
+}
+
+func (tm *TreeMap) Delete(key interface{}) {
+	if _, ok := tm.mpCache[key]; ok {
+		delete(tm.mpCache, key)
+	}
+}
+
+func (tm *TreeMap) Contains(key interface{}) bool {
+	if _, ok := tm.mpCache[key]; ok {
+		return true
+	}
+	return false
+}
+
+func (tm *TreeMap) Len() int {
+	return len(tm.mpCache)
+}
+
+type TreeARC struct {
+	arcCache *lru.ARCCache
+}
+
+func NewTreeARC(size int) *TreeARC {
+	ma := &TreeARC{}
+	ma.arcCache, _ = lru.NewARC(size)
+	return ma
+}
+
+func (ta *TreeARC) Add(key, value interface{}) {
+	if ta.arcCache.Contains(key) {
+		panic(fmt.Sprintln("*********for test TreeARC*******", key))
+		ta.arcCache.Remove(key)
+		return
+	}
+	ta.arcCache.Add(key, value)
+}
+
+func (ta *TreeARC) Get(key interface{}) (value interface{}, ok bool) {
+	return ta.arcCache.Get(key)
+}
+
+func (ta *TreeARC) Delete(key interface{}) {
+	ta.arcCache.Remove(key)
+}
+
+func (ta *TreeARC) Contains(key interface{}) bool {
+	return ta.arcCache.Contains(key)
+}
+
+func (ta *TreeARC) Len() int {
+	return ta.arcCache.Len()
+}
+
+type hashNode struct {
+	leftHash  []byte
+	rightHash []byte
+}
+
+func LoadTree2MemDb(db dbm.DB, hash []byte, trMem MemTreeOpera) {
+	if trMem == nil {
+		return
+	}
+	nDb := newNodeDB(db, true)
+	node, err := nDb.GetLightNode(nil, hash)
+	if err != nil {
+		fmt.Println("err", err)
+		return
+	}
+	pri := ""
+	if len(node.hash) > 32 {
+		pri = string(node.hash[:16])
+	}
+	treelog.Info("hash node", "hash pri", pri, "hash", common.ToHex(node.hash), "height", node.height)
+	start := time.Now()
+	leftHash := make([]byte, len(node.leftHash))
+	copy(leftHash, node.leftHash)
+	rightHash := make([]byte, len(node.rightHash))
+	copy(rightHash, node.rightHash)
+	trMem.Add(Hash64(node.hash), &hashNode{leftHash: leftHash, rightHash: rightHash})
+	node.LoadNodeInfo(nDb, trMem)
+	end := time.Now()
+	treelog.Info("hash node", "cost time", end.Sub(start), "node count", trMem.Len())
+	PrintMemStats(1)
+}
+
+func (node *Node) LoadNodeInfo(db *nodeDB, trMem MemTreeOpera) {
+	if node.height == 0 {
+		//trMem.Add(Hash64(node.hash), &hashNode{leftHash: node.leftHash, rightHash: node.rightHash})
+		leftHash := make([]byte, len(node.leftHash))
+		copy(leftHash, node.leftHash)
+		rightHash := make([]byte, len(node.rightHash))
+		copy(rightHash, node.rightHash)
+		trMem.Add(Hash64(node.hash), &hashNode{leftHash: leftHash, rightHash: rightHash})
+		return
+	}
+	if node.leftHash != nil {
+		left, err := db.GetLightNode(nil, node.leftHash)
+		if err != nil {
+			return
+		}
+		//trMem.Add(Hash64(left.hash), &hashNode{leftHash: left.leftHash, rightHash: left.rightHash})
+		leftHash := make([]byte, len(left.leftHash))
+		copy(leftHash, left.leftHash)
+		rightHash := make([]byte, len(left.rightHash))
+		copy(rightHash, left.rightHash)
+		trMem.Add(Hash64(left.hash), &hashNode{leftHash: leftHash, rightHash: rightHash})
+		left.LoadNodeInfo(db, trMem)
+	}
+	if node.rightHash != nil {
+		right, err := db.GetLightNode(nil, node.rightHash)
+		if err != nil {
+			return
+		}
+		//trMem.Add(Hash64(right.hash), &hashNode{leftHash: right.leftHash, rightHash: right.rightHash})
+		leftHash := make([]byte, len(right.leftHash))
+		copy(leftHash, right.leftHash)
+		rightHash := make([]byte, len(right.rightHash))
+		copy(rightHash, right.rightHash)
+		trMem.Add(Hash64(right.hash), &hashNode{leftHash: leftHash, rightHash: rightHash})
+		right.LoadNodeInfo(db, trMem)
+	}
+}
+
+func (ndb *nodeDB) GetLightNode(t *Tree, hash []byte) (*Node, error) {
+	// Doesn't exist, load from db.
+	var buf []byte
+	buf, err := ndb.db.Get(hash)
+
+	if len(buf) == 0 || err != nil {
+		return nil, ErrNodeNotExist
+	}
+	node, err := MakeNode(buf, t)
+	if err != nil {
+		panic(fmt.Sprintf("Error reading IAVLNode. bytes: %X  error: %v", buf, err))
+	}
+	node.hash = hash
+	node.key = nil
+	node.value = nil
+	return node, nil
+}
+
+func copyBytes(b []byte) (copiedBytes []byte) {
+	if b == nil {
+		return nil
+	}
+	copiedBytes = make([]byte, len(b))
+	copy(copiedBytes, b)
+	return copiedBytes
+}
\ No newline at end of file

vendor/github.com/33cn/chain33/system/store/mavl/db/node.go

@@ -12,6 +12,7 @@ import (
 	"github.com/33cn/chain33/common"
 	"github.com/33cn/chain33/types"
 	"github.com/golang/protobuf/proto"
+	. "github.com/dgryski/go-farm"
 )
 
 // Node merkle avl Node
@@ -499,6 +500,9 @@ func removeOrphan(t *Tree, node *Node) {
 	if t.ndb == nil {
 		return
 	}
+	if enableMemTree && t != nil {
+		t.obsoleteNode[Hash64(node.hash)] = struct{}{}
+	}
 	t.ndb.RemoveNode(t, node)
 }
 

vendor/github.com/33cn/chain33/system/store/mavl/db/tree.go

@@ -17,6 +17,7 @@ import (
 	"github.com/33cn/chain33/types"
 	"github.com/golang/protobuf/proto"
 	"github.com/hashicorp/golang-lru"
+	. "github.com/dgryski/go-farm"
 )
 
 const (
@@ -37,6 +38,9 @@ var (
 	// 当前树的最大高度
 	maxBlockHeight int64
 	heightMtx      sync.Mutex
+	enableMemTree  bool
+
+	memTree        MemTreeOpera
 )
 
 // EnableMavlPrefix 使能mavl加前缀
@@ -49,23 +53,48 @@ func EnableMVCC(enable bool) {
 	enableMvcc = enable
 }
 
+func EnableMemTree(enable bool) {
+	enableMemTree = enable
+}
+
+type memNode struct {
+	Key                  []byte
+	Value                []byte
+	LeftHash             []byte
+	RightHash            []byte
+	Height               int32
+	Size                 int32
+}
+
 // Tree merkle avl tree
 type Tree struct {
 	root        *Node
 	ndb         *nodeDB
 	blockHeight int64
+	// 树更新之后，废弃的节点(更新缓存中节点先对旧节点进行删除然后再更新)
+	obsoleteNode map[uint64]struct{}
+	updateNode   map[uint64]*memNode
 }
 
 // NewTree 新建一个merkle avl 树
 func NewTree(db dbm.DB, sync bool) *Tree {
 	if db == nil {
 		// In-memory IAVLTree
-		return &Tree{}
+		return &Tree{
+			    obsoleteNode: make(map[uint64]struct{}),
+			    updateNode: make(map[uint64]*memNode),
+			}
 	}
 	// Persistent IAVLTree
 	ndb := newNodeDB(db, sync)
+	// 使能情况下非空创建当前整tree的缓存
+	if enableMemTree && memTree == nil  {
+		memTree = NewTreeARC(300 * 10000)
+	}
 	return &Tree{
 		ndb: ndb,
+		obsoleteNode: make(map[uint64]struct{}),
+		updateNode: make(map[uint64]*memNode),
 	}
 }
 
@@ -123,6 +152,10 @@ func (t *Tree) Set(key []byte, value []byte) (updated bool) {
 	return updated
 }
 
+func (t *Tree) GetObsoleteNode() map[uint64]struct{} {
+	return t.obsoleteNode
+}
+
 // Hash 计算tree 的roothash
 func (t *Tree) Hash() []byte {
 	if t.root == nil {
@@ -148,6 +181,17 @@ func (t *Tree) Save() []byte {
 		if enablePrune {
 			t.root.saveRootHash(t)
 		}
+		// 更新memTree
+		if enableMemTree && memTree != nil {
+			for k, _ := range t.obsoleteNode {
+				memTree.Delete(k)
+			}
+			for k, v := range t.updateNode {
+				memTree.Add(k, v)
+			}
+			treelog.Debug("Tree.Save", "memTree len", memTree.Len(), "tree height", t.blockHeight)
+		}
+
 		beg := types.Now()
 		err := t.ndb.Commit()
 		treelog.Info("tree.commit", "cost", types.Since(beg))
@@ -395,6 +439,24 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 			return elem.(*Node), nil
 		}
 	}
+	//从memtree中获取
+	if enableMemTree && memTree != nil {
+		elem, ok := memTree.Get(Hash64(hash))
+		if ok {
+			sn := elem.(*memNode)
+			node := &Node{
+				key: sn.Key,
+				value: sn.Value,
+				height: sn.Height,
+				size: sn.Size,
+				leftHash: sn.LeftHash,
+				rightHash: sn.RightHash,
+				hash: hash,
+				persisted: true,
+			}
+			return node, nil
+		}
+	}
 	// Doesn't exist, load from db.
 	var buf []byte
 	buf, err := ndb.db.Get(hash)
@@ -409,6 +471,21 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 	node.hash = hash
 	node.persisted = true
 	ndb.cacheNode(node)
+	treelog.Debug("Tree.GetNode", "height", node.height)
+	// Save node hashInt64 to mem
+	if enableMemTree && memTree != nil /*&& node.height != 0*/ {
+		memN := &memNode{
+			Key: copyBytes(node.key),
+			LeftHash: copyBytes(node.leftHash),
+			RightHash: copyBytes(node.rightHash),
+			Height: node.height,
+			Size: node.size,
+		}
+		if node.height == 0 {
+			memN.Value = copyBytes(node.value)
+		}
+		memTree.Add(Hash64(node.hash), memN)
+	}
 	return node, nil
 }
 
@@ -459,6 +536,29 @@ func (ndb *nodeDB) SaveNode(t *Tree, node *Node) {
 	ndb.cacheNode(node)
 	delete(ndb.orphans, string(node.hash))
 	//treelog.Debug("SaveNode", "hash", node.hash, "height", node.height, "value", node.value)
+	// Save node hashInt64 to localmem
+	updateLocalTreeMem(t, node)
+}
+
+// Save node hashInt64 to localmem
+func updateLocalTreeMem(t *Tree, node *Node) {
+	if t == nil || node == nil {
+		return
+	}
+	if enableMemTree && t.updateNode != nil /*&& node.height != 0*/ { // 0高度不保存
+		memN := &memNode{
+			Key: copyBytes(node.key),
+			LeftHash: copyBytes(node.leftHash),
+			RightHash: copyBytes(node.rightHash),
+			Height: node.height,
+			Size: node.size,
+		}
+		if node.height == 0 {
+			memN.Value = copyBytes(node.value)
+		}
+		t.updateNode[Hash64(node.hash)] = memN
+		//treelog.Debug("Tree.SaveNode", "store struct size", unsafe.Sizeof(store), "byte size", len(storenode), "height", node.height)
+	}
 }
 
 //cache缓存节点

vendor/github.com/33cn/chain33/system/store/mavl/db/tree_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"unsafe"
 )
 
 func init() {
@@ -1255,6 +1256,87 @@ func TestDelLeafCountKV(t *testing.T) {
 	}
 }
 
+func TestGetObsoleteNode(t *testing.T) {
+	dir, err := ioutil.TempDir("", "datastore")
+	require.NoError(t, err)
+	t.Log(dir)
+	defer os.Remove(dir)
+
+	EnableMemTree(true)
+	defer EnableMemTree(false)
+
+	db := db.NewDB("mavltree", "leveldb", dir, 100)
+	tree := NewTree(db, true)
+
+	type record struct {
+		key   string
+		value string
+	}
+	records := []record{
+		{"abc", "abc"},
+		{"low", "low"},
+		{"fan", "fan"},
+	}
+
+	for _, r := range records {
+		updated := tree.Set([]byte(r.key), []byte(r.value))
+		if updated {
+			t.Error("should have not been updated")
+		}
+	}
+	hash := tree.Save()
+	obs := tree.GetObsoleteNode()
+	require.Equal(t, 0, len(obs))
+	mp := NewTreeMap(1000)
+	LoadTree2MemDb(db, hash, mp)
+
+	tree1 := NewTree(db, true)
+	tree1.Load(hash)
+	records1 := []record{
+		{"abc", "abc1"},
+		{"low", "low1"},
+		{"fan", "fan1"},
+	}
+
+	for _, r := range records1 {
+		tree1.Set([]byte(r.key), []byte(r.value))
+	}
+	hash1 := tree1.Save()
+	obs = tree1.GetObsoleteNode()
+	mp1 := NewTreeMap(1000)
+	LoadTree2MemDb(db, hash1, mp1)
+	require.Equal(t, mp.Len(), len(obs)) //做了全部更新，因此旧节点全部删除
+	for ob,_ := range obs {
+		_, ok :=  mp.Get(ob)
+		if !ok {
+			require.Error(t, fmt.Errorf("should exist"))
+		}
+	}
+
+	tree2 := NewTree(db, true)
+	tree2.Load(hash)
+	records2 := []record{
+		{"fan", "fan1"},
+		{"foo", "foo"},
+		{"foobaz", "foobaz"},
+		{"good", "good"},
+	}
+	for _, r := range records2 {
+		tree2.Set([]byte(r.key), []byte(r.value))
+	}
+	hash2 := tree2.Save()
+	obs = tree2.GetObsoleteNode()
+	mp2 := NewTreeMap(1000)
+	LoadTree2MemDb(db, hash2, mp2)
+	//require.Equal(t, 0, len(obs))
+	for ob,_ := range obs {
+		_, ok :=  mp.Get(ob)
+		if !ok {
+			require.Error(t, fmt.Errorf("should exist"))
+		}
+	}
+}
+
 func TestPruningFirstLevelNode(t *testing.T) {
 	dir, err := ioutil.TempDir("", "datastore")
 	require.NoError(t, err)
@@ -1899,3 +1981,33 @@ func saveBlock(dbm db.DB, height int64, hash []byte, txN int64, mvcc bool) (newH
 	}
 	return newHash, nil
 }
+
+
+func TestSize1(t *testing.T) {
+	type storeNode struct {
+		Key       []byte
+		Value     []byte
+		LeftHash  []byte
+		RightHash []byte
+		Height    int32
+		Size      int32
+	}
+	type storeNode1 struct {
+		Key       [][]byte
+		Height    int32
+		Size      int32
+	}
+	a := types.StoreNode{}
+	b := storeNode{}
+	var c []byte
+	d := storeNode1{}
+
+
+	d.Key = make([][]byte, 4)
+	d.Key[0] = []byte("11111111111111111111111111111")
+	d.Key[1] = []byte("22222222222222222222222222222")
+	d.Key[2] = []byte("33333333333333333333333333333")
+
+	PrintMemStats(1)
+	fmt.Println(unsafe.Sizeof(a), unsafe.Sizeof(b), unsafe.Sizeof(c), unsafe.Sizeof(d), len(d.Key), cap(d.Key))
+}
\ No newline at end of file

vendor/github.com/33cn/chain33/system/store/mavl/mavl.go

@@ -37,6 +37,7 @@ type Store struct {
 	enableMVCC       bool
 	enableMavlPrune  bool
 	pruneHeight      int32
+	enableMemTree    bool
 }
 
 func init() {
@@ -48,6 +49,7 @@ type subConfig struct {
 	EnableMVCC       bool  `json:"enableMVCC"`
 	EnableMavlPrune  bool  `json:"enableMavlPrune"`
 	PruneHeight      int32 `json:"pruneHeight"`
+	EnableMemTree    bool  `json:"enableMemTree"`
 }
 
 // New new mavl store module
@@ -57,15 +59,18 @@ func New(cfg *types.Store, sub []byte) queue.Module {
 	if sub != nil {
 		types.MustDecode(sub, &subcfg)
 	}
-	mavls := &Store{bs, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC, subcfg.EnableMavlPrune, subcfg.PruneHeight}
+	mavls := &Store{bs, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC,
+	subcfg.EnableMavlPrune, subcfg.PruneHeight, subcfg.EnableMemTree}
 	mavls.enableMavlPrefix = subcfg.EnableMavlPrefix
 	mavls.enableMVCC = subcfg.EnableMVCC
 	mavls.enableMavlPrune = subcfg.EnableMavlPrune
 	mavls.pruneHeight = subcfg.PruneHeight
+	mavls.enableMemTree = subcfg.EnableMemTree
 	mavl.EnableMavlPrefix(mavls.enableMavlPrefix)
 	mavl.EnableMVCC(mavls.enableMVCC)
 	mavl.EnablePrune(mavls.enableMavlPrune)
 	mavl.SetPruneHeight(int(mavls.pruneHeight))
+	mavl.EnableMemTree(mavls.enableMemTree)
 	bs.SetChild(mavls)
 	return mavls
 }