update chain33

vendor/github.com/33cn/chain33/.travis.yml

@@ -11,7 +11,7 @@ matrix:
     - name: check_fmt
       sudo: require
       go:
-        - "1.9"
+        - "1.11.x"
         - master
       install:
         - go get -u golang.org/x/tools/cmd/goimports

vendor/github.com/33cn/chain33/account/account_test.go

@@ -601,4 +601,3 @@ func TestDB_Burn(t *testing.T) {
 	t.Logf("Token mint addr balance [%d]", tokenCoin.LoadAccount(addr1).Balance)
 	require.Equal(t, int64(1000*1e8-10*1e8), tokenCoin.LoadAccount(addr1).Balance)
 }
-

vendor/github.com/33cn/chain33/cmd/chain33/bityuan.toml

@@ -90,6 +90,10 @@ enableMVCC=false
 enableMavlPrune=false
 # 裁剪高度间隔
 pruneHeight=10000
+# 是否使能mavl数据载入内存
+enableMemTree=false
+# 是否使能mavl叶子节点数据载入内存
+enableMemVal=false
 
 [wallet]
 # walletdb路径

vendor/github.com/33cn/chain33/cmd/chain33/chain33.test.toml

@@ -144,6 +144,10 @@ enableMavlPrefix=false
 enableMVCC=false
 enableMavlPrune=false
 pruneHeight=10000
+# 是否使能mavl数据载入内存
+enableMemTree=false
+# 是否使能mavl叶子节点数据载入内存
+enableMemVal=false
 
 [wallet]
 minFee=1000000

vendor/github.com/33cn/chain33/cmd/chain33/chain33.toml

@@ -210,6 +210,10 @@ enableMVCC=false
 enableMavlPrune=false
 # 裁剪高度间隔
 pruneHeight=10000
+# 是否使能mavl数据载入内存
+enableMemTree=false
+# 是否使能mavl叶子节点数据载入内存
+enableMemVal=false
 
 [wallet]
 # 交易发送最低手续费，单位0.00000001BTY(1e-8),默认100000，即0.001BTY

vendor/github.com/33cn/chain33/executor/execenv.go

@@ -203,11 +203,24 @@ func (e *executor) Exec(tx *types.Transaction, index int) (*types.Receipt, error
 	if err := drivers.CheckAddress(tx.GetRealToAddr(), e.height); err != nil {
 		return nil, err
 	}
+	if e.localDB != nil {
+		e.localDB.(*LocalDB).DisableWrite()
+		if exec.ExecutorOrder() != drivers.ExecLocalSameTime {
+			e.localDB.(*LocalDB).DisableRead()
+		}
+		defer func() {
+			e.localDB.(*LocalDB).EnableWrite()
+			if exec.ExecutorOrder() != drivers.ExecLocalSameTime {
+				e.localDB.(*LocalDB).EnableRead()
+			}
+		}()
+	}
 	//第一步先检查 CheckTx
 	if err := exec.CheckTx(tx, index); err != nil {
 		return nil, err
 	}
-	return exec.Exec(tx, index)
+	r, err := exec.Exec(tx, index)
+	return r, err
 }
 
 func (e *executor) execLocal(tx *types.Transaction, r *types.ReceiptData, index int) (*types.LocalDBSet, error) {

vendor/github.com/33cn/chain33/executor/executor_real_test.go

@@ -312,7 +312,10 @@ func (demo *demoApp) Exec(tx *types.Transaction, index int) (receipt *types.Rece
 	}
 	if seterrkey {
 		println("set err key value")
-		demo.GetLocalDB().Set([]byte("key1"), []byte("value1"))
+		err = demo.GetLocalDB().Set([]byte("key1"), []byte("value1"))
+		if err != nil {
+			return nil, err
+		}
 	}
 	receipt = &types.Receipt{Ty: types.ExecOk}
 	receipt.KV = append(receipt.KV, &types.KeyValue{
@@ -401,11 +404,14 @@ func TestExecLocalSameTime0(t *testing.T) {
 		return
 	}
 	for i, receipt := range detail.Receipts {
-		assert.Equal(t, receipt.GetTy(), int32(2), fmt.Sprint(i))
+		if i == 0 {
+			assert.Equal(t, receipt.GetTy(), int32(2), fmt.Sprint(i))
+		}
 		if i >= 1 {
+			assert.Equal(t, receipt.GetTy(), int32(1), fmt.Sprint(i))
 			fmt.Println(receipt)
 			assert.Equal(t, len(receipt.Logs), 2)
-			assert.Equal(t, receipt.Logs[1].Ty, int32(0))
+			assert.Equal(t, receipt.Logs[1].Ty, int32(1))
 		}
 	}
 }

vendor/github.com/33cn/chain33/executor/executor_test.go

@@ -5,6 +5,7 @@
 package executor
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -198,5 +199,6 @@ func TestExecutorErrAPIEnv(t *testing.T) {
 	msg := queue.NewMessage(0, "", 1, txlist)
 	exec.procExecTxList(msg)
 	_, err := exec.client.WaitTimeout(msg, 100*time.Second)
+	fmt.Println(err)
 	assert.Equal(t, true, api.IsAPIEnvError(err))
 }

vendor/github.com/33cn/chain33/executor/localdb.go

@@ -12,15 +12,17 @@ import (
 //数据的get set 主要经过 cache
 //如果需要进行list, 那么把get set 的内容加入到 后端数据库
 type LocalDB struct {
-	cache    map[string][]byte
-	txcache  map[string][]byte
-	keys     []string
-	intx     bool
-	hasbegin bool
-	kvs      []*types.KeyValue
-	txid     *types.Int64
-	client   queue.Client
-	api      client.QueueProtocolAPI
+	cache        map[string][]byte
+	txcache      map[string][]byte
+	keys         []string
+	intx         bool
+	hasbegin     bool
+	kvs          []*types.KeyValue
+	txid         *types.Int64
+	client       queue.Client
+	api          client.QueueProtocolAPI
+	disableread  bool
+	disablewrite bool
 }
 
 //NewLocalDB 创建一个新的LocalDB
@@ -41,6 +43,26 @@ func NewLocalDB(cli queue.Client) db.KVDB {
 	}
 }
 
+//DisableRead 禁止读取LocalDB数据库
+func (l *LocalDB) DisableRead() {
+	l.disableread = true
+}
+
+//DisableWrite 禁止写LocalDB数据库
+func (l *LocalDB) DisableWrite() {
+	l.disablewrite = true
+}
+
+//EnableRead 启动读取LocalDB数据库
+func (l *LocalDB) EnableRead() {
+	l.disableread = false
+}
+
+//EnableWrite 启动写LocalDB数据库
+func (l *LocalDB) EnableWrite() {
+	l.disablewrite = false
+}
+
 func (l *LocalDB) resetTx() {
 	l.intx = false
 	l.txcache = nil
@@ -128,6 +150,9 @@ func (l *LocalDB) Rollback() {
 
 //Get 获取key
 func (l *LocalDB) Get(key []byte) ([]byte, error) {
+	if l.disableread {
+		return nil, types.ErrDisableRead
+	}
 	skey := string(key)
 	if l.intx && l.txcache != nil {
 		if value, ok := l.txcache[skey]; ok {
@@ -160,6 +185,9 @@ func (l *LocalDB) Get(key []byte) ([]byte, error) {
 
 //Set 获取key
 func (l *LocalDB) Set(key []byte, value []byte) error {
+	if l.disablewrite {
+		return types.ErrDisableWrite
+	}
 	skey := string(key)
 	if l.intx {
 		if l.txcache == nil {
@@ -176,6 +204,9 @@ func (l *LocalDB) Set(key []byte, value []byte) error {
 
 // List 从数据库中查询数据列表
 func (l *LocalDB) List(prefix, key []byte, count, direction int32) ([][]byte, error) {
+	if l.disableread {
+		return nil, types.ErrDisableRead
+	}
 	err := l.save()
 	if err != nil {
 		return nil, err

vendor/github.com/33cn/chain33/executor/localdb_test.go

@@ -18,6 +18,34 @@ func TestLocalDBGet(t *testing.T) {
 	testDBGet(t, db)
 }
 
+func TestLocalDBEnable(t *testing.T) {
+	mock33 := testnode.New("", nil)
+	defer mock33.Close()
+	db := executor.NewLocalDB(mock33.GetClient())
+	ldb := db.(*executor.LocalDB)
+	defer ldb.Close()
+	_, err := ldb.Get([]byte("hello"))
+	assert.Equal(t, err, types.ErrNotFound)
+	ldb.DisableRead()
+	_, err = ldb.Get([]byte("hello"))
+
+	assert.Equal(t, err, types.ErrDisableRead)
+	_, err = ldb.List(nil, nil, 0, 0)
+	assert.Equal(t, err, types.ErrDisableRead)
+	ldb.EnableRead()
+	_, err = ldb.Get([]byte("hello"))
+	assert.Equal(t, err, types.ErrNotFound)
+	_, err = ldb.List(nil, nil, 0, 0)
+	assert.Equal(t, err, nil)
+	ldb.DisableWrite()
+	err = ldb.Set([]byte("hello"), nil)
+	assert.Equal(t, err, types.ErrDisableWrite)
+	ldb.EnableWrite()
+	err = ldb.Set([]byte("hello"), nil)
+	assert.Equal(t, err, nil)
+
+}
+
 func BenchmarkLocalDBGet(b *testing.B) {
 	mock33 := testnode.New("", nil)
 	defer mock33.Close()

vendor/github.com/33cn/chain33/system/dapp/manage/types/types.go

@@ -20,7 +20,7 @@ var (
 	}
 	logmap = map[int64]*types.LogInfo{
 		// 这里reflect.TypeOf类型必须是proto.Message类型，且是交易的回持结构
-		TyLogModifyConfig: {reflect.TypeOf(types.ReceiptConfig{}), "LogModifyConfig"},
+		TyLogModifyConfig: {Ty: reflect.TypeOf(types.ReceiptConfig{}), Name: "LogModifyConfig"},
 	}
 )
 

vendor/github.com/33cn/chain33/system/store/mavl/db/memmavl.go

+// Copyright Fuzamei Corp. 2018 All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mavl
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/33cn/chain33/common"
+	dbm "github.com/33cn/chain33/common/db"
+	farm "github.com/dgryski/go-farm"
+	"github.com/hashicorp/golang-lru"
+)
+
+// MemTreeOpera memtree操作接口
+type MemTreeOpera interface {
+	Add(key, value interface{})
+	Get(key interface{}) (value interface{}, ok bool)
+	Delete(key interface{})
+	Contains(key interface{}) bool
+	Len() int
+}
+
+// TreeMap map形式memtree
+type TreeMap struct {
+	mpCache map[interface{}]interface{}
+}
+
+// NewTreeMap new mem tree
+func NewTreeMap(size int) *TreeMap {
+	mp := &TreeMap{}
+	mp.mpCache = make(map[interface{}]interface{}, size)
+	return mp
+}
+
+// Add 添加元素
+func (tm *TreeMap) Add(key, value interface{}) {
+	if _, ok := tm.mpCache[key]; ok {
+		delete(tm.mpCache, key)
+		return
+	}
+	tm.mpCache[key] = value
+}
+
+// Get 获取元素
+func (tm *TreeMap) Get(key interface{}) (value interface{}, ok bool) {
+	if value, ok := tm.mpCache[key]; ok {
+		return value, ok
+	}
+	return nil, false
+}
+
+// Delete 删除元素
+func (tm *TreeMap) Delete(key interface{}) {
+	if _, ok := tm.mpCache[key]; ok {
+		delete(tm.mpCache, key)
+	}
+}
+
+// Contains 查看是否包含元素
+func (tm *TreeMap) Contains(key interface{}) bool {
+	if _, ok := tm.mpCache[key]; ok {
+		return true
+	}
+	return false
+}
+
+// Len 元素长度
+func (tm *TreeMap) Len() int {
+	return len(tm.mpCache)
+}
+
+// TreeARC lru的mem tree
+type TreeARC struct {
+	arcCache *lru.ARCCache
+}
+
+// NewTreeARC new lru mem tree
+func NewTreeARC(size int) *TreeARC {
+	ma := &TreeARC{}
+	ma.arcCache, _ = lru.NewARC(size)
+	return ma
+}
+
+// Add 添加元素
+func (ta *TreeARC) Add(key, value interface{}) {
+	if ta.arcCache.Contains(key) {
+		ta.arcCache.Remove(key)
+		return
+	}
+	ta.arcCache.Add(key, value)
+}
+
+// Get 获取元素
+func (ta *TreeARC) Get(key interface{}) (value interface{}, ok bool) {
+	return ta.arcCache.Get(key)
+}
+
+// Delete 删除元素
+func (ta *TreeARC) Delete(key interface{}) {
+	ta.arcCache.Remove(key)
+}
+
+// Contains 查看是否包含元素
+func (ta *TreeARC) Contains(key interface{}) bool {
+	return ta.arcCache.Contains(key)
+}
+
+// Len 元素长度
+func (ta *TreeARC) Len() int {
+	return ta.arcCache.Len()
+}
+
+// LoadTree2MemDb 从数据库中载入mem tree
+func LoadTree2MemDb(db dbm.DB, hash []byte, mp map[uint64]struct{}) {
+	nDb := newNodeDB(db, true)
+	node, err := nDb.getLightNode(nil, hash)
+	if err != nil {
+		fmt.Println("err", err)
+		return
+	}
+	pri := ""
+	if len(node.hash) > 32 {
+		pri = string(node.hash[:16])
+	}
+	treelog.Info("hash node", "hash pri", pri, "hash", common.ToHex(node.hash), "height", node.height)
+	start := time.Now()
+	leftHash := make([]byte, len(node.leftHash))
+	copy(leftHash, node.leftHash)
+	rightHash := make([]byte, len(node.rightHash))
+	copy(rightHash, node.rightHash)
+	mp[farm.Hash64(node.hash)] = struct{}{}
+	node.loadNodeInfo(nDb, mp)
+	end := time.Now()
+	treelog.Info("hash node", "cost time", end.Sub(start), "node count", len(mp))
+	PrintMemStats(1)
+}
+
+func (node *Node) loadNodeInfo(db *nodeDB, mp map[uint64]struct{}) {
+	if node.height == 0 {
+		//trMem.Add(Hash64(node.hash), &hashNode{leftHash: node.leftHash, rightHash: node.rightHash})
+		leftHash := make([]byte, len(node.leftHash))
+		copy(leftHash, node.leftHash)
+		rightHash := make([]byte, len(node.rightHash))
+		copy(rightHash, node.rightHash)
+		mp[farm.Hash64(node.hash)] = struct{}{}
+		return
+	}
+	if node.leftHash != nil {
+		left, err := db.getLightNode(nil, node.leftHash)
+		if err != nil {
+			return
+		}
+		//trMem.Add(Hash64(left.hash), &hashNode{leftHash: left.leftHash, rightHash: left.rightHash})
+		leftHash := make([]byte, len(left.leftHash))
+		copy(leftHash, left.leftHash)
+		rightHash := make([]byte, len(left.rightHash))
+		copy(rightHash, left.rightHash)
+		mp[farm.Hash64(left.hash)] = struct{}{}
+		left.loadNodeInfo(db, mp)
+	}
+	if node.rightHash != nil {
+		right, err := db.getLightNode(nil, node.rightHash)
+		if err != nil {
+			return
+		}
+		//trMem.Add(Hash64(right.hash), &hashNode{leftHash: right.leftHash, rightHash: right.rightHash})
+		leftHash := make([]byte, len(right.leftHash))
+		copy(leftHash, right.leftHash)
+		rightHash := make([]byte, len(right.rightHash))
+		copy(rightHash, right.rightHash)
+		mp[farm.Hash64(right.hash)] = struct{}{}
+		right.loadNodeInfo(db, mp)
+	}
+}
+
+func (ndb *nodeDB) getLightNode(t *Tree, hash []byte) (*Node, error) {
+	// Doesn't exist, load from db.
+	var buf []byte
+	buf, err := ndb.db.Get(hash)
+
+	if len(buf) == 0 || err != nil {
+		return nil, ErrNodeNotExist
+	}
+	node, err := MakeNode(buf, t)
+	if err != nil {
+		panic(fmt.Sprintf("Error reading IAVLNode. bytes: %X  error: %v", buf, err))
+	}
+	node.hash = hash
+	node.key = nil
+	node.value = nil
+	return node, nil
+}
+
+func copyBytes(b []byte) (copiedBytes []byte) {
+	if b == nil {
+		return nil
+	}
+	copiedBytes = make([]byte, len(b))
+	copy(copiedBytes, b)
+	return copiedBytes
+}

vendor/github.com/33cn/chain33/system/store/mavl/db/node.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/33cn/chain33/common"
 	"github.com/33cn/chain33/types"
+	farm "github.com/dgryski/go-farm"
 	"github.com/golang/protobuf/proto"
 )
 
@@ -499,6 +500,9 @@ func removeOrphan(t *Tree, node *Node) {
 	if t.ndb == nil {
 		return
 	}
+	if enableMemTree && t != nil {
+		t.obsoleteNode[uintkey(farm.Hash64(node.hash))] = struct{}{}
+	}
 	t.ndb.RemoveNode(t, node)
 }
 

vendor/github.com/33cn/chain33/system/store/mavl/db/tree.go

@@ -15,6 +15,7 @@ import (
 	dbm "github.com/33cn/chain33/common/db"
 	log "github.com/33cn/chain33/common/log/log15"
 	"github.com/33cn/chain33/types"
+	farm "github.com/dgryski/go-farm"
 	"github.com/golang/protobuf/proto"
 	"github.com/hashicorp/golang-lru"
 )
@@ -37,6 +38,10 @@ var (
 	// 当前树的最大高度
 	maxBlockHeight int64
 	heightMtx      sync.Mutex
+	//
+	enableMemTree bool
+	memTree       MemTreeOpera
+	enableMemVal  bool
 )
 
 // EnableMavlPrefix 使能mavl加前缀
@@ -49,23 +54,53 @@ func EnableMVCC(enable bool) {
 	enableMvcc = enable
 }
 
+// EnableMemTree 使能mavl数据载入内存
+func EnableMemTree(enable bool) {
+	enableMemTree = enable
+}
+
+// EnableMemVal 使能mavl叶子节点数据载入内存
+func EnableMemVal(enable bool) {
+	enableMemVal = enable
+}
+
+type memNode struct {
+	data   [][]byte //顺序为lefthash, righthash, key, value
+	Height int32
+	Size   int32
+}
+
+type uintkey uint64
+
 // Tree merkle avl tree
 type Tree struct {
 	root        *Node
 	ndb         *nodeDB
 	blockHeight int64
+	// 树更新之后，废弃的节点(更新缓存中节点先对旧节点进行删除然后再更新)
+	obsoleteNode map[uintkey]struct{}
+	updateNode   map[uintkey]*memNode
 }
 
 // NewTree 新建一个merkle avl 树
 func NewTree(db dbm.DB, sync bool) *Tree {
 	if db == nil {
 		// In-memory IAVLTree
-		return &Tree{}
+		return &Tree{
+			obsoleteNode: make(map[uintkey]struct{}),
+			updateNode:   make(map[uintkey]*memNode),
+		}
 	}
 	// Persistent IAVLTree
 	ndb := newNodeDB(db, sync)
+	// 使能情况下非空创建当前整tree的缓存
+	if enableMemTree && memTree == nil {
+		memTree = NewTreeMap(50 * 10000)
+	}
 	return &Tree{
-		ndb: ndb,
+		ndb:          ndb,
+		obsoleteNode: make(map[uintkey]struct{}),
+		updateNode:   make(map[uintkey]*memNode),
 	}
 }
 
@@ -116,13 +151,17 @@ func (t *Tree) Has(key []byte) bool {
 func (t *Tree) Set(key []byte, value []byte) (updated bool) {
 	//treelog.Info("IAVLTree.Set", "key", key, "value",value)
 	if t.root == nil {
-		t.root = NewNode(key, value)
+		t.root = NewNode(copyBytes(key), copyBytes(value))
 		return false
 	}
-	t.root, updated = t.root.set(t, key, value)
+	t.root, updated = t.root.set(t, copyBytes(key), copyBytes(value))
 	return updated
 }
 
+func (t *Tree) getObsoleteNode() map[uintkey]struct{} {
+	return t.obsoleteNode
+}
+
 // Hash 计算tree 的roothash
 func (t *Tree) Hash() []byte {
 	if t.root == nil {
@@ -148,6 +187,17 @@ func (t *Tree) Save() []byte {
 		if enablePrune {
 			t.root.saveRootHash(t)
 		}
+		// 更新memTree
+		if enableMemTree && memTree != nil {
+			for k := range t.obsoleteNode {
+				memTree.Delete(k)
+			}
+			for k, v := range t.updateNode {
+				memTree.Add(k, v)
+			}
+			treelog.Debug("Tree.Save", "memTree len", memTree.Len(), "tree height", t.blockHeight)
+		}
+
 		beg := types.Now()
 		err := t.ndb.Commit()
 		treelog.Info("tree.commit", "cost", types.Since(beg))
@@ -395,6 +445,13 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 			return elem.(*Node), nil
 		}
 	}
+	//从memtree中获取
+	if enableMemTree {
+		node, err := getNode4MemTree(hash)
+		if err == nil {
+			return node, nil
+		}
+	}
 	// Doesn't exist, load from db.
 	var buf []byte
 	buf, err := ndb.db.Get(hash)
@@ -409,6 +466,10 @@ func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
 	node.hash = hash
 	node.persisted = true
 	ndb.cacheNode(node)
+	// Save node hashInt64 to memTree
+	if enableMemTree {
+		updateGlobalMemTree(node)
+	}
 	return node, nil
 }
 
@@ -459,6 +520,85 @@ func (ndb *nodeDB) SaveNode(t *Tree, node *Node) {
 	ndb.cacheNode(node)
 	delete(ndb.orphans, string(node.hash))
 	//treelog.Debug("SaveNode", "hash", node.hash, "height", node.height, "value", node.value)
+	// Save node hashInt64 to localmem
+	if enableMemTree {
+		updateLocalMemTree(t, node)
+	}
+}
+
+func getNode4MemTree(hash []byte) (*Node, error) {
+	if memTree == nil {
+		return nil, ErrNodeNotExist
+	}
+	elem, ok := memTree.Get(uintkey(farm.Hash64(hash)))
+	if ok {
+		sn := elem.(*memNode)
+		node := &Node{
+			height:    sn.Height,
+			size:      sn.Size,
+			hash:      hash,
+			persisted: true,
+		}
+		node.leftHash = sn.data[0]
+		node.rightHash = sn.data[1]
+		node.key = sn.data[2]
+		if len(sn.data) == 4 {
+			node.value = sn.data[3]
+		}
+		return node, nil
+	}
+	return nil, ErrNodeNotExist
+}
+
+// Save node hashInt64 to memTree
+func updateGlobalMemTree(node *Node) {
+	if node == nil || memTree == nil {
+		return
+	}
+	if !enableMemVal && node.height == 0 {
+		return
+	}
+	memN := &memNode{
+		Height: node.height,
+		Size:   node.size,
+	}
+	if node.height == 0 {
+		memN.data = make([][]byte, 4)
+		memN.data[3] = node.value
+	} else {
+		memN.data = make([][]byte, 3)
+	}
+	memN.data[0] = node.leftHash
+	memN.data[1] = node.rightHash
+	memN.data[2] = node.key
+	memTree.Add(uintkey(farm.Hash64(node.hash)), memN)
+}
+
+// Save node hashInt64 to localmem
+func updateLocalMemTree(t *Tree, node *Node) {
+	if t == nil || node == nil {
+		return
+	}
+	if !enableMemVal && node.height == 0 {
+		return
+	}
+	if t.updateNode != nil {
+		memN := &memNode{
+			Height: node.height,
+			Size:   node.size,
+		}
+		if node.height == 0 {
+			memN.data = make([][]byte, 4)
+			memN.data[3] = node.value
+		} else {
+			memN.data = make([][]byte, 3)
+		}
+		memN.data[0] = node.leftHash
+		memN.data[1] = node.rightHash
+		memN.data[2] = node.key
+		t.updateNode[uintkey(farm.Hash64(node.hash))] = memN
+		//treelog.Debug("Tree.SaveNode", "store struct size", unsafe.Sizeof(store), "byte size", len(storenode), "height", node.height)
+	}
 }
 
 //cache缓存节点

vendor/github.com/33cn/chain33/system/store/mavl/db/tree_test.go

@@ -15,6 +15,8 @@ import (
 	"sync"
 	"testing"
 
+	"unsafe"
+
 	. "github.com/33cn/chain33/common"
 	"github.com/33cn/chain33/common/db"
 	"github.com/33cn/chain33/common/log"
@@ -1255,6 +1257,89 @@ func TestDelLeafCountKV(t *testing.T) {
 	}
 }
 
+func TestGetObsoleteNode(t *testing.T) {
+	dir, err := ioutil.TempDir("", "datastore")
+	require.NoError(t, err)
+	t.Log(dir)
+	defer os.Remove(dir)
+
+	EnableMemTree(true)
+	EnableMemVal(true)
+	defer EnableMemTree(false)
+	defer EnableMemVal(false)
+
+	db := db.NewDB("mavltree", "leveldb", dir, 100)
+	tree := NewTree(db, true)
+
+	type record struct {
+		key   string
+		value string
+	}
+	records := []record{
+		{"abc", "abc"},
+		{"low", "low"},
+		{"fan", "fan"},
+	}
+
+	for _, r := range records {
+		updated := tree.Set([]byte(r.key), []byte(r.value))
+		if updated {
+			t.Error("should have not been updated")
+		}
+	}
+	hash := tree.Save()
+	obs := tree.getObsoleteNode()
+	require.Equal(t, 0, len(obs))
+	mp := make(map[uint64]struct{})
+	LoadTree2MemDb(db, hash, mp)
+
+	tree1 := NewTree(db, true)
+	tree1.Load(hash)
+	records1 := []record{
+		{"abc", "abc1"},
+		{"low", "low1"},
+		{"fan", "fan1"},
+	}
+
+	for _, r := range records1 {
+		tree1.Set([]byte(r.key), []byte(r.value))
+	}
+	hash1 := tree1.Save()
+	obs = tree1.getObsoleteNode()
+	mp1 := make(map[uint64]struct{})
+	LoadTree2MemDb(db, hash1, mp1)
+	require.Equal(t, len(mp), len(obs)) //做了全部更新，因此旧节点全部删除
+	for ob := range obs {
+		_, ok := mp[uint64(ob)]
+		if !ok {
+			require.Error(t, fmt.Errorf("should exist"))
+		}
+	}
+
+	tree2 := NewTree(db, true)
+	tree2.Load(hash)
+	records2 := []record{
+		{"fan", "fan1"},
+		{"foo", "foo"},
+		{"foobaz", "foobaz"},
+		{"good", "good"},
+	}
+	for _, r := range records2 {
+		tree2.Set([]byte(r.key), []byte(r.value))
+	}
+	hash2 := tree2.Save()
+	obs = tree2.getObsoleteNode()
+	mp2 := make(map[uint64]struct{})
+	LoadTree2MemDb(db, hash2, mp2)
+	//require.Equal(t, 0, len(obs))
+	for ob := range obs {
+		_, ok := mp[uint64(ob)]
+		if !ok {
+			require.Error(t, fmt.Errorf("should exist"))
+		}
+	}
+}
+
 func TestPruningFirstLevelNode(t *testing.T) {
 	dir, err := ioutil.TempDir("", "datastore")
 	require.NoError(t, err)
@@ -1899,3 +1984,64 @@ func saveBlock(dbm db.DB, height int64, hash []byte, txN int64, mvcc bool) (newH
 	}
 	return newHash, nil
 }
+
+func TestSize1(t *testing.T) {
+	type storeNode struct {
+		Key       []byte
+		Value     []byte
+		LeftHash  []byte
+		RightHash []byte
+		Height    int32
+		Size      int32
+	}
+	type storeNode1 struct {
+		Key    [][]byte
+		Height int32
+		Size   int32
+	}
+	a := types.StoreNode{}
+	b := storeNode{}
+	var c []byte
+	d := storeNode1{}
+
+	//arcmp := NewTreeMap(350 * 10000)
+	//if arcmp == nil {
+	//	return
+	//}
+
+	//for i := 0; i < 300*10000; i++ {
+	//	data := &storeNode{
+	//		Key: []byte("12345678901234567890123456789012"),
+	//		Value: []byte("12345678901234567890123456789012"),
+	//		LeftHash: []byte("12345678901234567890123456789012"),
+	//		RightHash: []byte("12345678901234567890123456789012"),
+	//		//Key: copyBytes([]byte("12345678901234567890123456789012")),
+	//		//Value: copyBytes([]byte("12345678901234567890123456789012")),
+	//		//LeftHash: copyBytes([]byte("12345678901234567890123456789012")),
+	//		//RightHash: copyBytes([]byte("12345678901234567890123456789012")),
+	//		Height: 123,
+	//		Size: 123,
+	//	}
+	//	arcmp.Add(int64(i), data)
+	//}
+
+	//for i := 0; i < 100*10000; i++ {
+	//	data := &storeNode1{}
+	//	data.Height = 123
+	//	data.Size = 123
+	//	d.Key = make([][]byte, 4)
+	//	//d.Key[0] = []byte("12345678901234567890123456789012")
+	//	//d.Key[1] = []byte("12345678901234567890123456789012")
+	//	//d.Key[2] = []byte("12345678901234567890123456789012")
+	//	//d.Key[3] = []byte("12345678901234567890123456789012")
+	//
+	//	d.Key[0] = copyBytes([]byte("12345678901234567890123456789012"))
+	//	d.Key[1] = copyBytes([]byte("12345678901234567890123456789012"))
+	//	d.Key[2] = copyBytes([]byte("12345678901234567890123456789012"))
+	//	d.Key[3] = copyBytes([]byte("12345678901234567890123456789012"))
+	//	arcmp.Add(int64(i), data)
+	//}
+
+	PrintMemStats(1)
+	fmt.Println(unsafe.Sizeof(a), unsafe.Sizeof(b), unsafe.Sizeof(c), unsafe.Sizeof(d), len(d.Key), cap(d.Key))
+}

vendor/github.com/33cn/chain33/system/store/mavl/mavl.go

@@ -37,6 +37,8 @@ type Store struct {
 	enableMVCC       bool
 	enableMavlPrune  bool
 	pruneHeight      int32
+	enableMemTree    bool
+	enableMemVal     bool
 }
 
 func init() {
@@ -51,6 +53,10 @@ type subConfig struct {
 	EnableMavlPrune bool `json:"enableMavlPrune"`
 	// 裁剪高度间隔
 	PruneHeight int32 `json:"pruneHeight"`
+	// 是否使能内存树
+	EnableMemTree bool `json:"enableMemTree"`
+	// 是否使能内存树中叶子节点
+	EnableMemVal bool `json:"enableMemVal"`
 }
 
 // New new mavl store module
@@ -60,15 +66,20 @@ func New(cfg *types.Store, sub []byte) queue.Module {
 	if sub != nil {
 		types.MustDecode(sub, &subcfg)
 	}
-	mavls := &Store{bs, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC, subcfg.EnableMavlPrune, subcfg.PruneHeight}
+	mavls := &Store{bs, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC,
+		subcfg.EnableMavlPrune, subcfg.PruneHeight, subcfg.EnableMemTree, subcfg.EnableMemVal}
 	mavls.enableMavlPrefix = subcfg.EnableMavlPrefix
 	mavls.enableMVCC = subcfg.EnableMVCC
 	mavls.enableMavlPrune = subcfg.EnableMavlPrune
 	mavls.pruneHeight = subcfg.PruneHeight
+	mavls.enableMemTree = subcfg.EnableMemTree
+	mavls.enableMemVal = subcfg.EnableMemVal
 	mavl.EnableMavlPrefix(mavls.enableMavlPrefix)
 	mavl.EnableMVCC(mavls.enableMVCC)
 	mavl.EnablePrune(mavls.enableMavlPrune)
 	mavl.SetPruneHeight(int(mavls.pruneHeight))
+	mavl.EnableMemTree(mavls.enableMemTree)
+	mavl.EnableMemVal(mavls.enableMemVal)
 	bs.SetChild(mavls)
 	return mavls
 }

vendor/github.com/33cn/chain33/types/error.go

@@ -181,4 +181,7 @@ var (
 	ErrHeightOverflow      = errors.New("ErrHeightOverflow")
 	ErrRecordBlockSequence = errors.New("ErrRecordBlockSequence")
 	ErrExecPanic           = errors.New("ErrExecPanic")
+
+	ErrDisableWrite = errors.New("ErrDisableWrite")
+	ErrDisableRead  = errors.New("ErrDisableRead")
 )

vendor/github.com/33cn/chain33/types/proto/account.proto

@@ -80,4 +80,4 @@ message ReqAllExecBalance {
     string stateHash    = 3;
     string asset_exec   = 4;
     string asset_symbol = 5;
-}
+}
\ No newline at end of file

vendor/github.com/33cn/chain33/util/util.go

@@ -225,8 +225,9 @@ func ExecBlock(client queue.Client, prevStateRoot []byte, block *types.Block, er
 	//通过consensus module 再次检查
 	ulog.Debug("ExecBlock", "height------->", block.Height, "ntx", len(block.Txs))
 	beg := types.Now()
+	beg2 := beg
 	defer func() {
-		ulog.Info("ExecBlock", "height", block.Height, "ntx", len(block.Txs), "writebatchsync", sync, "cost", types.Since(beg))
+		ulog.Info("ExecBlock", "height", block.Height, "ntx", len(block.Txs), "writebatchsync", sync, "cost", types.Since(beg2))
 	}()
 
 	if errReturn && block.Height > 0 && !block.CheckSign() {
@@ -330,7 +331,6 @@ func ExecBlock(client queue.Client, prevStateRoot []byte, block *types.Block, er
 		}
 	}
 	ulog.Info("ExecBlock", "CheckBlock", types.Since(beg))
-	beg = types.Now()
 	// 写数据库失败时需要及时返回错误，防止错误数据被写入localdb中CHAIN33-567
 	err = ExecKVSetCommit(client, block.StateHash)
 	if err != nil {