Merge remote-tracking branch 'upstream/master'

Makefile

@@ -14,7 +14,7 @@ LDFLAGS := -ldflags "-w -s"
 PKG_LIST_VET := `go list ./... | grep -v "vendor" | grep -v plugin/dapp/evm/executor/vm/common/crypto/bn256`
 PKG_LIST := `go list ./... | grep -v "vendor" | grep -v "chain33/test" | grep -v "mocks" | grep -v "pbft"`
 PKG_LIST_INEFFASSIGN= `go list -f {{.Dir}} ./... | grep -v "vendor"`
-BUILD_FLAGS = -ldflags "-X github.com/33cn/chain33/common/version.GitCommit=`git rev-parse --short=8 HEAD`"
+BUILD_FLAGS = -ldflags "-X github.com/33cn/plugin/vendor/github.com/33cn/chain33/common/version.GitCommit=`git rev-parse --short=8 HEAD`"
 MKPATH=$(abspath $(lastword $(MAKEFILE_LIST)))
 MKDIR=$(dir $(MKPATH))
 proj := "build"
@@ -23,8 +23,8 @@ proj := "build"
 default: build depends
 
 build:
-	@go build $(BUILD_FLAGS) -v -i -o $(APP)
-	@go build -v -i -o $(CLI) $(SRC_CLI)
+	go build $(BUILD_FLAGS) -v -i -o $(APP)
+	go build $(BUILD_FLAGS) -v -i -o $(CLI) $(SRC_CLI)
 	@cp chain33.toml $(CHAIN33_PATH)/build/system-test-rpc.sh build/
 
 build_ci: depends ## Build the binary file for CI

chain33.toml

 Title="chain33"
 TestNet=true
 FixTime=false
-version="6.1.1"
+version="6.2.0"
 
 [log]
 # 日志级别，支持debug(dbug)/info/warn/error(eror)/crit
@@ -152,6 +152,8 @@ pruneHeight=10000
 enableMemTree=true
 # 是否使能mavl叶子节点数据载入内存
 enableMemVal=true
+# 缓存close ticket数目，该缓存越大同步速度越快，最大设置到1500000
+tkCloseCacheLen=100000
 
 [store.sub.kvmvccmavl]
 enableMVCCIter=true
@@ -163,6 +165,8 @@ pruneHeight=10000
 enableMemTree=true
 # 是否使能mavl叶子节点数据载入内存
 enableMemVal=true
+# 缓存close ticket数目，该缓存越大同步速度越快，最大设置到1500000
+tkCloseCacheLen=100000
 
 [wallet]
 minFee=100000

plugin/mempool/para/base.go

@@ -55,7 +55,7 @@ func (mem *Mempool) SetQueueClient(client queue.Client) {
 				tx := msg.GetData().(*types.Transaction)
 				reply, err = mem.mainGrpcCli.SendTransaction(context.Background(), tx)
 			case types.EventGetProperFee:
-				reply, err = mem.mainGrpcCli.GetProperFee(context.Background(), &types.ReqNil{})
+				reply, err = mem.mainGrpcCli.GetProperFee(context.Background(), &types.ReqProperFee{})
 			default:
 				msg.Reply(client.NewMessage(mem.key, types.EventReply, types.ErrActionNotSupport))
 			}

plugin/store/kvmvccmavl/kvmvcc_mavl.go

@@ -87,6 +87,8 @@ type subMavlConfig struct {
 	EnableMemTree bool `json:"enableMemTree"`
 	// 是否使能内存树中叶子节点
 	EnableMemVal bool `json:"enableMemVal"`
+	// 缓存close ticket数目
+	TkCloseCacheLen int32 `json:"tkCloseCacheLen"`
 }
 
 type subConfig struct {
@@ -99,6 +101,8 @@ type subConfig struct {
 	EnableMemTree bool `json:"enableMemTree"`
 	// 是否使能内存树中叶子节点
 	EnableMemVal bool `json:"enableMemVal"`
+	// 缓存close ticket数目
+	TkCloseCacheLen int32 `json:"tkCloseCacheLen"`
 }
 
 // New construct KVMVCCStore module
@@ -119,6 +123,7 @@ func New(cfg *types.Store, sub []byte) queue.Module {
 		subMavlcfg.PruneHeight = subcfg.PruneHeight
 		subMavlcfg.EnableMemTree = subcfg.EnableMemTree
 		subMavlcfg.EnableMemVal = subcfg.EnableMemVal
+		subMavlcfg.TkCloseCacheLen = subcfg.TkCloseCacheLen
 	}
 
 	bs := drivers.NewBaseStore(cfg)

plugin/store/kvmvccmavl/mavl.go

@@ -42,6 +42,7 @@ func NewMavl(sub *subMavlConfig, db dbm.DB) *MavlStore {
 		subcfg.PruneHeight = sub.PruneHeight
 		subcfg.EnableMemTree = sub.EnableMemTree
 		subcfg.EnableMemVal = sub.EnableMemVal
+		subcfg.TkCloseCacheLen = sub.TkCloseCacheLen
 	}
 	mavls := &MavlStore{db, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC, subcfg.EnableMavlPrune, subcfg.PruneHeight}
 	mavl.EnableMavlPrefix(subcfg.EnableMavlPrefix)
@@ -50,6 +51,7 @@ func NewMavl(sub *subMavlConfig, db dbm.DB) *MavlStore {
 	mavl.SetPruneHeight(int(subcfg.PruneHeight))
 	mavl.EnableMemTree(subcfg.EnableMemTree)
 	mavl.EnableMemVal(subcfg.EnableMemVal)
+	mavl.TkCloseCacheLen(subcfg.TkCloseCacheLen)
 	return mavls
 }
 

vendor/github.com/33cn/chain33/blockchain/blockstore.go

@@ -1239,13 +1239,19 @@ func (bs *BlockStore) SetStoreUpgradeMeta(meta *types.UpgradeMeta) error {
 	return bs.db.SetSync(version.StoreDBMeta, verByte)
 }
 
-//SequenceMustValid 配置的合法性检测
-func (bs *BlockStore) SequenceMustValid(recordSequence bool) {
+const (
+	seqStatusOk = iota
+	seqStatusNeedCreate
+	seqStatusNeedDelete
+)
+
+//CheckSequenceStatus 配置的合法性检测
+func (bs *BlockStore) CheckSequenceStatus(recordSequence bool) int {
 	lastHeight := bs.Height()
 	lastSequence, err := bs.LoadBlockLastSequence()
 	if err != nil {
 		if err != types.ErrHeightNotExist {
-			storeLog.Error("SequenceMustValid", "LoadBlockLastSequence err", err)
+			storeLog.Error("CheckSequenceStatus", "LoadBlockLastSequence err", err)
 			panic(err)
 		}
 	}
@@ -1253,32 +1259,140 @@ func (bs *BlockStore) SequenceMustValid(recordSequence bool) {
 	if recordSequence {
 		//中途开启isRecordBlockSequence报错
 		if lastSequence == -1 && lastHeight != -1 {
-			storeLog.Error("SequenceMustValid", "lastHeight", lastHeight, "lastSequence", lastSequence)
-			panic("isRecordBlockSequence is true must Synchronizing data from zero block")
+			storeLog.Error("CheckSequenceStatus", "lastHeight", lastHeight, "lastSequence", lastSequence)
+			return seqStatusNeedCreate
 		}
 		//lastSequence 必须大于等于lastheight
 		if lastHeight > lastSequence {
-			storeLog.Error("SequenceMustValid", "lastHeight", lastHeight, "lastSequence", lastSequence)
-			panic("lastSequence must greater than or equal to lastHeight")
+			storeLog.Error("CheckSequenceStatus", "lastHeight", lastHeight, "lastSequence", lastSequence)
+			return seqStatusNeedCreate
 		}
 		//通过lastSequence获取对应的blockhash ！= lastHeader.hash 报错
 		if lastSequence != -1 {
 			blockSequence, err := bs.GetBlockSequence(lastSequence)
 			if err != nil {
-				storeLog.Error("SequenceMustValid", "lastSequence", lastSequence, "GetBlockSequence err", err)
+				storeLog.Error("CheckSequenceStatus", "lastSequence", lastSequence, "GetBlockSequence err", err)
 				panic(err)
 			}
 			lastHeader := bs.LastHeader()
 			if !bytes.Equal(lastHeader.Hash, blockSequence.Hash) {
-				storeLog.Error("SequenceMustValid:", "lastHeight", lastHeight, "lastSequence", lastSequence, "lastHeader.Hash", common.ToHex(lastHeader.Hash), "blockSequence.Hash", common.ToHex(blockSequence.Hash))
-				panic("The hash values of lastSequence and lastHeight are different.")
+				storeLog.Error("CheckSequenceStatus:", "lastHeight", lastHeight, "lastSequence", lastSequence, "lastHeader.Hash", common.ToHex(lastHeader.Hash), "blockSequence.Hash", common.ToHex(blockSequence.Hash))
+				return seqStatusNeedCreate
 			}
 		}
-		return
+		return seqStatusOk
 	}
 	//去使能isRecordBlockSequence时的检测
 	if lastSequence != -1 {
-		storeLog.Error("SequenceMustValid", "lastSequence", lastSequence)
-		panic("can not disable isRecordBlockSequence")
+		storeLog.Error("CheckSequenceStatus", "lastSequence", lastSequence)
+		return seqStatusNeedDelete
+	}
+	return seqStatusOk
+}
+
+//CreateSequences 根据高度生成sequence记录
+func (bs *BlockStore) CreateSequences(batchSize int64) {
+	lastSeq, err := bs.LoadBlockLastSequence()
+	if err != nil {
+		if err != types.ErrHeightNotExist {
+			storeLog.Error("CreateSequences LoadBlockLastSequence", "error", err)
+			panic("CreateSequences LoadBlockLastSequence" + err.Error())
+		}
+	}
+	storeLog.Info("CreateSequences LoadBlockLastSequence", "start", lastSeq)
+
+	newBatch := bs.NewBatch(true)
+	lastHeight := bs.Height()
+
+	for i := lastSeq + 1; i <= lastHeight; i++ {
+		seq := i
+		header, err := bs.GetBlockHeaderByHeight(i)
+		if err != nil {
+			storeLog.Error("CreateSequences GetBlockHeaderByHeight", "height", i, "error", err)
+			panic("CreateSequences GetBlockHeaderByHeight" + err.Error())
+		}
+
+		// seq->hash
+		var blockSequence types.BlockSequence
+		blockSequence.Hash = header.Hash
+		blockSequence.Type = AddBlock
+		BlockSequenceByte, err := proto.Marshal(&blockSequence)
+		if err != nil {
+			storeLog.Error("CreateSequences Marshal BlockSequence", "height", i, "hash", common.ToHex(header.Hash), "error", err)
+			panic("CreateSequences Marshal BlockSequence" + err.Error())
+		}
+		newBatch.Set(calcSequenceToHashKey(seq, bs.isParaChain), BlockSequenceByte)
+
+		// hash -> seq
+		sequenceBytes := types.Encode(&types.Int64{Data: seq})
+		newBatch.Set(calcHashToSequenceKey(header.Hash, bs.isParaChain), sequenceBytes)
+
+		if i-lastSeq == batchSize {
+			storeLog.Info("CreateSequences ", "height", i)
+			newBatch.Set(calcLastSeqKey(bs.isParaChain), types.Encode(&types.Int64{Data: i}))
+			err = newBatch.Write()
+			if err != nil {
+				storeLog.Error("CreateSequences newBatch.Write", "error", err)
+				panic("CreateSequences newBatch.Write" + err.Error())
+			}
+			lastSeq = i
+			newBatch.Reset()
+		}
+	}
+	// last seq
+	newBatch.Set(calcLastSeqKey(bs.isParaChain), types.Encode(&types.Int64{Data: lastHeight}))
+	err = newBatch.Write()
+	if err != nil {
+		storeLog.Error("CreateSequences newBatch.Write", "error", err)
+		panic("CreateSequences newBatch.Write" + err.Error())
+	}
+	storeLog.Info("CreateSequences done")
+}
+
+//DeleteSequences 删除本地数据库里的sequence记录
+func (bs *BlockStore) DeleteSequences(batchSize int64) {
+	lastSeq, err := bs.LoadBlockLastSequence()
+	if err != nil {
+		if err != types.ErrHeightNotExist {
+			storeLog.Error("DeleteSequences LoadBlockLastSequence", "error", err)
+			panic("DeleteSequences LoadBlockLastSequence" + err.Error())
+		}
+	}
+	storeLog.Info("DeleteSequences LoadBlockLastSequence", "start", lastSeq)
+
+	newBatch := bs.NewBatch(true)
+
+	for i := lastSeq; i >= 0; i-- {
+		seq := i
+		header, err := bs.GetBlockHeaderByHeight(i)
+		if err != nil {
+			storeLog.Error("DeleteSequences GetBlockHeaderByHeight", "height", i, "error", err)
+			panic("DeleteSequences GetBlockHeaderByHeight" + err.Error())
+		}
+
+		// seq->hash
+		newBatch.Delete(calcSequenceToHashKey(seq, bs.isParaChain))
+		// hash -> seq
+		newBatch.Delete(calcHashToSequenceKey(header.Hash, bs.isParaChain))
+
+		if lastSeq-i == batchSize {
+			storeLog.Info("DeleteSequences ", "height", i)
+			newBatch.Set(calcLastSeqKey(bs.isParaChain), types.Encode(&types.Int64{Data: i - 1}))
+			err = newBatch.Write()
+			if err != nil {
+				storeLog.Error("DeleteSequences newBatch.Write", "error", err)
+				panic("DeleteSequences newBatch.Write" + err.Error())
+			}
+			lastSeq = i - 1
+			newBatch.Reset()
+		}
+	}
+	// last seq
+	newBatch.Delete(calcLastSeqKey(bs.isParaChain))
+	err = newBatch.Write()
+	if err != nil {
+		storeLog.Error("DeleteSequences newBatch.Write", "error", err)
+		panic("DeleteSequences newBatch.Write" + err.Error())
 	}
+	storeLog.Info("DeleteSequences done")
 }

vendor/github.com/33cn/chain33/blockchain/blockstore_test.go

@@ -6,7 +6,10 @@ import (
 	"io/ioutil"
 	"os"
 
+	"fmt"
+
 	dbm "github.com/33cn/chain33/common/db"
+	"github.com/33cn/chain33/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -135,3 +138,48 @@ func TestParaSeqSaveAndGet(t *testing.T) {
 	_, err = chain.ProcGetMainSeqByHash([]byte("s0-not-exist"))
 	assert.NotNil(t, err)
 }
+
+func TestSeqCreateAndDelete(t *testing.T) {
+	dir, err := ioutil.TempDir("", "example")
+	assert.Nil(t, err)
+	defer os.RemoveAll(dir) // clean up
+	os.RemoveAll(dir)       //删除已存在目录
+
+	blockStoreDB := dbm.NewDB("blockchain", "leveldb", dir, 100)
+
+	blockStore := NewBlockStore(nil, blockStoreDB, nil)
+	assert.NotNil(t, blockStore)
+	blockStore.saveSequence = false
+	blockStore.isParaChain = true
+
+	batch := blockStore.NewBatch(true)
+	for i := 0; i <= 100; i++ {
+		var header types.Header
+		h0 := calcHeightToBlockHeaderKey(int64(i))
+		header.Hash = []byte(fmt.Sprintf("%d", i))
+		types.Encode(&header)
+		batch.Set(h0, types.Encode(&header))
+	}
+	blockStore.height = 100
+	batch.Write()
+
+	blockStore.saveSequence = true
+	blockStore.CreateSequences(10)
+	seq, err := blockStore.LoadBlockLastSequence()
+	assert.Nil(t, err)
+	assert.Equal(t, int64(100), seq)
+
+	seq, err = blockStore.GetSequenceByHash([]byte("1"))
+	assert.Nil(t, err)
+	assert.Equal(t, int64(1), seq)
+
+	seq, err = blockStore.GetSequenceByHash([]byte("0"))
+	assert.Nil(t, err)
+	assert.Equal(t, int64(0), seq)
+
+	blockStore.saveSequence = false
+	blockStore.DeleteSequences(10)
+	seq, err = blockStore.LoadBlockLastSequence()
+	assert.NotNil(t, err)
+	assert.Equal(t, int64(-1), seq)
+}

vendor/github.com/33cn/chain33/blockchain/chain.go

@@ -18,7 +18,7 @@ import (
 	log "github.com/33cn/chain33/common/log/log15"
 	"github.com/33cn/chain33/queue"
 	"github.com/33cn/chain33/types"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 //var
@@ -245,7 +245,12 @@ func (chain *BlockChain) GetOrphanPool() *OrphanPool {
 //InitBlockChain 区块链初始化
 func (chain *BlockChain) InitBlockChain() {
 	//isRecordBlockSequence配置的合法性检测
-	chain.blockStore.SequenceMustValid(chain.isRecordBlockSequence)
+	seqStatus := chain.blockStore.CheckSequenceStatus(chain.isRecordBlockSequence)
+	if seqStatus == seqStatusNeedCreate {
+		chain.blockStore.CreateSequences(100000)
+	} else if seqStatus == seqStatusNeedDelete {
+		chain.blockStore.DeleteSequences(100000)
+	}
 
 	//先缓存最新的128个block信息到cache中
 	curheight := chain.GetBlockHeight()

vendor/github.com/33cn/chain33/build/system-test-rpc.sh

@@ -339,8 +339,12 @@ chain33_GetSeqCallBackLastNum() {
 }
 
 chain33_GetCoinSymbol() {
+    symbol="bty"
+    if [ "$IS_PARA" == true ]; then
+        symbol="paracoin"
+    fi
     r1=$(curl -ksd '{"method":"Chain33.GetCoinSymbol","params":[]}' ${MAIN_HTTP} | jq -r ".result.data")
-    [ "$r1" == "bty" ]
+    [ "$r1" == "$symbol" ]
     echo_rst "$FUNCNAME" "$?"
 }
 

vendor/github.com/33cn/chain33/client/mocks/api.go

@@ -522,13 +522,13 @@ func (_m *QueueProtocolAPI) GetNetInfo() (*types.NodeNetInfo, error) {
 	return r0, r1
 }
 
-// GetProperFee provides a mock function with given fields:
-func (_m *QueueProtocolAPI) GetProperFee() (*types.ReplyProperFee, error) {
-	ret := _m.Called()
+// GetProperFee provides a mock function with given fields: req
+func (_m *QueueProtocolAPI) GetProperFee(req *types.ReqProperFee) (*types.ReplyProperFee, error) {
+	ret := _m.Called(req)
 
 	var r0 *types.ReplyProperFee
-	if rf, ok := ret.Get(0).(func() *types.ReplyProperFee); ok {
-		r0 = rf()
+	if rf, ok := ret.Get(0).(func(*types.ReqProperFee) *types.ReplyProperFee); ok {
+		r0 = rf(req)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*types.ReplyProperFee)
@@ -536,8 +536,8 @@ func (_m *QueueProtocolAPI) GetProperFee() (*types.ReplyProperFee, error) {
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func() error); ok {
-		r1 = rf()
+	if rf, ok := ret.Get(1).(func(*types.ReqProperFee) error); ok {
+		r1 = rf(req)
 	} else {
 		r1 = ret.Error(1)
 	}

vendor/github.com/33cn/chain33/client/queueprotocol.go

@@ -469,8 +469,8 @@ func (q *QueueProtocol) GetLastMempool() (*types.ReplyTxList, error) {
 }
 
 // GetProperFee get proper fee from mempool
-func (q *QueueProtocol) GetProperFee() (*types.ReplyProperFee, error) {
-	msg, err := q.query(mempoolKey, types.EventGetProperFee, &types.ReqNil{})
+func (q *QueueProtocol) GetProperFee(req *types.ReqProperFee) (*types.ReplyProperFee, error) {
+	msg, err := q.query(mempoolKey, types.EventGetProperFee, req)
 	if err != nil {
 		log.Error("GetProperFee", "Error", err.Error())
 		return nil, err

vendor/github.com/33cn/chain33/client/queueprotocol_test.go

@@ -343,7 +343,11 @@ func testGetLastMempool(t *testing.T, api client.QueueProtocolAPI) {
 }
 
 func testGetProperFee(t *testing.T, api client.QueueProtocolAPI) {
-	_, err := api.GetProperFee()
+	_, err := api.GetProperFee(nil)
+	if err != nil {
+		t.Error("Call GetProperFee Failed.", err)
+	}
+	_, err = api.GetProperFee(&types.ReqProperFee{})
 	if err != nil {
 		t.Error("Call GetProperFee Failed.", err)
 	}
@@ -989,7 +993,7 @@ func testGetLastMemPoolGRPC(t *testing.T, rpc *mockGRPCSystem) {
 
 func testGetProperFeeGRPC(t *testing.T, rpc *mockGRPCSystem) {
 	var res types.ReplyProperFee
-	err := rpc.newRpcCtx("GetProperFee", &types.ReqNil{}, &res)
+	err := rpc.newRpcCtx("GetProperFee", &types.ReqProperFee{}, &res)
 	if err != nil {
 		t.Error("Call GetProperFee Failed.", err)
 	}

vendor/github.com/33cn/chain33/client/queueprotocolapi.go

@@ -25,7 +25,7 @@ type QueueProtocolAPI interface {
 	// types.EventGetLastMempool
 	GetLastMempool() (*types.ReplyTxList, error)
 	// types.EventGetProperFee
-	GetProperFee() (*types.ReplyProperFee, error)
+	GetProperFee(req *types.ReqProperFee) (*types.ReplyProperFee, error)
 	// +++++++++++++++ execs interfaces begin
 	// types.EventBlockChainQuery
 	Query(driver, funcname string, param types.Message) (types.Message, error)

vendor/github.com/33cn/chain33/client/rpc_ctx_test.go

@@ -220,7 +220,7 @@ func (c *GrpcCtx) Run() (err error) {
 		}
 		errRet = err
 	case "GetProperFee":
-		reply, err := rpc.GetProperFee(context.Background(), c.Params.(*types.ReqNil))
+		reply, err := rpc.GetProperFee(context.Background(), c.Params.(*types.ReqProperFee))
 		if err == nil {
 			*c.Res.(*types.ReplyProperFee) = *reply
 		}

vendor/github.com/33cn/chain33/cmd/chain33/bityuan.toml

@@ -98,6 +98,8 @@ pruneHeight=10000
 enableMemTree=false
 # 是否使能mavl叶子节点数据载入内存
 enableMemVal=false
+# 缓存close ticket数目，该缓存越大同步速度越快，最大设置到1500000
+tkCloseCacheLen=100000
 
 [wallet]
 # walletdb路径

vendor/github.com/33cn/chain33/cmd/chain33/chain33.test.toml

@@ -152,6 +152,8 @@ pruneHeight=10000
 enableMemTree=false
 # 是否使能mavl叶子节点数据载入内存
 enableMemVal=false
+# 缓存close ticket数目，该缓存越大同步速度越快，最大设置到1500000
+tkCloseCacheLen=100000
 
 [wallet]
 minFee=1000000

vendor/github.com/33cn/chain33/cmd/chain33/chain33.toml

@@ -226,6 +226,8 @@ pruneHeight=10000
 enableMemTree=false
 # 是否使能mavl叶子节点数据载入内存
 enableMemVal=false
+# 缓存close ticket数目，该缓存越大同步速度越快，最大设置到1500000
+tkCloseCacheLen=100000
 
 [wallet]
 # 交易发送最低手续费，单位0.00000001BTY(1e-8),默认100000，即0.001BTY

vendor/github.com/33cn/chain33/common/db/mvcc_test.go

@@ -195,8 +195,9 @@ func TestAddDelMVCC(t *testing.T) {
 	_, err = m.AddMVCC(genkv(2), hashN(2), hashN(1), 1)
 	assert.Equal(t, err, types.ErrPrevVersion)
 
+	//hash 2 还不存在
 	_, err = m.AddMVCC(genkv(2), hashN(2), hashN(0), 3)
-	assert.Equal(t, err, types.ErrPrevVersion)
+	assert.Equal(t, err, types.ErrNotFound)
 
 	_, err = m.AddMVCC(genkv(2), hashN(2), hashN(3), 3)
 	assert.Equal(t, err, types.ErrNotFound)

vendor/github.com/33cn/chain33/p2p/nat/nat.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+
 	//"log"
 	"net"
 	"strings"
@@ -32,7 +33,7 @@ import (
 	"time"
 
 	//"github.com/ethereum/go-ethereum/log"
-	"github.com/jackpal/go-nat-pmp"
+	natpmp "github.com/jackpal/go-nat-pmp"
 )
 
 // Interface An implementation of nat.Interface can map local ports to ports

vendor/github.com/33cn/chain33/p2p/nat/natpmp.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/jackpal/go-nat-pmp"
+	natpmp "github.com/jackpal/go-nat-pmp"
 )
 
 // natPMPClient adapts the NAT-PMP protocol implementation so it conforms to

vendor/github.com/33cn/chain33/rpc/grpchandler.go

@@ -206,8 +206,8 @@ func (g *Grpc) GetLastMemPool(ctx context.Context, in *pb.ReqNil) (*pb.ReplyTxLi
 }
 
 // GetProperFee return last mempool proper fee
-func (g *Grpc) GetProperFee(ctx context.Context, in *pb.ReqNil) (*pb.ReplyProperFee, error) {
-	return g.cli.GetProperFee()
+func (g *Grpc) GetProperFee(ctx context.Context, in *pb.ReqProperFee) (*pb.ReplyProperFee, error) {
+	return g.cli.GetProperFee(in)
 }
 
 // GetBlockOverview get block overview

vendor/github.com/33cn/chain33/rpc/grpchandler_test.go

@@ -203,11 +203,11 @@ func TestGetLastMemPool(t *testing.T) {
 }
 
 func testGetProperFeeOK(t *testing.T) {
-	qapi.On("GetProperFee").Return(nil, nil)
-	data, err := g.GetProperFee(getOkCtx(), nil)
+	var in *types.ReqProperFee
+	qapi.On("GetProperFee", in).Return(nil, nil)
+	data, err := g.GetProperFee(getOkCtx(), in)
 	assert.Nil(t, err, "the error should be nil")
 	assert.Nil(t, data)
-
 }
 
 func TestGetProperFee(t *testing.T) {

vendor/github.com/33cn/chain33/rpc/jrpchandler.go

@@ -614,8 +614,8 @@ func (c *Chain33) GetLastMemPool(in types.ReqNil, result *interface{}) error {
 }
 
 // GetProperFee get  contents in proper fee
-func (c *Chain33) GetProperFee(in types.ReqNil, result *interface{}) error {
-	reply, err := c.cli.GetProperFee()
+func (c *Chain33) GetProperFee(in types.ReqProperFee, result *interface{}) error {
+	reply, err := c.cli.GetProperFee(&in)
 	if err != nil {
 		return err
 	}

vendor/github.com/33cn/chain33/rpc/jrpchandler_test.go

@@ -929,12 +929,11 @@ func TestChain33_GetProperFee(t *testing.T) {
 	api := new(mocks.QueueProtocolAPI)
 	testChain33 := newTestChain33(api)
 
-	// expected := &types.ReqBlocks{}
-	api.On("GetProperFee").Return(nil, errors.New("error value"))
+	expected := types.ReqProperFee{}
+	api.On("GetProperFee", &expected).Return(nil, errors.New("error value"))
 
 	var testResult interface{}
-	actual := types.ReqNil{}
-	err := testChain33.GetProperFee(actual, &testResult)
+	err := testChain33.GetProperFee(expected, &testResult)
 	t.Log(err)
 	assert.Equal(t, nil, testResult)
 	assert.NotNil(t, err)

vendor/github.com/33cn/chain33/system/mempool/base.go

@@ -282,10 +282,16 @@ func (mem *Mempool) RemoveTxsOfBlock(block *types.Block) bool {
 }
 
 // GetProperFeeRate 获取合适的手续费率
-func (mem *Mempool) GetProperFeeRate() int64 {
+func (mem *Mempool) GetProperFeeRate(req *types.ReqProperFee) int64 {
+	if req == nil || req.TxCount == 0 {
+		req = &types.ReqProperFee{TxCount: 20}
+	}
+	if req.TxSize == 0 {
+		req.TxSize = 10240
+	}
 	baseFeeRate := mem.cache.GetProperFee()
 	if mem.cfg.IsLevelFee {
-		levelFeeRate := mem.getLevelFeeRate(mem.cfg.MinTxFee)
+		levelFeeRate := mem.getLevelFeeRate(mem.cfg.MinTxFee, req.TxCount, req.TxSize)
 		if levelFeeRate > baseFeeRate {
 			return levelFeeRate
 		}
@@ -293,15 +299,15 @@ func (mem *Mempool) GetProperFeeRate() int64 {
 	return baseFeeRate
 }
 
-// getLevelFeeRate 获取合适的阶梯手续费率
-func (mem *Mempool) getLevelFeeRate(baseFeeRate int64) int64 {
+// getLevelFeeRate 获取合适的阶梯手续费率, 可以外部传入count, size进行前瞻性估计
+func (mem *Mempool) getLevelFeeRate(baseFeeRate int64, appendCount, appendSize int32) int64 {
 	var feeRate int64
-	sumByte := mem.cache.TotalByte()
+	sumByte := mem.cache.TotalByte() + int64(appendSize)
 	maxTxNumber := types.GetP(mem.Height()).MaxTxNumber
 	switch {
-	case sumByte >= int64(types.MaxBlockSize/20) || int64(mem.Size()) >= maxTxNumber/2:
+	case sumByte >= int64(types.MaxBlockSize/20) || int64(mem.Size()+int(appendCount)) >= maxTxNumber/2:
 		feeRate = 100 * baseFeeRate
-	case sumByte >= int64(types.MaxBlockSize/100) || int64(mem.Size()) >= maxTxNumber/10:
+	case sumByte >= int64(types.MaxBlockSize/100) || int64(mem.Size()+int(appendCount)) >= maxTxNumber/10:
 		feeRate = 10 * baseFeeRate
 	default:
 		feeRate = baseFeeRate

vendor/github.com/33cn/chain33/system/mempool/check.go

@@ -124,7 +124,7 @@ func (mem *Mempool) checkTxs(msg *queue.Message) *queue.Message {
 // checkLevelFee 检查阶梯手续费
 func (mem *Mempool) checkLevelFee(tx *types.TransactionCache) error {
 	//获取mempool里所有交易手续费总和
-	feeRate := mem.getLevelFeeRate(mem.cfg.MinTxFee)
+	feeRate := mem.getLevelFeeRate(mem.cfg.MinTxFee, 0, 0)
 	totalfee, err := tx.GetTotalFee(feeRate)
 	if err != nil {
 		return err

vendor/github.com/33cn/chain33/system/mempool/eventprocess.go

@@ -194,7 +194,8 @@ func (mem *Mempool) eventGetAddrTxs(msg *queue.Message) {
 
 // eventGetProperFee 获取排队策略中合适的手续费率
 func (mem *Mempool) eventGetProperFee(msg *queue.Message) {
-	properFee := mem.GetProperFeeRate()
+	req, _ := msg.GetData().(*types.ReqProperFee)
+	properFee := mem.GetProperFeeRate(req)
 	msg.Reply(mem.client.NewMessage("rpc", types.EventReplyProperFee,
 		&types.ReplyProperFee{ProperFee: properFee}))
 }

vendor/github.com/33cn/chain33/system/mempool/mempool_test.go

@@ -624,10 +624,26 @@ func TestGetLatestTx(t *testing.T) {
 	}
 }
 
+func testProperFee(t *testing.T, client queue.Client, req *types.ReqProperFee, expectFee int64) int64 {
+	msg := client.NewMessage("mempool", types.EventGetProperFee, req)
+	client.Send(msg, true)
+	reply, err := client.Wait(msg)
+	if err != nil {
+		t.Error(err)
+		return 0
+	}
+	fee := reply.GetData().(*types.ReplyProperFee).GetProperFee()
+	assert.Equal(t, expectFee, fee)
+	return fee
+}
+
 func TestGetProperFee(t *testing.T) {
 	q, mem := initEnv(0)
 	defer q.Close()
 	defer mem.Close()
+	defer func() {
+		mem.cfg.IsLevelFee = false
+	}()
 
 	// add 10 txs
 	err := add10Tx(mem.client)
@@ -635,24 +651,24 @@ func TestGetProperFee(t *testing.T) {
 		t.Error("add tx error", err.Error())
 		return
 	}
-
+	maxTxNum := types.GetP(mem.Height()).MaxTxNumber
+	maxSize := types.MaxBlockSize
 	msg11 := mem.client.NewMessage("mempool", types.EventTx, tx11)
 	mem.client.Send(msg11, true)
 	mem.client.Wait(msg11)
 
-	msg := mem.client.NewMessage("mempool", types.EventGetProperFee, nil)
-	mem.client.Send(msg, true)
-
-	reply, err := mem.client.Wait(msg)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if reply.GetData().(*types.ReplyProperFee).GetProperFee() != mem.cfg.MinTxFee {
-		t.Error("TestGetProperFee failed", reply.GetData().(*types.ReplyProperFee).GetProperFee(), mem.cfg.MinTxFee)
-	}
+	baseFee := testProperFee(t, mem.client, nil, mem.cfg.MinTxFee)
+	mem.cfg.IsLevelFee = true
+	testProperFee(t, mem.client, nil, baseFee)
+	testProperFee(t, mem.client, &types.ReqProperFee{}, baseFee)
+	//more than 1/2 max num
+	testProperFee(t, mem.client, &types.ReqProperFee{TxCount: int32(maxTxNum / 2)}, 100*baseFee)
+	//more than 1/10 max num
+	testProperFee(t, mem.client, &types.ReqProperFee{TxCount: int32(maxTxNum / 10)}, 10*baseFee)
+	//more than 1/20 max size
+	testProperFee(t, mem.client, &types.ReqProperFee{TxCount: 1, TxSize: int32(maxSize / 20)}, 100*baseFee)
+	//more than 1/100 max size
+	testProperFee(t, mem.client, &types.ReqProperFee{TxCount: 1, TxSize: int32(maxSize / 100)}, 10*baseFee)
 }
 
 func TestCheckLowFee(t *testing.T) {

vendor/github.com/33cn/chain33/system/store/mavl/db/memmavl.go

@@ -12,7 +12,7 @@ import (
 	"github.com/33cn/chain33/common"
 	dbm "github.com/33cn/chain33/common/db"
 	farm "github.com/dgryski/go-farm"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 // MemTreeOpera memtree操作接口

vendor/github.com/33cn/chain33/system/store/mavl/db/tree.go

@@ -18,7 +18,7 @@ import (
 	"github.com/33cn/chain33/types"
 	farm "github.com/dgryski/go-farm"
 	"github.com/golang/protobuf/proto"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 const (
@@ -40,10 +40,11 @@ var (
 	maxBlockHeight int64
 	heightMtx      sync.Mutex
 	//
-	enableMemTree bool
-	enableMemVal  bool
-	memTree       MemTreeOpera
-	tkCloseCache  MemTreeOpera
+	enableMemTree   bool
+	enableMemVal    bool
+	memTree         MemTreeOpera
+	tkCloseCache    MemTreeOpera
+	tkCloseCacheLen int32 = 10 * 10000
 )
 
 // EnableMavlPrefix 使能mavl加前缀
@@ -66,6 +67,11 @@ func EnableMemVal(enable bool) {
 	enableMemVal = enable
 }
 
+// TkCloseCacheLen 设置缓存close ticket数目
+func TkCloseCacheLen(len int32) {
+	tkCloseCacheLen = len
+}
+
 // ReleaseGlobalMem 释放全局缓存
 func ReleaseGlobalMem() {
 	if memTree != nil {
@@ -110,7 +116,7 @@ func NewTree(db dbm.DB, sync bool) *Tree {
 	// 使能情况下非空创建当前整tree的缓存
 	if enableMemTree && memTree == nil {
 		memTree = NewTreeMap(50 * 10000)
-		tkCloseCache = NewTreeARC(10 * 10000)
+		tkCloseCache = NewTreeARC(int(tkCloseCacheLen))
 	}
 	return &Tree{
 		ndb:          ndb,

vendor/github.com/33cn/chain33/system/store/mavl/mavl.go

@@ -39,6 +39,7 @@ type Store struct {
 	pruneHeight      int32
 	enableMemTree    bool
 	enableMemVal     bool
+	tkCloseCacheLen  int32
 }
 
 func init() {
@@ -57,6 +58,8 @@ type subConfig struct {
 	EnableMemTree bool `json:"enableMemTree"`
 	// 是否使能内存树中叶子节点
 	EnableMemVal bool `json:"enableMemVal"`
+	// 缓存close ticket数目
+	TkCloseCacheLen int32 `json:"tkCloseCacheLen"`
 }
 
 // New new mavl store module
@@ -67,19 +70,21 @@ func New(cfg *types.Store, sub []byte) queue.Module {
 		types.MustDecode(sub, &subcfg)
 	}
 	mavls := &Store{bs, &sync.Map{}, subcfg.EnableMavlPrefix, subcfg.EnableMVCC,
-		subcfg.EnableMavlPrune, subcfg.PruneHeight, subcfg.EnableMemTree, subcfg.EnableMemVal}
+		subcfg.EnableMavlPrune, subcfg.PruneHeight, subcfg.EnableMemTree, subcfg.EnableMemVal, subcfg.TkCloseCacheLen}
 	mavls.enableMavlPrefix = subcfg.EnableMavlPrefix
 	mavls.enableMVCC = subcfg.EnableMVCC
 	mavls.enableMavlPrune = subcfg.EnableMavlPrune
 	mavls.pruneHeight = subcfg.PruneHeight
 	mavls.enableMemTree = subcfg.EnableMemTree
 	mavls.enableMemVal = subcfg.EnableMemVal
+	mavls.tkCloseCacheLen = subcfg.TkCloseCacheLen
 	mavl.EnableMavlPrefix(mavls.enableMavlPrefix)
 	mavl.EnableMVCC(mavls.enableMVCC)
 	mavl.EnablePrune(mavls.enableMavlPrune)
 	mavl.SetPruneHeight(int(mavls.pruneHeight))
 	mavl.EnableMemTree(mavls.enableMemTree)
 	mavl.EnableMemVal(mavls.enableMemVal)
+	mavl.TkCloseCacheLen(mavls.tkCloseCacheLen)
 	bs.SetChild(mavls)
 	return mavls
 }

vendor/github.com/33cn/chain33/types/mocks/chain33client.go

@@ -763,7 +763,7 @@ func (_m *Chain33Client) GetPeerInfo(ctx context.Context, in *types.ReqNil, opts
 }
 
 // GetProperFee provides a mock function with given fields: ctx, in, opts
-func (_m *Chain33Client) GetProperFee(ctx context.Context, in *types.ReqNil, opts ...grpc.CallOption) (*types.ReplyProperFee, error) {
+func (_m *Chain33Client) GetProperFee(ctx context.Context, in *types.ReqProperFee, opts ...grpc.CallOption) (*types.ReplyProperFee, error) {
 	_va := make([]interface{}, len(opts))
 	for _i := range opts {
 		_va[_i] = opts[_i]
@@ -774,7 +774,7 @@ func (_m *Chain33Client) GetProperFee(ctx context.Context, in *types.ReqNil, opt
 	ret := _m.Called(_ca...)
 
 	var r0 *types.ReplyProperFee
-	if rf, ok := ret.Get(0).(func(context.Context, *types.ReqNil, ...grpc.CallOption) *types.ReplyProperFee); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, *types.ReqProperFee, ...grpc.CallOption) *types.ReplyProperFee); ok {
 		r0 = rf(ctx, in, opts...)
 	} else {
 		if ret.Get(0) != nil {
@@ -783,7 +783,7 @@ func (_m *Chain33Client) GetProperFee(ctx context.Context, in *types.ReqNil, opt
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func(context.Context, *types.ReqNil, ...grpc.CallOption) error); ok {
+	if rf, ok := ret.Get(1).(func(context.Context, *types.ReqProperFee, ...grpc.CallOption) error); ok {
 		r1 = rf(ctx, in, opts...)
 	} else {
 		r1 = ret.Error(1)

vendor/github.com/33cn/chain33/types/proto/rpc.proto

@@ -75,7 +75,7 @@ service chain33 {
     rpc GetLastMemPool(ReqNil) returns (ReplyTxList) {}
 
     //获取最新的ProperFee
-    rpc GetProperFee(ReqNil) returns (ReplyProperFee) {}
+    rpc GetProperFee(ReqProperFee) returns (ReplyProperFee) {}
 
     // 获取钱包状态
     rpc GetWalletStatus(ReqNil) returns (WalletStatus) {}

vendor/github.com/33cn/chain33/types/proto/transaction.proto

@@ -165,6 +165,11 @@ message ReplyTxList {
     repeated Transaction txs = 1;
 }
 
+message ReqProperFee {
+    int32 txCount = 1;
+    int32 txSize  = 2;
+}
+
 message ReplyProperFee {
     int64 properFee = 1;
 }

vendor/github.com/33cn/chain33/wallet/bipwallet/go-bip32/utils.go

@@ -15,7 +15,7 @@ import (
 	"io"
 
 	"github.com/33cn/chain33/wallet/bipwallet/basen"
-	"github.com/33cn/chain33/wallet/bipwallet/btcutilecc"
+	btcutil "github.com/33cn/chain33/wallet/bipwallet/btcutilecc"
 	"golang.org/x/crypto/ripemd160"
 )
 

vendor/github.com/33cn/chain33/wallet/seedtool/main.go

+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/33cn/chain33/common/address"
+
+	"github.com/33cn/chain33/types"
+	"github.com/33cn/chain33/wallet"
+	"github.com/33cn/chain33/wallet/bipwallet"
+
+	"net/http"
+	_ "net/http/pprof"
+)
+
+var seed = flag.String("seed", "", "source seed")
+var targetaddr = flag.String("addr", "", "address of target")
+var lang = flag.Int("lang", 1, "lang: 0 englist, 1 chinese")
+var oldseed = flag.Bool("oldseed", false, "is seed old")
+var accountnum = flag.Int("nacc", 5, "gen account count")
+
+func main() {
+	flag.Parse()
+	wallet.InitSeedLibrary()
+	log.Println("seed", *seed)
+	log.Println("target", *targetaddr)
+	go http.ListenAndServe("localhost:6060", nil)
+	seedlist := strings.Split(*seed, " ")
+	//第一种情况，用户写错一个字
+	n := 0
+	wordlist := wallet.ChineseSeedCache
+	if *lang == 0 {
+		wordlist = wallet.EnglishSeedCache
+	}
+	checkseed(*seed)
+	for k := range wordlist {
+		log.Println("change ", k, int(100*(float64(n)/float64(2048))))
+		n++
+		var seeds []string
+		for i := 0; i < len(seedlist); i++ {
+			item := seedlist[i]
+			seedlist[i] = k
+			newseed := strings.Join(seedlist, " ")
+			seeds = append(seeds, newseed)
+			seedlist[i] = item
+		}
+		checkmultithread(seeds)
+	}
+	log.Println("not found")
+}
+
+func checkmultithread(seeds []string) {
+	done := make(chan struct{}, len(seeds))
+	for i := 0; i < len(seeds); i++ {
+		go func(seed string) {
+			checkseed(seed)
+			done <- struct{}{}
+		}(seeds[i])
+	}
+	for i := 0; i < len(seeds); i++ {
+		<-done
+	}
+}
+
+func checkseed(newseed string) {
+	addrlist, err := genaddrlist(newseed)
+	if err != nil {
+		return
+	}
+	if _, ok := addrlist[*targetaddr]; ok {
+		log.Println("find new seed", newseed)
+		os.Exit(0)
+	}
+}
+func genaddrlist(seed string) (map[string]bool, error) {
+	var wallet *bipwallet.HDWallet
+	var err error
+	if *oldseed {
+		wallet, err = bipwallet.NewWalletFromSeed(bipwallet.TypeBty, []byte(seed))
+		if err != nil {
+			log.Println("GetPrivkeyBySeed NewWalletFromSeed", "err", err)
+			return nil, types.ErrNewWalletFromSeed
+		}
+	} else {
+		wallet, err = bipwallet.NewWalletFromMnemonic(bipwallet.TypeBty, seed)
+		if err != nil {
+			//log.Println("GetPrivkeyBySeed NewWalletFromMnemonic", "err", err)
+			wallet, err = bipwallet.NewWalletFromSeed(bipwallet.TypeBty, []byte(seed))
+			if err != nil {
+				log.Println("GetPrivkeyBySeed NewWalletFromSeed", "err", err)
+				return nil, types.ErrNewWalletFromSeed
+			}
+		}
+	}
+	addrlist := make(map[string]bool)
+	for index := 0; index <= *accountnum; index++ {
+		//通过索引生成Key pair
+		_, pub, err := childkey(wallet, uint32(index))
+		if err != nil {
+			log.Println("GetPrivkeyBySeed NewKeyPair", "err", err)
+			return nil, types.ErrNewKeyPair
+		}
+		addr := address.PubKeyToAddress(pub)
+		addrlist[addr.String()] = true
+	}
+	return addrlist, err
+}
+
+func childkey(w *bipwallet.HDWallet, index uint32) (priv, pub []byte, err error) {
+	if *oldseed {
+		key, err := w.MasterKey.NewChildKey(index)
+		if err != nil {
+			return nil, nil, err
+		}
+		return key.Key, key.PublicKey().Key, err
+	}
+	return w.NewKeyPair(index)
+}

vendor/github.com/33cn/chain33/wallet/sendtx.go

@@ -113,10 +113,11 @@ func (wallet *Wallet) sendTransaction(payload types.Message, execer []byte, priv
 	}
 	tx := &types.Transaction{Execer: execer, Payload: types.Encode(payload), Fee: minFee, To: to}
 	tx.Nonce = rand.Int63()
-	tx.Fee, err = tx.GetRealFee(wallet.getFee())
+	proper, err := wallet.api.GetProperFee(nil)
 	if err != nil {
 		return nil, err
 	}
+	tx.Fee = proper.ProperFee
 	tx.SetExpire(time.Second * 120)
 	tx.Sign(int32(SignType), priv)
 	reply, err := wallet.sendTx(tx)
@@ -226,11 +227,11 @@ func (wallet *Wallet) createSendToAddress(addrto string, amount int64, note stri
 		return nil, err
 	}
 	tx.SetExpire(time.Second * 120)
-	fee, err := tx.GetRealFee(wallet.getFee())
+	proper, err := wallet.api.GetProperFee(nil)
 	if err != nil {
 		return nil, err
 	}
-	tx.Fee = fee
+	tx.Fee = proper.ProperFee
 	if tx.To == "" {
 		tx.To = addrto
 	}

vendor/github.com/33cn/chain33/wallet/wallet_test.go

@@ -35,7 +35,6 @@ func initEnv() (*Wallet, queue.Module, queue.Queue) {
 
 	wallet := New(cfg.Wallet, sub.Wallet)
 	wallet.SetQueueClient(q.Client())
-
 	store := store.New(cfg.Store, sub.Store)
 	store.SetQueueClient(q.Client())
 
@@ -133,6 +132,8 @@ func mempoolModProc(q queue.Queue) {
 			//walletlog.Info("mempool", "msg.Ty", msg.Ty)
 			if msg.Ty == types.EventTx {
 				msg.Reply(client.NewMessage("wallet", types.EventReply, &types.Reply{IsOk: true}))
+			} else if msg.Ty == types.EventGetProperFee {
+				msg.Reply(client.NewMessage("wallet", types.EventReply, &types.ReplyProperFee{ProperFee: 1000000}))
 			}
 		}
 	}()