commit tree improve

plugin/dapp/mix/cmd/build/testcase.sh

 #!/usr/bin/env bash
 
-#1ks returner
+#1ks returner chain31
 MIX_CLI1="docker exec ${NODE1} /root/chain33-cli "
-#1jr  authorize
+#1jr  authorize chain32
 MIX_CLI2="docker exec ${NODE2} /root/chain33-cli "
-#1nl receiver
+#1nl receiver  chain30
 MIX_CLI3="docker exec ${NODE4} /root/chain33-cli "
 
 xsedfix=""

plugin/dapp/mix/commands/mix.go

@@ -202,6 +202,7 @@ func QueryCmd() *cobra.Command {
 	cmd.AddCommand(GetTreePathCmd())
 	cmd.AddCommand(GetTreeLeavesCmd())
 	cmd.AddCommand(GetTreeRootsCmd())
+	cmd.AddCommand(GetTreeStatusCmd())
 	cmd.AddCommand(ShowMixTxsCmd())
 	cmd.AddCommand(ShowPaymentPubKeyCmd())
 	return cmd
@@ -267,7 +268,7 @@ func treeLeaves(cmd *cobra.Command, args []string) {
 
 	var params rpctypes.Query4Jrpc
 	params.Execer = mixTy.MixX
-	params.FuncName = "Query_GetLeavesList"
+	params.FuncName = "GetLeavesList"
 	req := mixTy.TreeInfoReq{
 		RootHash: root,
 	}
@@ -285,20 +286,52 @@ func GetTreeRootsCmd() *cobra.Command {
 		Short: "Get archive roots",
 		Run:   treeRoot,
 	}
-	//addGetPathCmdFlags(cmd)
+	addGetRootsflags(cmd)
 	return cmd
 }
 
+func addGetRootsflags(cmd *cobra.Command) {
+	cmd.Flags().Int64P("seq", "s", 0, "sequence, default 0 is for current status")
+	cmd.MarkFlagRequired("seq")
+
+}
+
 func treeRoot(cmd *cobra.Command, args []string) {
 	rpcLaddr, _ := cmd.Flags().GetString("rpc_laddr")
+	seq, _ := cmd.Flags().GetInt64("seq")
 
 	var params rpctypes.Query4Jrpc
 	params.Execer = mixTy.MixX
 	params.FuncName = "GetRootList"
 
+	params.Payload = types.MustPBToJSON(&types.ReqInt{Height: seq})
+
+	var res mixTy.RootListResp
+	ctx := jsonclient.NewRPCCtx(rpcLaddr, "Chain33.Query", params, &res)
+	ctx.Run()
+}
+
+// GetTreeStatusCmd get commit leaves tree status
+func GetTreeStatusCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "status",
+		Short: "Get commit leaves tree status",
+		Run:   treeStatus,
+	}
+	//addGetRootsflags(cmd)
+	return cmd
+}
+
+func treeStatus(cmd *cobra.Command, args []string) {
+	rpcLaddr, _ := cmd.Flags().GetString("rpc_laddr")
+
+	var params rpctypes.Query4Jrpc
+	params.Execer = mixTy.MixX
+	params.FuncName = "GetTreeStatus"
+
 	params.Payload = types.MustPBToJSON(&types.ReqNil{})
 
-	var res mixTy.TreeListResp
+	var res mixTy.TreeStatusResp
 	ctx := jsonclient.NewRPCCtx(rpcLaddr, "Chain33.Query", params, &res)
 	ctx.Run()
 }

plugin/dapp/mix/executor/action.go

@@ -7,8 +7,6 @@ package executor
 import (
 	"github.com/33cn/chain33/account"
 	"github.com/33cn/chain33/client"
-	"github.com/consensys/gurvy/bn256/fr"
-
 	dbm "github.com/33cn/chain33/common/db"
 	"github.com/33cn/chain33/system/dapp"
 
@@ -80,15 +78,3 @@ func makeReceipt(key []byte, logTy int32, data proto.Message) *types.Receipt {
 		},
 	}
 }
-
-func transferFr2Bytes(v string) []byte {
-	var leaf fr.Element
-	leaf.SetString(v)
-	return leaf.Bytes()
-}
-
-func transferFr2String(val []byte) string {
-	var leaf fr.Element
-	leaf.SetBytes(val)
-	return leaf.String()
-}

plugin/dapp/mix/executor/authorize.go

@@ -17,13 +17,17 @@ import (
 
 func (a *action) authParamCheck(input *mixTy.AuthorizePublicInput) error {
 	//check tree rootHash exist
-	if !checkTreeRootHashExist(a.db, transferFr2Bytes(input.TreeRootHash)) {
+	exist, err := checkTreeRootHashExist(a.db, mixTy.Str2Byte(input.TreeRootHash))
+	if err != nil {
+		return errors.Wrapf(err, "roothash=%s not found", input.TreeRootHash)
+	}
+	if !exist {
 		return errors.Wrapf(mixTy.ErrTreeRootHashNotFound, "roothash=%s", input.TreeRootHash)
 	}
 
 	//authorize key should not exist
 	authKey := calcAuthorizeHashKey(input.AuthorizeHash)
-	_, err := a.db.Get(authKey)
+	_, err = a.db.Get(authKey)
 	if err == nil {
 		return errors.Wrapf(mixTy.ErrAuthorizeHashExist, "auth=%s", input.AuthorizeHash)
 	}

plugin/dapp/mix/executor/deposit.go

@@ -110,7 +110,7 @@ func (a *action) Deposit(deposit *mixTy.MixDepositAction) (*types.Receipt, error
 	//push new commit to merkle tree
 	var leaves [][]byte
 	for _, n := range notes {
-		leaves = append(leaves, transferFr2Bytes(n))
+		leaves = append(leaves, mixTy.Str2Byte(n))
 	}
 	rpt, err := pushTree(a.db, leaves)
 	if err != nil {

plugin/dapp/mix/executor/kv.go

@@ -9,27 +9,33 @@ import (
 )
 
 var (
-	verifyKeys              string
-	authPubKeys             string
-	paymentPubKey           string
+	verifyKeys    string
+	authPubKeys   string
+	paymentPubKey string
+
+	commitTreeCurrentStatus string
+	commitTreeSubRoots      string
+	commitTreeSubLeaves     string
 	commitTreeArchiveRoots  string
-	commitTreeCurrentRoots  string
-	commitTreeCurrentLeaves string
 	commitTreeRootLeaves    string
-	authorizeHash           string
-	authorizeSpendHash      string
-	nullifierHash           string
+
+	authorizeHash      string
+	authorizeSpendHash string
+	nullifierHash      string
 )
 
 func setPrefix() {
 	verifyKeys = "mavl-mix-verify-keys-"
 	authPubKeys = "mavl-mix-auth-pubkeys-"
 	paymentPubKey = "mavl-mix-payment-pubkey-"
-	commitTreeArchiveRoots = "mavl-mix-commitTree-roots-archive-"
-	commitTreeCurrentRoots = "mavl-mix-commitTree-roots-current-"
-	commitTreeCurrentLeaves = "mavl-mix-commitTree-current-leaves-"
 
-	commitTreeRootLeaves = "mavl-mix-commitTree-rootLeaves-"
+	commitTreeCurrentStatus = "mavl-mix-commitTree-current-status-"
+
+	commitTreeSubRoots = "mavl-mix-commitTree-sub-roots-"
+	commitTreeSubLeaves = "mavl-mix-commitTree-sub-leaves-"
+	commitTreeRootLeaves = "mavl-mix-commitTree-root-Leaves-"
+	commitTreeArchiveRoots = "mavl-mix-commitTree-archive-roots-"
+
 	authorizeHash = "mavl-mix-authorizeHash"
 	authorizeSpendHash = "mavl-mix-authorizeSpendHash-"
 	nullifierHash = "mavl-mix-nullifierHash"
@@ -49,16 +55,20 @@ func getPaymentPubKey(addr string) []byte {
 	return []byte(fmt.Sprintf(paymentPubKey+"%s", addr))
 }
 
-func calcCommitTreeArchiveRootsKey() []byte {
-	return []byte(fmt.Sprintf(commitTreeArchiveRoots))
+func calcCommitTreeCurrentStatusKey() []byte {
+	return []byte(fmt.Sprintf(commitTreeCurrentStatus))
+}
+
+func calcArchiveRootsKey(seq uint64) []byte {
+	return []byte(fmt.Sprintf(commitTreeArchiveRoots+"%022d", seq))
 }
 
-func calcCurrentCommitRootsKey() []byte {
-	return []byte(fmt.Sprintf(commitTreeCurrentRoots))
+func calcSubRootsKey(seq int32) []byte {
+	return []byte(fmt.Sprintf(commitTreeSubRoots+"%010d", seq))
 }
 
-func calcCurrentCommitLeavesKey() []byte {
-	return []byte(fmt.Sprintf(commitTreeCurrentLeaves))
+func calcSubLeavesKey(seq int32) []byte {
+	return []byte(fmt.Sprintf(commitTreeSubLeaves+"%010d", seq))
 }
 
 func calcCommitTreeRootLeaves(rootHash string) []byte {

plugin/dapp/mix/executor/merkletree/tree.go

@@ -22,7 +22,7 @@ type Tree struct {
 	// tree. When a new leaf is inserted, it is inserted as a subtree of height
 	// 0. If there is another subtree of the same height, both can be removed,
 	// combined, and then inserted as a subtree of height n + 1.
-	head *subTree
+	head *SubTree
 	hash hash.Hash
 
 	// Helper variables used to construct proofs that the data at 'proofIndex'
@@ -45,8 +45,8 @@ type Tree struct {
 // A subTree contains the Merkle root of a complete (2^height leaves) subTree
 // of the Tree. 'sum' is the Merkle root of the subTree. If 'next' is not nil,
 // it will be a tree with a higher height.
-type subTree struct {
-	next   *subTree
+type SubTree struct {
+	next   *SubTree
 	height int // Int is okay because a height over 300 is physically unachievable.
 	sum    []byte
 }
@@ -81,7 +81,7 @@ func nodeSum(h hash.Hash, a, b []byte) []byte {
 }
 
 // joinSubTrees combines two equal sized subTrees into a larger subTree.
-func joinSubTrees(h hash.Hash, a, b *subTree) *subTree {
+func joinSubTrees(h hash.Hash, a, b *SubTree) *SubTree {
 	// if DEBUG {
 	// 	if b.next != a {
 	// 		panic("invalid subtree join - 'a' is not paired with 'b'")
@@ -91,7 +91,7 @@ func joinSubTrees(h hash.Hash, a, b *subTree) *subTree {
 	// 	}
 	// }
 
-	return &subTree{
+	return &SubTree{
 		next:   a.next,
 		height: a.height + 1,
 		sum:    nodeSum(h, a.sum, b.sum),
@@ -190,7 +190,7 @@ func (t *Tree) Push(data []byte) {
 	// is going to be the data for cached trees, and is going to be the result
 	// of calling leafSum() on the data for standard trees. Doing a check here
 	// prevents needing to duplicate the entire 'Push' function for the trees.
-	t.head = &subTree{
+	t.head = &SubTree{
 		next:   t.head,
 		height: 0,
 	}
@@ -244,7 +244,7 @@ func (t *Tree) PushSubTree(height int, sum []byte) error {
 	}
 
 	// Insert the cached tree as the new head.
-	t.head = &subTree{
+	t.head = &SubTree{
 		height: height,
 		next:   t.head,
 		sum:    sum,
@@ -340,3 +340,30 @@ func (t *Tree) joinAllSubTrees() {
 		t.head = joinSubTrees(t.hash, t.head.next, t.head)
 	}
 }
+
+func (t *Tree) GetAllSubTrees() []*SubTree {
+	var subs []*SubTree
+	if t.head == nil {
+		return nil
+	}
+	subs = append(subs, t.head)
+	head := t.head
+	for head.next != nil {
+		subs = append(subs, head.next)
+		head = head.next
+	}
+
+	//调换,从最初的开始
+	for i := 0; i < len(subs)/2; i++ {
+		subs[i], subs[len(subs)-1-i] = subs[len(subs)-1-i], subs[i]
+	}
+	return subs
+}
+
+func (t *SubTree) GetHeight() int {
+	return t.height
+}
+
+func (t *SubTree) GetSum() []byte {
+	return t.sum
+}

plugin/dapp/mix/executor/query.go

@@ -8,6 +8,7 @@ import (
 	"github.com/33cn/chain33/types"
 
 	mixTy "github.com/33cn/plugin/plugin/dapp/mix/types"
+	"github.com/pkg/errors"
 )
 
 // Query_GetTreePath 根据leaf获取path 证明和roothash
@@ -24,18 +25,31 @@ func (m *Mix) Query_GetLeavesList(in *mixTy.TreeInfoReq) (types.Message, error)
 		return nil, types.ErrInvalidParam
 	}
 	var leaves *mixTy.CommitTreeLeaves
-	var err error
+
 	if len(in.RootHash) > 0 {
-		leaves, err = getCommitLeaves(m.GetStateDB(), calcCommitTreeRootLeaves(in.RootHash))
+		l, err := getCommitLeaves(m.GetStateDB(), calcCommitTreeRootLeaves(in.RootHash))
+		if err != nil {
+			return nil, err
+		}
+		leaves.Leaves = append(leaves.Leaves, l.Leaves...)
 	} else {
-		leaves, err = getCommitLeaves(m.GetStateDB(), calcCurrentCommitLeavesKey())
-	}
-	if err != nil {
-		return nil, err
+		status, err := getCommitTreeStatus(m.GetStateDB())
+		if err != nil {
+			return nil, err
+		}
+		for i := int32(1); i <= status.SubLeavesSeq; i++ {
+			l, err := getCommitLeaves(m.GetStateDB(), calcSubLeavesKey(i))
+			if err != nil {
+				return nil, errors.Wrapf(err, "get leaves of seq=%d", i)
+			}
+			leaves.Leaves = append(leaves.Leaves, l.Leaves...)
+		}
+
 	}
+
 	var resp mixTy.TreeListResp
 	for _, k := range leaves.Leaves {
-		resp.Leaves = append(resp.Leaves, transferFr2String(k))
+		resp.Leaves = append(resp.Leaves, mixTy.Byte2Str(k))
 	}
 
 	return &resp, nil
@@ -43,19 +57,51 @@ func (m *Mix) Query_GetLeavesList(in *mixTy.TreeInfoReq) (types.Message, error)
 }
 
 // Query_GetRootList query  title
-func (m *Mix) Query_GetRootList(in *types.ReqNil) (types.Message, error) {
-	roots, err := getArchiveCommitRoots(m.GetStateDB())
-	if err != nil {
-		return nil, err
+func (m *Mix) Query_GetRootList(in *types.ReqInt) (types.Message, error) {
+	var roots mixTy.CommitTreeRoots
+	if in.Height > 0 {
+		r, err := getArchiveRoots(m.GetStateDB(), uint64(in.Height))
+		if err != nil {
+			return nil, err
+		}
+		roots.Roots = append(roots.Roots, r.Roots...)
+	} else {
+		status, err := getCommitTreeStatus(m.GetStateDB())
+		if err != nil {
+			return nil, err
+		}
+		for i := int32(1); i <= status.SubLeavesSeq; i++ {
+			r, err := getSubRoots(m.GetStateDB(), i)
+			if err != nil {
+				return nil, errors.Wrapf(err, "get roots of seq=%d", i)
+			}
+			roots.Roots = append(roots.Roots, r.Roots...)
+		}
 	}
-	var resp mixTy.TreeListResp
+
+	var resp mixTy.RootListResp
 	for _, k := range roots.Roots {
-		resp.Leaves = append(resp.Leaves, transferFr2String(k))
+		resp.Roots = append(resp.Roots, mixTy.Byte2Str(k))
 	}
 
 	return &resp, nil
 }
 
+func (m *Mix) Query_GetTreeStatus(in *types.ReqNil) (types.Message, error) {
+	status, err := getCommitTreeStatus(m.GetStateDB())
+	if err != nil {
+		return nil, err
+	}
+
+	var resp mixTy.TreeStatusResp
+	resp.SubLeavesSeq = status.SubLeavesSeq
+	resp.ArchiveRootsSeq = status.ArchiveRootsSeq
+	for _, h := range status.SubTrees.SubTrees {
+		resp.SubTrees = append(resp.SubTrees, &mixTy.SubTreeResp{Height: h.Height, Hash: mixTy.Byte2Str(h.Hash)})
+	}
+	return &resp, nil
+}
+
 // Query_ListMixTxs 批量查询
 func (m *Mix) Query_ListMixTxs(in *mixTy.MixTxListReq) (types.Message, error) {
 	return m.listMixInfos(in)

plugin/dapp/mix/executor/transfer.go

@@ -161,7 +161,7 @@ func (a *action) Transfer(transfer *mixTy.MixTransferAction) (*types.Receipt, er
 	//push new commit to merkle tree
 	var leaves [][]byte
 	for _, h := range outputs {
-		leaves = append(leaves, transferFr2Bytes(h.NoteHash))
+		leaves = append(leaves, mixTy.Str2Byte(h.NoteHash))
 	}
 	rpt, err := pushTree(a.db, leaves)
 	if err != nil {

plugin/dapp/mix/executor/withdraw.go

@@ -19,13 +19,17 @@ import (
 func spendVerify(db dbm.KV, treeRootHash, nulliferHash, authorizeSpendHash string) error {
 	//zk-proof校验
 	//check tree rootHash exist
-	if !checkTreeRootHashExist(db, transferFr2Bytes(treeRootHash)) {
+	exist, err := checkTreeRootHashExist(db, mixTy.Str2Byte(treeRootHash))
+	if err != nil {
+		return errors.Wrapf(err, "roothash=%s not found", treeRootHash)
+	}
+	if !exist {
 		return errors.Wrapf(mixTy.ErrTreeRootHashNotFound, "roothash=%s", treeRootHash)
 	}
 
 	//nullifier should not exist
 	nullifierKey := calcNullifierHashKey(nulliferHash)
-	_, err := db.Get(nullifierKey)
+	_, err = db.Get(nullifierKey)
 	if err == nil {
 		return errors.Wrapf(mixTy.ErrNulliferHashExist, "nullifier=%s", nulliferHash)
 	}

plugin/dapp/mix/proto/mix.proto

@@ -171,6 +171,54 @@ message ExistValue {
     bool exist = 2;
 }
 
+
+//子树用高度和hash描述，2^10=1024个叶子节点，子树高度不会超过10
+message CommitSubTree{
+    int32 height = 1;
+    bytes hash = 2;
+}
+//merkel tree由子树和叶子组成，三种场景:1,初始只一个叶子, 2,全是子树, 3,子树加一个叶子，不会有两个叶子，两个叶子一定构成子树
+message CommitSubTrees{
+    repeated CommitSubTree subTrees = 1;
+}
+
+//描述当前commitTree的状态
+//一颗merkel树1024叶子，考虑数据库读取原因，每个叶子都会存到相应seq db，待第1024次时候归档，重新开始新的merkle树，
+message CommitTreeStatus{
+    CommitSubTrees subTrees = 1;
+    //1~1024叶子的归档
+    int32 subLeavesSeq = 2;
+    //root的归档 从1开始,数据库占位，不然全空会往前搜索
+    uint64 archiveRootsSeq = 3;
+
+}
+
+message ReceiptCommitTreeStatus{
+    CommitTreeStatus prev = 1;
+    CommitTreeStatus current = 2;
+}
+
+message ReceiptCommitSubLeaves{
+    int32  seq = 1;
+    string leaf = 2;
+}
+
+message ReceiptCommitSubRoots{
+    int32  seq = 1;
+    string root = 2;
+}
+
+message ReceiptArchiveLeaves{
+    int32 count = 1;
+    string rootHash = 2;
+    string lastLeaf = 3;
+}
+message ReceiptArchiveTreeRoot{
+    uint64 seq = 1;
+    string rootHash = 2;
+}
+
+
 message CommitTreeLeaves {
     repeated bytes leaves = 1;
 }
@@ -196,6 +244,20 @@ message TreeInfoReq{
 message TreeListResp{
     repeated string leaves = 1;
 }
+message RootListResp{
+    repeated string roots = 1;
+}
+
+message SubTreeResp{
+    int32  height = 1;
+    string hash = 2;
+}
+message TreeStatusResp{
+    int32 subLeavesSeq = 1;
+    uint64 archiveRootsSeq = 2;
+    repeated SubTreeResp subTrees = 3;
+}
+
 
 // mix wallet part
 // receiverPubKey = hash(spendPriKey) for zk-snark note spend

plugin/dapp/mix/types/mix.go

@@ -19,20 +19,21 @@ const (
 // 执行器的日志类型
 const (
 	// TyLogParacrossCommit commit log key
-	TyLogMixLocalDeposit         = 750
-	TyLogMixLocalNullifier       = 751
-	TyLogMixLocalAuth            = 752
-	TyLogMixLocalAuthSpend       = 753
-	TyLogMixConfigVk             = 754
-	TyLogMixConfigAuth           = 755
-	TyLogCurrentCommitTreeLeaves = 756
-	TyLogCurrentCommitTreeRoots  = 757
-	TyLogCommitTreeRootLeaves    = 758
-	TyLogCommitTreeArchiveRoots  = 759
-	TyLogNulliferSet             = 760
-	TyLogAuthorizeSet            = 761
-	TyLogAuthorizeSpendSet       = 762
-	TyLogMixConfigPaymentKey     = 763
+	TyLogMixLocalDeposit       = 750
+	TyLogMixLocalNullifier     = 751
+	TyLogMixLocalAuth          = 752
+	TyLogMixLocalAuthSpend     = 753
+	TyLogMixConfigVk           = 754
+	TyLogMixConfigAuth         = 755
+	TyLogSubLeaves             = 756
+	TyLogSubRoots              = 757
+	TyLogArchiveRootLeaves     = 758
+	TyLogCommitTreeArchiveRoot = 759
+	TyLogCommitTreeStatus      = 760
+	TyLogNulliferSet           = 761
+	TyLogAuthorizeSet          = 762
+	TyLogAuthorizeSpendSet     = 763
+	TyLogMixConfigPaymentKey   = 764
 )
 
 //action type

plugin/dapp/mix/types/type.go

@@ -69,12 +69,15 @@ func (p *MixType) GetName() string {
 // GetLogMap get receipt log map
 func (p *MixType) GetLogMap() map[int64]*types.LogInfo {
 	return map[int64]*types.LogInfo{
-		TyLogMixConfigVk:             {Ty: reflect.TypeOf(ZkVerifyKeys{}), Name: "LogMixConfigVk"},
-		TyLogMixConfigAuth:           {Ty: reflect.TypeOf(AuthKeys{}), Name: "LogMixConfigAuthPubKey"},
-		TyLogCurrentCommitTreeLeaves: {Ty: reflect.TypeOf(CommitTreeLeaves{}), Name: "LogCommitTreeLeaves"},
-		TyLogCurrentCommitTreeRoots:  {Ty: reflect.TypeOf(CommitTreeRoots{}), Name: "LogCommitTreeRoots"},
-		TyLogMixConfigPaymentKey:     {Ty: reflect.TypeOf(PaymentKey{}), Name: "LogConfigReceivingKey"},
-		TyLogNulliferSet:             {Ty: reflect.TypeOf(ExistValue{}), Name: "LogNullifierSet"},
+		TyLogMixConfigVk:           {Ty: reflect.TypeOf(ZkVerifyKeys{}), Name: "LogMixConfigVk"},
+		TyLogMixConfigAuth:         {Ty: reflect.TypeOf(AuthKeys{}), Name: "LogMixConfigAuthPubKey"},
+		TyLogSubLeaves:             {Ty: reflect.TypeOf(ReceiptCommitSubLeaves{}), Name: "LogSubLeaves"},
+		TyLogCommitTreeStatus:      {Ty: reflect.TypeOf(ReceiptCommitTreeStatus{}), Name: "LogCommitTreeStatus"},
+		TyLogSubRoots:              {Ty: reflect.TypeOf(ReceiptCommitSubRoots{}), Name: "LogSubRoots"},
+		TyLogArchiveRootLeaves:     {Ty: reflect.TypeOf(ReceiptArchiveLeaves{}), Name: "LogArchiveRootLeaves"},
+		TyLogCommitTreeArchiveRoot: {Ty: reflect.TypeOf(ReceiptArchiveTreeRoot{}), Name: "LogCommitTreeArchiveRoot"},
+		TyLogMixConfigPaymentKey:   {Ty: reflect.TypeOf(PaymentKey{}), Name: "LogConfigReceivingKey"},
+		TyLogNulliferSet:           {Ty: reflect.TypeOf(ExistValue{}), Name: "LogNullifierSet"},
 	}
 }