add storage dapp

plugin/dapp/init/init.go

@@ -25,4 +25,5 @@ import (
 	_ "github.com/33cn/plugin/plugin/dapp/trade"      //auto gen
 	_ "github.com/33cn/plugin/plugin/dapp/unfreeze"   //auto gen
 	_ "github.com/33cn/plugin/plugin/dapp/valnode"    //auto gen
+	_ "github.com/33cn/plugin/plugin/dapp/storage"    //auto gen
 )

plugin/dapp/storage/cmd/Makefile

+all:
+	chmod +x ./build.sh
+	./build.sh $(OUT) $(FLAG)

plugin/dapp/storage/cmd/build.sh

+#!/bin/sh
+# 官方ci集成脚本
+strpwd=$(pwd)
+strcmd=${strpwd##*dapp/}
+strapp=${strcmd%/cmd*}
+
+OUT_DIR="${1}/$strapp"
+#FLAG=$2
+
+mkdir -p "${OUT_DIR}"
+cp ./build/* "${OUT_DIR}"

plugin/dapp/storage/commands/commands.go

+/*Package commands implement dapp client commands*/
+package commands
+
+import (
+	"github.com/spf13/cobra"
+)
+
+/*
+ * 实现合约对应客户端
+ */
+
+// Cmd storage client command
+func Cmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "storage",
+		Short: "storage command",
+		Args:  cobra.MinimumNArgs(1),
+	}
+	cmd.AddCommand(
+	//add sub command
+	)
+	return cmd
+}

plugin/dapp/storage/executor/exec.go

+package executor
+
+import (
+	"github.com/33cn/chain33/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+/*
+ * 实现交易的链上执行接口
+ * 关键数据上链（statedb）并生成交易回执（log）
+ */
+
+func (s *storage) Exec_ContentStorage(payload *storagetypes.ContentOnlyNotaryStorage, tx *types.Transaction, index int) (*types.Receipt, error) {
+	action := newStorageAction(s, tx, index)
+	return action.storage(&storagetypes.Storage{Value: &storagetypes.Storage_ContentStorage{ContentStorage: payload}})
+}
+
+func (s *storage) Exec_HashStorage(payload *storagetypes.HashOnlyNotaryStorage, tx *types.Transaction, index int) (*types.Receipt, error) {
+	action := newStorageAction(s, tx, index)
+	return action.storage(&storagetypes.Storage{Value: &storagetypes.Storage_HashStorage{HashStorage: payload}})
+}
+
+func (s *storage) Exec_LinkStorage(payload *storagetypes.LinkNotaryStorage, tx *types.Transaction, index int) (*types.Receipt, error) {
+	action := newStorageAction(s, tx, index)
+	return action.storage(&storagetypes.Storage{Value: &storagetypes.Storage_LinkStorage{LinkStorage: payload}})
+}
+
+func (s *storage) Exec_EncryptStorage(payload *storagetypes.EncryptNotaryStorage, tx *types.Transaction, index int) (*types.Receipt, error) {
+	action := newStorageAction(s, tx, index)
+	return action.storage(&storagetypes.Storage{Value: &storagetypes.Storage_EncryptStorage{EncryptStorage: payload}})
+}
+
+func (s *storage) Exec_EncryptShareStorage(payload *storagetypes.EncryptShareNotaryStorage, tx *types.Transaction, index int) (*types.Receipt, error) {
+	action := newStorageAction(s, tx, index)
+	return action.storage(&storagetypes.Storage{Value: &storagetypes.Storage_EncryptShareStorage{EncryptShareStorage: payload}})
+}

plugin/dapp/storage/executor/exec_del_local.go

+package executor
+
+import (
+	"github.com/33cn/chain33/types"
+)
+
+/*
+ * 实现区块回退时本地执行的数据清除
+ */
+
+// ExecDelLocal 回退自动删除，重写基类
+func (s *storage) ExecDelLocal(tx *types.Transaction, receipt *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	kvs, err := s.DelRollbackKV(tx, tx.Execer)
+	if err != nil {
+		return nil, err
+	}
+	dbSet := &types.LocalDBSet{}
+	dbSet.KV = append(dbSet.KV, kvs...)
+	return dbSet, nil
+}

plugin/dapp/storage/executor/exec_local.go

+package executor
+
+import (
+	"github.com/33cn/chain33/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+/*
+ * 实现交易相关数据本地执行，数据不上链
+ * 非关键数据，本地存储(localDB), 用于辅助查询，效率高
+ */
+
+func (s *storage) ExecLocal_ContentStorage(payload *storagetypes.ContentOnlyNotaryStorage, tx *types.Transaction, receiptData *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	dbSet := &types.LocalDBSet{}
+	//implement code
+	return s.addAutoRollBack(tx, dbSet.KV), nil
+}
+
+func (s *storage) ExecLocal_HashStorage(payload *storagetypes.HashOnlyNotaryStorage, tx *types.Transaction, receiptData *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	dbSet := &types.LocalDBSet{}
+	//implement code
+	return s.addAutoRollBack(tx, dbSet.KV), nil
+}
+
+func (s *storage) ExecLocal_LinkStorage(payload *storagetypes.LinkNotaryStorage, tx *types.Transaction, receiptData *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	dbSet := &types.LocalDBSet{}
+	//implement code
+	return s.addAutoRollBack(tx, dbSet.KV), nil
+}
+
+func (s *storage) ExecLocal_EncryptStorage(payload *storagetypes.EncryptNotaryStorage, tx *types.Transaction, receiptData *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	dbSet := &types.LocalDBSet{}
+	//implement code
+	return s.addAutoRollBack(tx, dbSet.KV), nil
+}
+
+func (s *storage) ExecLocal_EncryptShareStorage(payload *storagetypes.EncryptShareNotaryStorage, tx *types.Transaction, receiptData *types.ReceiptData, index int) (*types.LocalDBSet, error) {
+	dbSet := &types.LocalDBSet{}
+	//implement code
+	return s.addAutoRollBack(tx, dbSet.KV), nil
+}
+
+//设置自动回滚
+func (s *storage) addAutoRollBack(tx *types.Transaction, kv []*types.KeyValue) *types.LocalDBSet {
+
+	dbSet := &types.LocalDBSet{}
+	dbSet.KV = s.AddRollbackKV(tx, tx.Execer, kv)
+	return dbSet
+}

plugin/dapp/storage/executor/kv.go

+package executor
+
+/*
+ * 用户合约存取kv数据时，key值前缀需要满足一定规范
+ * 即key = keyPrefix + userKey
+ * 需要字段前缀查询时，使用’-‘作为分割符号
+ */
+
+var (
+	//KeyPrefixStateDB state db key必须前缀
+	KeyPrefixStateDB = "mavl-storage-"
+	//KeyPrefixLocalDB local db的key必须前缀
+	KeyPrefixLocalDB = "LODB-storage-"
+)
+// Key Storage to save key
+func Key(txHash string) (key []byte) {
+	key = append(key, []byte(KeyPrefixStateDB)...)
+	key = append(key, []byte(txHash)...)
+	return key
+}
\ No newline at end of file

plugin/dapp/storage/executor/query.go

+package executor
+
+import (
+	"github.com/33cn/chain33/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+
+//从statedb 读取原始数据
+func (s *storage) Query_QueryStorage(in *storagetypes.QueryStorage) (types.Message, error) {
+	return QueryStorage(s.GetStateDB(), in)
+}
+
+//通过状态查询ids
+func (s *storage) Query_BatchQueryStorage(in *storagetypes.BatchQueryStorage) (types.Message, error) {
+	return BatchQueryStorage(s.GetStateDB(), in)
+}
\ No newline at end of file

plugin/dapp/storage/executor/storage.go

+package executor
+
+import (
+	log "github.com/33cn/chain33/common/log/log15"
+	drivers "github.com/33cn/chain33/system/dapp"
+	"github.com/33cn/chain33/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+/*
+ * 执行器相关定义
+ * 重载基类相关接口
+ */
+
+var (
+	//日志
+	elog = log.New("module", "storage.executor")
+)
+
+var driverName = storagetypes.StorageX
+
+// Init register dapp
+func Init(name string, cfg *types.Chain33Config, sub []byte) {
+	drivers.Register(cfg, GetName(), newStorage, cfg.GetDappFork(driverName, "Enable"))
+	InitExecType()
+}
+
+// InitExecType Init Exec Type
+func InitExecType() {
+	ety := types.LoadExecutorType(driverName)
+	ety.InitFuncList(types.ListMethod(&storage{}))
+}
+
+type storage struct {
+	drivers.DriverBase
+}
+
+func newStorage() drivers.Driver {
+	t := &storage{}
+	t.SetChild(t)
+	t.SetExecutorType(types.LoadExecutorType(driverName))
+	return t
+}
+
+// GetName get driver name
+func GetName() string {
+	return newStorage().GetName()
+}
+
+func (s *storage) GetDriverName() string {
+	return driverName
+}
+
+// CheckTx 实现自定义检验交易接口，供框架调用
+func (s *storage) CheckTx(tx *types.Transaction, index int) error {
+	// implement code
+	return nil
+}

plugin/dapp/storage/executor/storagedb.go

+package executor
+
+import (
+	"fmt"
+	"github.com/33cn/chain33/common"
+	dbm "github.com/33cn/chain33/common/db"
+	"github.com/33cn/chain33/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+	"github.com/gogo/protobuf/proto"
+)
+
+type StorageAction struct {
+	db        dbm.KV
+	txhash    []byte
+	fromaddr  string
+	blocktime int64
+	height    int64
+	index     int
+}
+
+func newStorageAction(s *storage, tx *types.Transaction, index int) *StorageAction {
+	hash := tx.Hash()
+	fromaddr := tx.From()
+	return &StorageAction{s.GetStateDB(), hash, fromaddr,
+		s.GetBlockTime(), s.GetHeight(), index}
+}
+func (s *StorageAction) GetKVSet(payload proto.Message) (kvset []*types.KeyValue) {
+	kvset = append(kvset, &types.KeyValue{Key: Key(common.ToHex(s.txhash)), Value: types.Encode(payload)})
+	return kvset
+}
+
+//TODO 这里得数据是否存储到状态数据库中？
+func (s *StorageAction) storage(payload proto.Message) (*types.Receipt, error) {
+	//TODO 这里可以加具体得文本内容限制，超过指定大小的数据不容许写到状态数据库中
+	var logs []*types.ReceiptLog
+	kv := s.GetKVSet(payload)
+	receipt := &types.Receipt{Ty: types.ExecOk, KV: kv, Logs: logs}
+	return receipt, nil
+}
+
+func QueryStorageByTxHash(db dbm.KV, txhash string) (*storagetypes.Storage, error) {
+	data, err := db.Get(Key(txhash))
+	if err != nil {
+		elog.Debug("QueryStorage", "get", err)
+		return nil, err
+	}
+	var storage storagetypes.Storage
+	//decode
+	err = types.Decode(data, &storage)
+	if err != nil {
+		elog.Debug("QueryStorage", "decode", err)
+		return nil, err
+	}
+	return &storage, nil
+}
+func QueryStorage(db dbm.KV, in *storagetypes.QueryStorage) (types.Message, error) {
+	if in.TxHash == "" {
+		return nil, fmt.Errorf("txhash can't equail nil")
+	}
+	return QueryStorageByTxHash(db, in.TxHash)
+}
+func BatchQueryStorage(db dbm.KV, in *storagetypes.BatchQueryStorage) (types.Message, error) {
+	if len(in.TxHashs) > 10 {
+		return nil, fmt.Errorf("The number of batch queries is too large! the maximux is %d,but current num %d", 10, len(in.TxHashs))
+	}
+	var storage storagetypes.BatchReplyStorage
+	for _, txhash := range in.TxHashs {
+		msg, err := QueryStorageByTxHash(db, txhash)
+		if err != nil {
+			return msg, err
+		}
+		storage.Storages = append(storage.Storages, msg)
+	}
+	return &storage, nil
+}

plugin/dapp/storage/plugin.go

+package types
+
+import (
+	"github.com/33cn/chain33/pluginmgr"
+	"github.com/33cn/plugin/plugin/dapp/storage/commands"
+	"github.com/33cn/plugin/plugin/dapp/storage/executor"
+	"github.com/33cn/plugin/plugin/dapp/storage/rpc"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+/*
+ * 初始化dapp相关的组件
+ */
+
+func init() {
+	pluginmgr.Register(&pluginmgr.PluginBase{
+		Name:     storagetypes.StorageX,
+		ExecName: executor.GetName(),
+		Exec:     executor.Init,
+		Cmd:      commands.Cmd,
+		RPC:      rpc.Init,
+	})
+}

plugin/dapp/storage/proto/Makefile

+all:
+	./create_protobuf.sh

plugin/dapp/storage/proto/create_protobuf.sh

+#!/bin/sh
+# proto生成命令，将pb.go文件生成到types/目录下, chain33_path支持引用chain33框架的proto文件
+chain33_path=$(go list -f '{{.Dir}}' "github.com/33cn/chain33")
+protoc --go_out=plugins=grpc:../types ./*.proto --proto_path=. --proto_path="${chain33_path}/types/proto/"

plugin/dapp/storage/proto/storage.proto

+syntax = "proto3";
+package types;
+//后面如果有其他数据模型可继续往上面添加
+message Storage {
+    oneof value {
+        ContentOnlyNotaryStorage contentStorage = 1;
+        HashOnlyNotaryStorage hashStorage = 2;
+        LinkNotaryStorage  linkStorage = 3;
+        EncryptNotaryStorage encryptStorage = 4;
+        EncryptShareNotaryStorage encryptShareStorage = 5;
+    }
+}
+
+message StorageAction {
+    oneof value {
+        ContentOnlyNotaryStorage contentStorage = 1;
+        HashOnlyNotaryStorage hashStorage = 2;
+        LinkNotaryStorage  linkStorage = 3;
+        EncryptNotaryStorage encryptStorage = 4;
+        EncryptShareNotaryStorage encryptShareStorage = 5;
+    }
+    int32 ty = 6;
+}
+// 内容存证模型
+message ContentOnlyNotaryStorage {
+    //长度需要小于512k
+    bytes content   = 1;
+}
+
+//哈希存证模型，推荐使用sha256哈希，限制256位得摘要值
+message HashOnlyNotaryStorage {
+
+    //长度固定为32字节
+    bytes hash   = 1;
+}
+
+// 链接存证模型
+message LinkNotaryStorage {
+    //存证内容的链接，可以写入URL,或者其他可用于定位源文件得线索.
+    bytes link   = 1;
+    //源文件得hash值，推荐使用sha256哈希，限制256位得摘要值
+    bytes hash = 2;
+}
+
+// 隐私存证模型，如果一个文件需要存证，且不公开内容，可以选择将源文件通过对称加密算法加密后上链
+message EncryptNotaryStorage {
+    //存证明文内容的hash值，推荐使用sha256哈希，限制256位得摘要值
+    bytes contentHash   = 1;
+    //源文件得密文，由加密key及nonce对明文加密得到该值。
+    bytes encryptContent = 2;
+    //加密iv，通过AES进行加密时制定随机生成的iv,解密时需要使用该值
+    bytes nonce = 3;
+}
+// 隐私存证模型
+message EncryptContentOnlyNotaryStorage {
+    //存证内容的hash值，推荐使用sha256哈希，限制256位得摘要值
+   //   bytes contentHash   = 1;
+    //源文件得密文。
+    bytes encryptContent = 1;
+    //加密iv，通过AES进行加密时制定随机生成的iv,解密时需要使用该值
+    bytes nonce = 2;
+}
+
+// 分享隐私存证模型
+message EncryptShareNotaryStorage {
+    //存证明文内容的hash值，推荐使用sha256哈希，限制256位得摘要值
+    bytes contentHash   = 1;
+    //源文件得密文。
+    bytes encryptContent = 2;
+    //密钥的kdf推导路径。密钥tree父节点根据该路径可以推导出私钥key
+    bytes keyName = 3;
+    //加密key的wrap key。加密key随机生成，对明文进行加密，该key有私密key进行key wrap后公开。
+    //使用时，通过私密key对wrap key解密得到加密key对密文进行解密。
+    bytes keyWrap = 4;
+    //加密iv，通过AES进行加密时制定随机生成的iv,解密时需要使用该值
+    bytes nonce = 5;
+}
+
+service storage {
+
+}
+//根据txhash去状态数据库中查询存储内容
+message QueryStorage {
+    string txHash = 1;
+}
+//批量查询有可能导致数据库崩溃
+message BatchQueryStorage {
+    repeated string txHashs = 1;
+}
+
+message BatchReplyStorage {
+    repeated Storage storages = 1;
+}
\ No newline at end of file

plugin/dapp/storage/rpc/rpc.go

+package rpc
+
+/*
+ * 实现json rpc和grpc service接口
+ * json rpc用Jrpc结构作为接收实例
+ * grpc使用channelClient结构作为接收实例
+ */

plugin/dapp/storage/rpc/types.go

+package rpc
+
+import (
+	rpctypes "github.com/33cn/chain33/rpc/types"
+	storagetypes "github.com/33cn/plugin/plugin/dapp/storage/types"
+)
+
+/*
+ * rpc相关结构定义和初始化
+ */
+
+// 实现grpc的service接口
+type channelClient struct {
+	rpctypes.ChannelClient
+}
+
+// Jrpc 实现json rpc调用实例
+type Jrpc struct {
+	cli *channelClient
+}
+
+// Grpc grpc
+type Grpc struct {
+	*channelClient
+}
+
+// Init init rpc
+func Init(name string, s rpctypes.RPCServer) {
+	cli := &channelClient{}
+	grpc := &Grpc{channelClient: cli}
+	cli.Init(name, s, &Jrpc{cli: cli}, grpc)
+	//存在grpc service时注册grpc server，需要生成对应的pb.go文件
+	storagetypes.RegisterStorageServer(s.GRPC(), grpc)
+}

plugin/dapp/storage/types/storage.go

+package types
+
+import (
+	//"encoding/json"
+	log "github.com/33cn/chain33/common/log/log15"
+	"github.com/33cn/chain33/types"
+)
+
+/*
+ * 交易相关类型定义
+ * 交易action通常有对应的log结构，用于交易回执日志记录
+ * 每一种action和log需要用id数值和name名称加以区分
+ */
+
+// action类型id和name，这些常量可以自定义修改
+const (
+	TyUnknowAction = iota + 100
+	TyContentStorageAction
+	TyHashStorageAction
+	TyLinkStorageAction
+	TyEncryptStorageAction
+	TyEncryShareStorageAction
+
+	NameContentStorageAction    = "ContentStorage"
+	NameHashStorageAction       = "HashStorage"
+	NameLinkStorageAction       = "LinkStorage"
+	NameEncryptStorageAction    = "EncryptStorage"
+	NameEncryShareStorageAction = "EncryShareStorage"
+)
+
+// log类型id值
+const (
+	TyUnknownLog = iota + 100
+	TyContentStorageLog
+	TyHashStorageLog
+	TyLinkStorageLog
+	TyEncryptStorageLog
+	TyEncryShareStorageLog
+)
+
+var (
+	//StorageX 执行器名称定义
+	StorageX = "storage"
+	//定义actionMap
+	actionMap = map[string]int32{
+		NameContentStorageAction:    TyContentStorageAction,
+		NameHashStorageAction:       TyHashStorageAction,
+		NameLinkStorageAction:       TyLinkStorageAction,
+		NameEncryptStorageAction:    TyEncryptStorageAction,
+		NameEncryShareStorageAction: TyEncryShareStorageAction,
+	}
+	//定义log的id和具体log类型及名称，填入具体自定义log类型
+	logMap = map[int64]*types.LogInfo{
+		//LogID:	{Ty: reflect.TypeOf(LogStruct), Name: LogName},
+	}
+	tlog = log.New("module", "storage.types")
+)
+
+// init defines a register function
+func init() {
+	types.AllowUserExec = append(types.AllowUserExec, []byte(StorageX))
+	//注册合约启用高度
+	types.RegFork(StorageX, InitFork)
+	types.RegExec(StorageX, InitExecutor)
+}
+
+// InitFork defines register fork
+func InitFork(cfg *types.Chain33Config) {
+	cfg.RegisterDappFork(StorageX, "Enable", 0)
+}
+
+// InitExecutor defines register executor
+func InitExecutor(cfg *types.Chain33Config) {
+	types.RegistorExecutor(StorageX, NewType(cfg))
+}
+
+type storageType struct {
+	types.ExecTypeBase
+}
+
+func NewType(cfg *types.Chain33Config) *storageType {
+	c := &storageType{}
+	c.SetChild(c)
+	c.SetConfig(cfg)
+	return c
+}
+
+// GetPayload 获取合约action结构
+func (s *storageType) GetPayload() types.Message {
+	return &StorageAction{}
+}
+
+// GeTypeMap 获取合约action的id和name信息
+func (s *storageType) GetTypeMap() map[string]int32 {
+	return actionMap
+}
+
+// GetLogMap 获取合约log相关信息
+func (s *storageType) GetLogMap() map[int64]*types.LogInfo {
+	return logMap
+}