Add filter package

go.mod

@@ -60,6 +60,7 @@ require (
 )
 
 require (
+	github.com/bits-and-blooms/bloom/v3 v3.1.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

go.sum

@@ -51,8 +51,11 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.2.1 h1:M+/hrU9xlMp7t4TyTDQW97d3tRPVuKFC6zBEK16QnXY=
 github.com/bits-and-blooms/bitset v1.2.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/bits-and-blooms/bloom/v3 v3.1.0 h1:o3Adl6bGuD9eZzMiLDepS5jqmoEAv/ZH+fFe/MH1quA=
+github.com/bits-and-blooms/bloom/v3 v3.1.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=

pkg/filter/filter.go

+package order
+
+import (
+	"bytes"
+	"sync"
+
+	bloom "github.com/bits-and-blooms/bloom/v3"
+	"github.com/meshplus/bitxhub-kit/storage"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	filterDbKey = "bloom_filter"
+	m           = 10000000 // bits
+	k           = 4        // calc hash times
+)
+
+type ReqLookUp struct {
+	sync.Mutex
+	filter  *bloom.BloomFilter
+	storage storage.Storage
+	buffer  bytes.Buffer
+	logger  logrus.FieldLogger // logger
+}
+
+func NewReqLookUp(storage storage.Storage, logger logrus.FieldLogger) (*ReqLookUp, error) {
+	filter := bloom.New(m, k)
+	filterDB := storage.Get([]byte(filterDbKey))
+	if filterDB != nil {
+		var b bytes.Buffer
+		if _, err := b.Write(filterDB); err != nil {
+			return nil, err
+		}
+		if _, err := filter.ReadFrom(&b); err != nil {
+			return nil, err
+		}
+	}
+	return &ReqLookUp{
+		filter:  filter,
+		storage: storage,
+		logger:  logger,
+	}, nil
+}
+
+func (r *ReqLookUp) Add(key []byte) {
+	r.filter.Add(key)
+}
+
+func (r *ReqLookUp) LookUp(key []byte) bool {
+	return r.filter.TestAndAdd(key)
+}
+
+func (r *ReqLookUp) Build() error {
+	r.Lock()
+	defer r.Unlock()
+	if _, err := r.filter.WriteTo(&r.buffer); err != nil {
+		return err
+	}
+	r.storage.Put([]byte(filterDbKey), r.buffer.Bytes())
+	r.buffer.Reset()
+	return nil
+}

pkg/filter/filter_test.go

+package order
+
+import (
+	"testing"
+
+	"github.com/meshplus/bitxhub-kit/log"
+	"github.com/meshplus/bitxhub-kit/storage/leveldb"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReqLookUp_Add(t *testing.T) {
+	storage, err := leveldb.New("./build")
+	require.Nil(t, err)
+	r, err := NewReqLookUp(storage, log.NewWithModule("bloom_filter"))
+	require.Nil(t, err)
+	r.Add([]byte("abcd"))
+	require.Nil(t, err)
+	err = r.Build()
+	require.Nil(t, err)
+	require.True(t, r.LookUp([]byte("abcd")))
+}