Skip to content

P
plugin
Commit 6bf4628e authored by harrylee's avatar harrylee Committed by vipwzw
Browse files
Options

ajust code for raft and pbft

parent e89dbbfa
Show whitespace changes
Inline Side-by-side
Showing with 1070 additions and 1001 deletions
go.mod
View file @ 6bf4628e
......@@ -20,8 +20,6 @@ require (
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/golang/protobuf v1.3.2
github.com/hashicorp/golang-lru v0.5.0
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.8.0
github.com/prometheus/client_golang v1.1.0 // indirect
github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d
......
plugin/consensus/pbft/pbft.go
View file @ 6bf4628e
......@@ -277,12 +277,12 @@ func (rep *Replica) hasRequest(REQ *pb.Request) bool {
return rep.hasRequestPrepare(REQ)
case *pb.Request_Commit:
return rep.hasRequestCommit(REQ)
case *pb.Request_Viewchange:
return rep.hasRequestViewChange(REQ)
case *pb.Request_Ack:
return rep.hasRequestAck(REQ)
case *pb.Request_Newview:
return rep.hasRequestNewView(REQ)
//case *pb.Request_Viewchange:
// return rep.hasRequestViewChange(REQ)
//case *pb.Request_Ack:
// return rep.hasRequestAck(REQ)
//case *pb.Request_Newview:
// return rep.hasRequestNewView(REQ)
default:
return false
}
......@@ -335,160 +335,160 @@ func (rep *Replica) hasRequestCommit(REQ *pb.Request) bool {
return false
}
func (rep *Replica) hasRequestViewChange(REQ *pb.Request) bool {
view := REQ.GetViewchange().View
replica := REQ.GetViewchange().Replica
for _, req := range rep.requests["view-change"] {
v := req.GetViewchange().View
r := req.GetViewchange().Replica
if v == view && r == replica {
return true
}
}
return false
}
func (rep *Replica) hasRequestAck(REQ *pb.Request) bool {
view := REQ.GetAck().View
replica := REQ.GetAck().Replica
viewchanger := REQ.GetAck().Viewchanger
for _, req := range rep.requests["ack"] {
v := req.GetAck().View
r := req.GetAck().Replica
vc := req.GetAck().Viewchanger
if v == view && r == replica && vc == viewchanger {
return true
}
}
return false
}
func (rep *Replica) hasRequestNewView(REQ *pb.Request) bool {
view := REQ.GetNewview().View
for _, req := range rep.requests["new-view"] {
v := req.GetNewview().View
if v == view {
return true
}
}
return false
}
//func (rep *Replica) hasRequestViewChange(REQ *pb.Request) bool {
// view := REQ.GetViewchange().View
// replica := REQ.GetViewchange().Replica
// for _, req := range rep.requests["view-change"] {
// v := req.GetViewchange().View
// r := req.GetViewchange().Replica
// if v == view && r == replica {
// return true
// }
// }
// return false
//}
//func (rep *Replica) hasRequestAck(REQ *pb.Request) bool {
// view := REQ.GetAck().View
// replica := REQ.GetAck().Replica
// viewchanger := REQ.GetAck().Viewchanger
// for _, req := range rep.requests["ack"] {
// v := req.GetAck().View
// r := req.GetAck().Replica
// vc := req.GetAck().Viewchanger
// if v == view && r == replica && vc == viewchanger {
// return true
// }
// }
// return false
//}
//func (rep *Replica) hasRequestNewView(REQ *pb.Request) bool {
// view := REQ.GetNewview().View
// for _, req := range rep.requests["new-view"] {
// v := req.GetNewview().View
// if v == view {
// return true
// }
// }
// return false
//}
// Clear requests
func (rep *Replica) clearRequestsBySeq(sequence uint32) {
rep.clearRequestClients()
rep.clearRequestPrepreparesBySeq(sequence)
rep.clearRequestPreparesBySeq(sequence)
rep.clearRequestCommitsBySeq(sequence)
//rep.clearRequestCheckpointsBySeq(sequence)
}
func (rep *Replica) clearRequestClients() {
clientReqs := rep.requests["client"]
lastTimestamp := rep.theLastReply().Timestamp
for idx, req := range rep.requests["client"] {
timestamp := req.GetClient().Timestamp
if lastTimestamp >= timestamp && idx < (len(clientReqs)-1) {
clientReqs = append(clientReqs[:idx], clientReqs[idx+1:]...)
} else {
clientReqs = clientReqs[:idx-1]
}
}
rep.requests["client"] = clientReqs
}
func (rep *Replica) clearRequestPrepreparesBySeq(sequence uint32) {
prePrepares := rep.requests["pre-prepare"]
for idx, req := range rep.requests["pre-prepare"] {
s := req.GetPreprepare().Sequence
if s <= sequence && idx < (len(prePrepares)-1) {
prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
} else {
prePrepares = prePrepares[:idx-1]
}
}
rep.requests["pre-prepare"] = prePrepares
}
func (rep *Replica) clearRequestPreparesBySeq(sequence uint32) {
prepares := rep.requests["prepare"]
for idx, req := range rep.requests["prepare"] {
s := req.GetPrepare().Sequence
if s <= sequence && idx < (len(prepares)-1) {
prepares = append(prepares[:idx], prepares[idx+1:]...)
} else {
prepares = prepares[:idx-1]
}
}
rep.requests["prepare"] = prepares
}
func (rep *Replica) clearRequestCommitsBySeq(sequence uint32) {
commits := rep.requests["commit"]
for idx, req := range rep.requests["commit"] {
s := req.GetCommit().Sequence
if s <= sequence && idx < (len(commits)-1) {
commits = append(commits[:idx], commits[idx+1:]...)
} else {
commits = commits[:idx-1]
}
}
rep.requests["commit"] = commits
}
func (rep *Replica) clearRequestCheckpointsBySeq(sequence uint32) {
checkpoints := rep.requests["checkpoint"]
for idx, req := range rep.requests["checkpoint"] {
s := req.GetCheckpoint().Sequence
if s <= sequence && idx < (len(checkpoints)-1) {
checkpoints = append(checkpoints[:idx], checkpoints[idx+1:]...)
} else {
checkpoints = checkpoints[:idx-1]
}
}
rep.requests["checkpoint"] = checkpoints
}
func (rep *Replica) clearRequestsByView(view uint32) {
rep.clearRequestPrepreparesByView(view)
rep.clearRequestPreparesByView(view)
rep.clearRequestCommitsByView(view)
//add others
}
func (rep *Replica) clearRequestPrepreparesByView(view uint32) {
prePrepares := rep.requests["pre-prepare"]
for idx, req := range rep.requests["pre-prepare"] {
v := req.GetPreprepare().View
if v < view {
prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
}
}
rep.requests["pre-prepare"] = prePrepares
}
func (rep *Replica) clearRequestPreparesByView(view uint32) {
prepares := rep.requests["prepare"]
for idx, req := range rep.requests["prepare"] {
v := req.GetPrepare().View
if v < view {
prepares = append(prepares[:idx], prepares[idx+1:]...)
}
}
rep.requests["prepare"] = prepares
}
func (rep *Replica) clearRequestCommitsByView(view uint32) {
commits := rep.requests["commit"]
for idx, req := range rep.requests["commit"] {
v := req.GetCommit().View
if v < view {
commits = append(commits[:idx], commits[idx+1:]...)
}
}
rep.requests["commit"] = commits
}
//func (rep *Replica) clearRequestsBySeq(sequence uint32) {
// rep.clearRequestClients()
// rep.clearRequestPrepreparesBySeq(sequence)
// rep.clearRequestPreparesBySeq(sequence)
// rep.clearRequestCommitsBySeq(sequence)
// //rep.clearRequestCheckpointsBySeq(sequence)
//}
//func (rep *Replica) clearRequestClients() {
// clientReqs := rep.requests["client"]
// lastTimestamp := rep.theLastReply().Timestamp
// for idx, req := range rep.requests["client"] {
// timestamp := req.GetClient().Timestamp
// if lastTimestamp >= timestamp && idx < (len(clientReqs)-1) {
// clientReqs = append(clientReqs[:idx], clientReqs[idx+1:]...)
// } else {
// clientReqs = clientReqs[:idx-1]
// }
// }
// rep.requests["client"] = clientReqs
//}
//
//func (rep *Replica) clearRequestPrepreparesBySeq(sequence uint32) {
// prePrepares := rep.requests["pre-prepare"]
// for idx, req := range rep.requests["pre-prepare"] {
// s := req.GetPreprepare().Sequence
// if s <= sequence && idx < (len(prePrepares)-1) {
// prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
// } else {
// prePrepares = prePrepares[:idx-1]
// }
// }
// rep.requests["pre-prepare"] = prePrepares
//}
//
//func (rep *Replica) clearRequestPreparesBySeq(sequence uint32) {
// prepares := rep.requests["prepare"]
// for idx, req := range rep.requests["prepare"] {
// s := req.GetPrepare().Sequence
// if s <= sequence && idx < (len(prepares)-1) {
// prepares = append(prepares[:idx], prepares[idx+1:]...)
// } else {
// prepares = prepares[:idx-1]
// }
// }
// rep.requests["prepare"] = prepares
//}
//
//func (rep *Replica) clearRequestCommitsBySeq(sequence uint32) {
// commits := rep.requests["commit"]
// for idx, req := range rep.requests["commit"] {
// s := req.GetCommit().Sequence
// if s <= sequence && idx < (len(commits)-1) {
// commits = append(commits[:idx], commits[idx+1:]...)
// } else {
// commits = commits[:idx-1]
// }
// }
// rep.requests["commit"] = commits
//}
//
//func (rep *Replica) clearRequestCheckpointsBySeq(sequence uint32) {
// checkpoints := rep.requests["checkpoint"]
// for idx, req := range rep.requests["checkpoint"] {
// s := req.GetCheckpoint().Sequence
// if s <= sequence && idx < (len(checkpoints)-1) {
// checkpoints = append(checkpoints[:idx], checkpoints[idx+1:]...)
// } else {
// checkpoints = checkpoints[:idx-1]
// }
// }
// rep.requests["checkpoint"] = checkpoints
//}
//
//func (rep *Replica) clearRequestsByView(view uint32) {
// rep.clearRequestPrepreparesByView(view)
// rep.clearRequestPreparesByView(view)
// rep.clearRequestCommitsByView(view)
// //add others
//}
//
//func (rep *Replica) clearRequestPrepreparesByView(view uint32) {
// prePrepares := rep.requests["pre-prepare"]
// for idx, req := range rep.requests["pre-prepare"] {
// v := req.GetPreprepare().View
// if v < view {
// prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
// }
// }
// rep.requests["pre-prepare"] = prePrepares
//}
//
//func (rep *Replica) clearRequestPreparesByView(view uint32) {
// prepares := rep.requests["prepare"]
// for idx, req := range rep.requests["prepare"] {
// v := req.GetPrepare().View
// if v < view {
// prepares = append(prepares[:idx], prepares[idx+1:]...)
// }
// }
// rep.requests["prepare"] = prepares
//}
//
//func (rep *Replica) clearRequestCommitsByView(view uint32) {
// commits := rep.requests["commit"]
// for idx, req := range rep.requests["commit"] {
// v := req.GetCommit().View
// if v < view {
// commits = append(commits[:idx], commits[idx+1:]...)
// }
// }
// rep.requests["commit"] = commits
//}
// Handle requests
......@@ -511,17 +511,17 @@ func (rep *Replica) handleRequest(REQ *pb.Request) {
rep.handleRequestCommit(REQ)
case *pb.Request_Checkpoint:
rep.handleRequestCheckpoint(REQ)
case *pb.Request_Viewchange:
rep.handleRequestViewChange(REQ)
case *pb.Request_Ack:
rep.handleRequestAck(REQ)
//case *pb.Request_Checkpoint:
//
// rep.handleRequestCheckpoint(REQ)
//
//case *pb.Request_Viewchange:
//
// rep.handleRequestViewChange(REQ)
//
//case *pb.Request_Ack:
//
// rep.handleRequestAck(REQ)
default:
plog.Info("Replica %d received unrecognized request type\n", rep.ID)
......@@ -766,599 +766,599 @@ func (rep *Replica) handleRequestCommit(REQ *pb.Request) {
}
func (rep *Replica) handleRequestCheckpoint(REQ *pb.Request) {
sequence := REQ.GetCheckpoint().Sequence
if !rep.sequenceInRange(sequence) {
return
}
digest := REQ.GetCheckpoint().Digest
replica := REQ.GetCheckpoint().Replica
count := 0
for _, req := range rep.requests["checkpoint"] {
s := req.GetCheckpoint().Sequence
d := req.GetCheckpoint().Digest
r := req.GetCheckpoint().Replica
if s != sequence || !EQ(d, digest) {
continue
}
if r == replica {
plog.Info("Replica %d sent multiple checkpoint requests\n", replica)
//continue
}
count++
if !rep.overTwoThirds(count) {
continue
}
// rep.clearEntries(sequence)
rep.clearRequestsBySeq(sequence)
checkpoint := ToCheckpoint(sequence, digest)
rep.addCheckpoint(checkpoint)
plog.Info("checkpoint and clear request done")
return
}
}
func (rep *Replica) handleRequestViewChange(REQ *pb.Request) {
view := REQ.GetViewchange().View
if view < rep.view {
return
}
reqViewChange := REQ.GetViewchange()
for _, prep := range reqViewChange.GetPreps() {
v := prep.View
s := prep.Sequence
if v >= view || !rep.sequenceInRange(s) {
return
}
}
for _, prePrep := range reqViewChange.GetPrepreps() {
v := prePrep.View
s := prePrep.Sequence
if v >= view || !rep.sequenceInRange(s) {
return
}
}
for _, checkpoint := range reqViewChange.GetCheckpoints() {
s := checkpoint.Sequence
if !rep.sequenceInRange(s) {
return
}
}
if rep.hasRequest(REQ) {
return
}
rep.logRequest(REQ)
viewchanger := reqViewChange.Replica
req := ToRequestAck(
view,
rep.ID,
viewchanger,
ReqDigest(REQ))
go func() {
rep.requestChan <- req
}()
}
func (rep *Replica) handleRequestAck(REQ *pb.Request) {
view := REQ.GetAck().View
primaryID := rep.newPrimary(view)
if rep.ID != primaryID {
return
}
if rep.hasRequest(REQ) {
return
}
rep.logRequest(REQ)
replica := REQ.GetAck().Replica
viewchanger := REQ.GetAck().Viewchanger
digest := REQ.GetAck().Digest
reqViewChange := make(chan *pb.Request, 1)
twoThirds := make(chan bool, 1)
go func() {
for _, req := range rep.requests["view-change"] {
v := req.GetViewchange().View
vc := req.GetViewchange().Replica
if v == view && vc == viewchanger {
reqViewChange <- req
}
}
reqViewChange <- nil
}()
go func() {
count := 0
for _, req := range rep.requests["ack"] {
v := req.GetAck().View
r := req.GetAck().Replica
vc := req.GetAck().Viewchanger
d := req.GetAck().Digest
if v != view || vc != viewchanger || !EQ(d, digest) {
continue
}
if r == replica {
plog.Info("Replica %d sent multiple ack requests\n", replica)
continue
}
count++
if rep.twoThirds(count) {
twoThirds <- true
return
}
}
twoThirds <- false
}()
req := <-reqViewChange
if req == nil || !<-twoThirds {
return
}
rep.logPendingVC(req)
// When to send new view?
rep.requestNewView(view)
}
func (rep *Replica) handleRequestNewView(REQ *pb.Request) {
view := REQ.GetNewview().View
if view == 0 || view < rep.view {
return
}
replica := REQ.GetNewview().Replica
primary := rep.newPrimary(view)
if replica != primary {
return
}
if rep.hasRequest(REQ) {
return
}
rep.logRequest(REQ)
rep.processNewView(REQ)
}
func (rep *Replica) correctViewChanges(viewChanges []*pb.ViewChange) (requests []*pb.Request) {
// Returns requests if correct, else returns nil
valid := false
for _, vc := range viewChanges {
for _, req := range rep.requests["view-change"] {
d := ReqDigest(req)
if !EQ(d, vc.Digest) {
continue
}
requests = append(requests, req)
v := req.GetViewchange().View
// VIEW or rep.view??
if v == rep.view {
valid = true
break
}
}
if !valid {
return nil
}
}
if rep.isPrimary(rep.ID) {
reps := make(map[uint32]int)
valid = false
for _, req := range rep.requests["ack"] {
reqAck := req.GetAck()
reps[reqAck.Replica]++
if rep.twoThirds(reps[reqAck.Replica]) { //-2
valid = true
break
}
}
if !valid {
return nil
}
}
return
}
func (rep *Replica) correctSummaries(requests []*pb.Request, summaries []*pb.Summary) (correct bool) {
// Verify SUMMARIES
var start uint32
var digest []byte
digests := make(map[uint32][]byte)
for _, summary := range summaries {
s := summary.Sequence
d := summary.Digest
if _d, ok := digests[s]; ok && !EQ(_d, d) {
return
} else if !ok {
digests[s] = d
}
if s < start || start == uint32(0) {
start = s
digest = d
}
}
var A1 []*pb.Request
var A2 []*pb.Request
valid := false
for _, req := range requests {
reqViewChange := req.GetViewchange()
s := reqViewChange.Sequence
if s <= start {
A1 = append(A1, req)
}
checkpoints := reqViewChange.GetCheckpoints()
for _, checkpoint := range checkpoints {
if checkpoint.Sequence == start && EQ(checkpoint.Digest, digest) {
A2 = append(A2, req)
break
}
}
if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
valid = true
break
}
}
if !valid {
return
}
end := start + CheckPointPeriod*ConstantFactor
for seq := start; seq <= end; seq++ {
valid = false
for _, summary := range summaries {
if summary.Sequence != seq {
continue
}
if summary.Digest != nil {
var view uint32
for _, req := range requests {
reqViewChange := req.GetViewchange()
preps := reqViewChange.GetPreps()
for _, prep := range preps {
s := prep.Sequence
d := prep.Digest
if s != summary.Sequence || !EQ(d, summary.Digest) {
continue
}
v := prep.View
if v > view {
view = v
}
}
}
verifiedA1 := make(chan bool, 1)
// Verify A1
go func() {
var A1 []*pb.Request
FOR_LOOP:
for _, req := range requests {
reqViewChange := req.GetViewchange()
s := reqViewChange.Sequence
if s >= summary.Sequence {
continue
}
preps := reqViewChange.GetPreps()
for _, prep := range preps {
s = prep.Sequence
if s != summary.Sequence {
continue
}
d := prep.Digest
v := prep.View
if v > view || (v == view && !EQ(d, summary.Digest)) {
continue FOR_LOOP
}
}
A1 = append(A1, req)
if rep.twoThirds(len(A1)) {
verifiedA1 <- true
return
}
}
verifiedA1 <- false
}()
verifiedA2 := make(chan bool, 1)
// Verify A2
go func() {
var A2 []*pb.Request
for _, req := range requests {
reqViewChange := req.GetViewchange()
prePreps := reqViewChange.GetPrepreps()
for _, prePrep := range prePreps {
s := prePrep.Sequence
d := prePrep.Digest
v := prePrep.View
if s == summary.Sequence && EQ(d, summary.Digest) && v >= view {
A2 = append(A2, req)
break
}
}
if rep.oneThird(len(A2)) {
verifiedA2 <- true
return
}
}
verifiedA2 <- false
}()
if !<-verifiedA1 || !<-verifiedA2 {
continue
}
valid = true
break
} else {
var A1 []*pb.Request
FOR_LOOP:
for _, req := range requests {
reqViewChange := req.GetViewchange()
s := reqViewChange.Sequence
if s >= summary.Sequence {
continue
}
preps := reqViewChange.GetPreps()
for _, prep := range preps {
if prep.Sequence == summary.Sequence {
continue FOR_LOOP
}
}
A1 = append(A1, req)
if rep.twoThirds(len(A1)) {
valid = true
break
}
}
if valid {
break
}
}
}
if !valid {
return
}
}
return true
}
func (rep *Replica) processNewView(REQ *pb.Request) (success bool) {
if rep.activeView {
return
}
reqNewView := REQ.GetNewview()
viewChanges := reqNewView.GetViewchanges()
requests := rep.correctViewChanges(viewChanges)
if requests == nil {
return
}
summaries := reqNewView.GetSummaries()
correct := rep.correctSummaries(requests, summaries)
if !correct {
return
}
var h uint32
for _, checkpoint := range rep.checkpoints {
if checkpoint.Sequence < h || h == uint32(0) {
h = checkpoint.Sequence
}
}
var s uint32
for _, summary := range summaries {
if summary.Sequence < s || s == uint32(0) {
s = summary.Sequence
}
if summary.Sequence > h {
valid := false
for _, req := range rep.requests["view-change"] { //in
if EQ(ReqDigest(req), summary.Digest) {
valid = true
break
}
}
if !valid {
return
}
}
}
if h < s {
return
}
// Process new view
rep.activeView = true
for _, summary := range summaries {
if rep.ID != reqNewView.Replica {
req := ToRequestPrepare(
reqNewView.View,
summary.Sequence,
summary.Digest,
rep.ID) // the backup sends/logs prepare
go func() {
if !rep.hasRequest(req) {
rep.requestChan <- req
}
}()
if summary.Sequence <= h {
continue
}
if !rep.hasRequest(req) {
rep.logRequest(req)
}
} else {
if summary.Sequence <= h {
break
}
}
req := ToRequestPreprepare(
reqNewView.View,
summary.Sequence,
summary.Digest,
reqNewView.Replica) // new primary pre-prepares
if !rep.hasRequest(req) {
rep.logRequest(req)
}
}
var maxSequence uint32
for _, req := range rep.requests["pre-prepare"] {
reqPrePrepare := req.GetPreprepare()
if reqPrePrepare.Sequence > maxSequence {
maxSequence = reqPrePrepare.Sequence
}
}
rep.sequence = maxSequence
return true
}
func (rep *Replica) prePrepBySequence(sequence uint32) []*pb.Entry {
var view uint32
var requests []*pb.Request
for _, req := range rep.requests["pre-prepare"] {
v := req.GetPreprepare().View
s := req.GetPreprepare().Sequence
if v >= view && s == sequence {
view = v
requests = append(requests, req)
}
}
if requests == nil {
return nil
}
var prePreps []*pb.Entry
for _, req := range requests {
v := req.GetPreprepare().View
if v == view {
s := req.GetPreprepare().Sequence
d := req.GetPreprepare().Digest
prePrep := ToEntry(s, d, v)
prePreps = append(prePreps, prePrep)
}
}
FOR_LOOP:
for _, prePrep := range prePreps {
for _, req := range rep.allRequests() { //TODO: optimize
if EQ(ReqDigest(req), prePrep.Digest) {
continue FOR_LOOP
}
}
return nil
}
return prePreps
}
func (rep *Replica) prepBySequence(sequence uint32) ([]*pb.Entry, []*pb.Entry) {
prePreps := rep.prePrepBySequence(sequence)
if prePreps == nil {
return nil, nil
}
var preps []*pb.Entry
FOR_LOOP:
for _, prePrep := range prePreps {
view := prePrep.View
digest := prePrep.Digest
replicas := make(map[uint32]int)
for _, req := range rep.requests["prepare"] {
reqPrepare := req.GetPrepare()
v := reqPrepare.View
s := reqPrepare.Sequence
d := reqPrepare.Digest
if v == view && s == sequence && EQ(d, digest) {
r := reqPrepare.Replica
replicas[r]++
if rep.twoThirds(replicas[r]) {
prep := ToEntry(s, d, v)
preps = append(preps, prep)
continue FOR_LOOP
}
}
}
return prePreps, nil
}
return prePreps, preps
}
//func (rep *Replica) handleRequestCheckpoint(REQ *pb.Request) {
//
// sequence := REQ.GetCheckpoint().Sequence
//
// if !rep.sequenceInRange(sequence) {
// return
// }
//
// digest := REQ.GetCheckpoint().Digest
// replica := REQ.GetCheckpoint().Replica
//
// count := 0
// for _, req := range rep.requests["checkpoint"] {
// s := req.GetCheckpoint().Sequence
// d := req.GetCheckpoint().Digest
// r := req.GetCheckpoint().Replica
// if s != sequence || !EQ(d, digest) {
// continue
// }
// if r == replica {
// plog.Info("Replica %d sent multiple checkpoint requests\n", replica)
// //continue
// }
// count++
// if !rep.overTwoThirds(count) {
// continue
// }
// // rep.clearEntries(sequence)
// rep.clearRequestsBySeq(sequence)
// checkpoint := ToCheckpoint(sequence, digest)
// rep.addCheckpoint(checkpoint)
// plog.Info("checkpoint and clear request done")
// return
// }
//
//}
//func (rep *Replica) handleRequestViewChange(REQ *pb.Request) {
//
// view := REQ.GetViewchange().View
//
// if view < rep.view {
// return
// }
//
// reqViewChange := REQ.GetViewchange()
//
// for _, prep := range reqViewChange.GetPreps() {
// v := prep.View
// s := prep.Sequence
// if v >= view || !rep.sequenceInRange(s) {
// return
// }
// }
//
// for _, prePrep := range reqViewChange.GetPrepreps() {
// v := prePrep.View
// s := prePrep.Sequence
// if v >= view || !rep.sequenceInRange(s) {
// return
// }
// }
//
// for _, checkpoint := range reqViewChange.GetCheckpoints() {
// s := checkpoint.Sequence
// if !rep.sequenceInRange(s) {
// return
// }
// }
//
// if rep.hasRequest(REQ) {
// return
// }
//
// rep.logRequest(REQ)
//
// viewchanger := reqViewChange.Replica
//
// req := ToRequestAck(
// view,
// rep.ID,
// viewchanger,
// ReqDigest(REQ))
//
// go func() {
// rep.requestChan <- req
// }()
//}
//func (rep *Replica) handleRequestAck(REQ *pb.Request) {
//
// view := REQ.GetAck().View
// primaryID := rep.newPrimary(view)
//
// if rep.ID != primaryID {
// return
// }
//
// if rep.hasRequest(REQ) {
// return
// }
//
// rep.logRequest(REQ)
//
// replica := REQ.GetAck().Replica
// viewchanger := REQ.GetAck().Viewchanger
// digest := REQ.GetAck().Digest
//
// reqViewChange := make(chan *pb.Request, 1)
// twoThirds := make(chan bool, 1)
//
// go func() {
// for _, req := range rep.requests["view-change"] {
// v := req.GetViewchange().View
// vc := req.GetViewchange().Replica
// if v == view && vc == viewchanger {
// reqViewChange <- req
// }
// }
// reqViewChange <- nil
// }()
//
// go func() {
// count := 0
// for _, req := range rep.requests["ack"] {
// v := req.GetAck().View
// r := req.GetAck().Replica
// vc := req.GetAck().Viewchanger
// d := req.GetAck().Digest
// if v != view || vc != viewchanger || !EQ(d, digest) {
// continue
// }
// if r == replica {
// plog.Info("Replica %d sent multiple ack requests\n", replica)
// continue
// }
// count++
// if rep.twoThirds(count) {
// twoThirds <- true
// return
// }
// }
// twoThirds <- false
// }()
//
// req := <-reqViewChange
//
// if req == nil || !<-twoThirds {
// return
// }
//
// rep.logPendingVC(req)
//
// // When to send new view?
// rep.requestNewView(view)
//}
//func (rep *Replica) handleRequestNewView(REQ *pb.Request) {
//
// view := REQ.GetNewview().View
//
// if view == 0 || view < rep.view {
// return
// }
//
// replica := REQ.GetNewview().Replica
// primary := rep.newPrimary(view)
//
// if replica != primary {
// return
// }
//
// if rep.hasRequest(REQ) {
// return
// }
//
// rep.logRequest(REQ)
//
// rep.processNewView(REQ)
//}
//func (rep *Replica) correctViewChanges(viewChanges []*pb.ViewChange) (requests []*pb.Request) {
//
// // Returns requests if correct, else returns nil
//
// valid := false
// for _, vc := range viewChanges {
// for _, req := range rep.requests["view-change"] {
// d := ReqDigest(req)
// if !EQ(d, vc.Digest) {
// continue
// }
// requests = append(requests, req)
// v := req.GetViewchange().View
// // VIEW or rep.view??
// if v == rep.view {
// valid = true
// break
// }
// }
// if !valid {
// return nil
// }
// }
//
// if rep.isPrimary(rep.ID) {
// reps := make(map[uint32]int)
// valid = false
// for _, req := range rep.requests["ack"] {
// reqAck := req.GetAck()
// reps[reqAck.Replica]++
// if rep.twoThirds(reps[reqAck.Replica]) { //-2
// valid = true
// break
// }
// }
// if !valid {
// return nil
// }
// }
//
// return
//}
//func (rep *Replica) correctSummaries(requests []*pb.Request, summaries []*pb.Summary) (correct bool) {
//
// // Verify SUMMARIES
//
// var start uint32
// var digest []byte
// digests := make(map[uint32][]byte)
//
// for _, summary := range summaries {
// s := summary.Sequence
// d := summary.Digest
// if _d, ok := digests[s]; ok && !EQ(_d, d) {
// return
// } else if !ok {
// digests[s] = d
// }
// if s < start || start == uint32(0) {
// start = s
// digest = d
// }
// }
//
// var A1 []*pb.Request
// var A2 []*pb.Request
//
// valid := false
// for _, req := range requests {
// reqViewChange := req.GetViewchange()
// s := reqViewChange.Sequence
// if s <= start {
// A1 = append(A1, req)
// }
// checkpoints := reqViewChange.GetCheckpoints()
// for _, checkpoint := range checkpoints {
// if checkpoint.Sequence == start && EQ(checkpoint.Digest, digest) {
// A2 = append(A2, req)
// break
// }
// }
// if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
// valid = true
// break
// }
// }
//
// if !valid {
// return
// }
//
// end := start + CheckPointPeriod*ConstantFactor
//
// for seq := start; seq <= end; seq++ {
//
// valid = false
//
// for _, summary := range summaries {
//
// if summary.Sequence != seq {
// continue
// }
//
// if summary.Digest != nil {
//
// var view uint32
//
// for _, req := range requests {
// reqViewChange := req.GetViewchange()
// preps := reqViewChange.GetPreps()
// for _, prep := range preps {
// s := prep.Sequence
// d := prep.Digest
// if s != summary.Sequence || !EQ(d, summary.Digest) {
// continue
// }
// v := prep.View
// if v > view {
// view = v
// }
// }
// }
//
// verifiedA1 := make(chan bool, 1)
//
// // Verify A1
// go func() {
//
// var A1 []*pb.Request
//
// FOR_LOOP:
// for _, req := range requests {
// reqViewChange := req.GetViewchange()
// s := reqViewChange.Sequence
// if s >= summary.Sequence {
// continue
// }
// preps := reqViewChange.GetPreps()
// for _, prep := range preps {
// s = prep.Sequence
// if s != summary.Sequence {
// continue
// }
// d := prep.Digest
// v := prep.View
// if v > view || (v == view && !EQ(d, summary.Digest)) {
// continue FOR_LOOP
// }
// }
// A1 = append(A1, req)
// if rep.twoThirds(len(A1)) {
// verifiedA1 <- true
// return
// }
// }
// verifiedA1 <- false
// }()
//
// verifiedA2 := make(chan bool, 1)
//
// // Verify A2
// go func() {
//
// var A2 []*pb.Request
//
// for _, req := range requests {
// reqViewChange := req.GetViewchange()
// prePreps := reqViewChange.GetPrepreps()
// for _, prePrep := range prePreps {
// s := prePrep.Sequence
// d := prePrep.Digest
// v := prePrep.View
// if s == summary.Sequence && EQ(d, summary.Digest) && v >= view {
// A2 = append(A2, req)
// break
// }
// }
// if rep.oneThird(len(A2)) {
// verifiedA2 <- true
// return
// }
// }
// verifiedA2 <- false
// }()
//
// if !<-verifiedA1 || !<-verifiedA2 {
// continue
// }
//
// valid = true
// break
//
// } else {
//
// var A1 []*pb.Request
//
// FOR_LOOP:
//
// for _, req := range requests {
//
// reqViewChange := req.GetViewchange()
//
// s := reqViewChange.Sequence
//
// if s >= summary.Sequence {
// continue
// }
//
// preps := reqViewChange.GetPreps()
// for _, prep := range preps {
// if prep.Sequence == summary.Sequence {
// continue FOR_LOOP
// }
// }
//
// A1 = append(A1, req)
// if rep.twoThirds(len(A1)) {
// valid = true
// break
// }
// }
// if valid {
// break
// }
// }
// }
// if !valid {
// return
// }
// }
//
// return true
//}
//func (rep *Replica) processNewView(REQ *pb.Request) (success bool) {
//
// if rep.activeView {
// return
// }
//
// reqNewView := REQ.GetNewview()
//
// viewChanges := reqNewView.GetViewchanges()
// requests := rep.correctViewChanges(viewChanges)
//
// if requests == nil {
// return
// }
//
// summaries := reqNewView.GetSummaries()
// correct := rep.correctSummaries(requests, summaries)
//
// if !correct {
// return
// }
//
// var h uint32
// for _, checkpoint := range rep.checkpoints {
// if checkpoint.Sequence < h || h == uint32(0) {
// h = checkpoint.Sequence
// }
// }
//
// var s uint32
// for _, summary := range summaries {
// if summary.Sequence < s || s == uint32(0) {
// s = summary.Sequence
// }
// if summary.Sequence > h {
// valid := false
// for _, req := range rep.requests["view-change"] { //in
// if EQ(ReqDigest(req), summary.Digest) {
// valid = true
// break
// }
// }
// if !valid {
// return
// }
// }
// }
//
// if h < s {
// return
// }
//
// // Process new view
// rep.activeView = true
//
// for _, summary := range summaries {
//
// if rep.ID != reqNewView.Replica {
// req := ToRequestPrepare(
// reqNewView.View,
// summary.Sequence,
// summary.Digest,
// rep.ID) // the backup sends/logs prepare
//
// go func() {
// if !rep.hasRequest(req) {
// rep.requestChan <- req
// }
// }()
// if summary.Sequence <= h {
// continue
// }
//
// if !rep.hasRequest(req) {
// rep.logRequest(req)
// }
// } else {
// if summary.Sequence <= h {
// break
// }
// }
//
// req := ToRequestPreprepare(
// reqNewView.View,
// summary.Sequence,
// summary.Digest,
// reqNewView.Replica) // new primary pre-prepares
//
// if !rep.hasRequest(req) {
// rep.logRequest(req)
// }
// }
//
// var maxSequence uint32
// for _, req := range rep.requests["pre-prepare"] {
// reqPrePrepare := req.GetPreprepare()
// if reqPrePrepare.Sequence > maxSequence {
// maxSequence = reqPrePrepare.Sequence
// }
// }
// rep.sequence = maxSequence
// return true
//}
//func (rep *Replica) prePrepBySequence(sequence uint32) []*pb.Entry {
// var view uint32
// var requests []*pb.Request
// for _, req := range rep.requests["pre-prepare"] {
// v := req.GetPreprepare().View
// s := req.GetPreprepare().Sequence
// if v >= view && s == sequence {
// view = v
// requests = append(requests, req)
// }
// }
// if requests == nil {
// return nil
// }
// var prePreps []*pb.Entry
// for _, req := range requests {
// v := req.GetPreprepare().View
// if v == view {
// s := req.GetPreprepare().Sequence
// d := req.GetPreprepare().Digest
// prePrep := ToEntry(s, d, v)
// prePreps = append(prePreps, prePrep)
// }
// }
//FOR_LOOP:
// for _, prePrep := range prePreps {
// for _, req := range rep.allRequests() { //TODO: optimize
// if EQ(ReqDigest(req), prePrep.Digest) {
// continue FOR_LOOP
// }
// }
// return nil
// }
// return prePreps
//}
//
//func (rep *Replica) prepBySequence(sequence uint32) ([]*pb.Entry, []*pb.Entry) {
//
// prePreps := rep.prePrepBySequence(sequence)
//
// if prePreps == nil {
// return nil, nil
// }
//
// var preps []*pb.Entry
//
//FOR_LOOP:
// for _, prePrep := range prePreps {
//
// view := prePrep.View
// digest := prePrep.Digest
//
// replicas := make(map[uint32]int)
// for _, req := range rep.requests["prepare"] {
// reqPrepare := req.GetPrepare()
// v := reqPrepare.View
// s := reqPrepare.Sequence
// d := reqPrepare.Digest
// if v == view && s == sequence && EQ(d, digest) {
// r := reqPrepare.Replica
// replicas[r]++
// if rep.twoThirds(replicas[r]) {
// prep := ToEntry(s, d, v)
// preps = append(preps, prep)
// continue FOR_LOOP
// }
// }
// }
// return prePreps, nil
// }
//
// return prePreps, preps
//}
/*
func (rep *Replica) prepBySequence(sequence uint32) *pb.Entry {
......@@ -1420,187 +1420,187 @@ func (rep *Replica) prePrepBySequence(sequence uint32) *pb.Entry {
}
*/
func (rep *Replica) requestViewChange(view uint32) {
if view != rep.view+1 {
return
}
rep.view = view
rep.activeView = false
var prePreps []*pb.Entry
var preps []*pb.Entry
start := rep.lowWaterMark() + 1
end := rep.highWaterMark()
for s := start; s <= end; s++ {
_prePreps, _preps := rep.prepBySequence(s)
if _prePreps != nil {
prePreps = append(prePreps, _prePreps...)
}
if _preps != nil {
preps = append(preps, _preps...)
}
}
sequence := rep.lowWaterMark()
req := ToRequestViewChange(
view,
sequence,
rep.checkpoints, //
preps,
prePreps,
rep.ID)
rep.logRequest(req)
go func() {
rep.requestChan <- req
}()
rep.clearRequestsByView(view)
}
func (rep *Replica) createNewView(view uint32) (request *pb.Request) {
// Returns RequestNewView if successful, else returns nil
// create viewChanges
viewChanges := make([]*pb.ViewChange, len(rep.pendingVC))
for idx := range viewChanges {
req := rep.pendingVC[idx]
viewchanger := req.GetViewchange().Replica
vc := ToViewChange(viewchanger, ReqDigest(req))
viewChanges[idx] = vc
}
var summaries []*pb.Summary
var summary *pb.Summary
start := rep.lowWaterMark() + 1
end := rep.highWaterMark()
// select starting checkpoint
FOR_LOOP_1:
for seq := start; seq <= end; seq++ {
overLWM := 0
var digest []byte
digests := make(map[string]int)
for _, req := range rep.pendingVC {
reqViewChange := req.GetViewchange()
if reqViewChange.Sequence <= seq {
overLWM++
}
for _, checkpoint := range reqViewChange.GetCheckpoints() {
if checkpoint.Sequence == seq {
d := checkpoint.Digest
digests[string(d)]++
if rep.oneThird(digests[string(d)]) {
digest = d
break
}
}
}
if rep.twoThirds(overLWM) && rep.oneThird(digests[string(digest)]) {
summary = ToSummary(seq, digest)
continue FOR_LOOP_1
}
}
}
if summary == nil {
return
}
summaries = append(summaries, summary)
start = summary.Sequence
end = start + CheckPointPeriod*ConstantFactor
// select summaries
// TODO: optimize
FOR_LOOP_2:
for seq := start; seq <= end; seq++ {
for _, REQ := range rep.pendingVC {
sequence := REQ.GetViewchange().Sequence
if sequence != seq {
continue
}
var A1 []*pb.Request
var A2 []*pb.Request
view := REQ.GetViewchange().View
digest := ReqDigest(REQ)
FOR_LOOP_3:
for _, req := range rep.pendingVC {
reqViewChange := req.GetViewchange()
if reqViewChange.Sequence < sequence {
preps := reqViewChange.GetPreps()
for _, prep := range preps {
if prep.Sequence != sequence {
continue
}
if prep.View > view || (prep.View == view && !EQ(prep.Digest, digest)) {
continue FOR_LOOP_3
}
}
A1 = append(A1, req)
}
prePreps := reqViewChange.GetPrepreps()
for _, prePrep := range prePreps {
if prePrep.Sequence != sequence {
continue
}
if prePrep.View >= view && EQ(prePrep.Digest, digest) {
A2 = append(A2, req)
continue FOR_LOOP_3
}
}
}
if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
summary = ToSummary(sequence, digest)
summaries = append(summaries, summary)
continue FOR_LOOP_2
}
}
}
request = ToRequestNewView(view, viewChanges, summaries, rep.ID)
return
}
func (rep *Replica) requestNewView(view uint32) {
req := rep.createNewView(view)
if req == nil || rep.hasRequest(req) {
return
}
// Process new view
success := rep.processNewView(req)
if !success {
return
}
rep.logRequest(req)
go func() {
rep.requestChan <- req
}()
}
//func (rep *Replica) requestViewChange(view uint32) {
//
// if view != rep.view+1 {
// return
// }
// rep.view = view
// rep.activeView = false
//
// var prePreps []*pb.Entry
// var preps []*pb.Entry
//
// start := rep.lowWaterMark() + 1
// end := rep.highWaterMark()
//
// for s := start; s <= end; s++ {
// _prePreps, _preps := rep.prepBySequence(s)
// if _prePreps != nil {
// prePreps = append(prePreps, _prePreps...)
// }
// if _preps != nil {
// preps = append(preps, _preps...)
// }
// }
//
// sequence := rep.lowWaterMark()
//
// req := ToRequestViewChange(
// view,
// sequence,
// rep.checkpoints, //
// preps,
// prePreps,
// rep.ID)
//
// rep.logRequest(req)
//
// go func() {
// rep.requestChan <- req
// }()
//
// rep.clearRequestsByView(view)
//}
//
//func (rep *Replica) createNewView(view uint32) (request *pb.Request) {
//
// // Returns RequestNewView if successful, else returns nil
// // create viewChanges
// viewChanges := make([]*pb.ViewChange, len(rep.pendingVC))
//
// for idx := range viewChanges {
// req := rep.pendingVC[idx]
// viewchanger := req.GetViewchange().Replica
// vc := ToViewChange(viewchanger, ReqDigest(req))
// viewChanges[idx] = vc
// }
//
// var summaries []*pb.Summary
// var summary *pb.Summary
//
// start := rep.lowWaterMark() + 1
// end := rep.highWaterMark()
//
// // select starting checkpoint
//FOR_LOOP_1:
// for seq := start; seq <= end; seq++ {
//
// overLWM := 0
// var digest []byte
// digests := make(map[string]int)
//
// for _, req := range rep.pendingVC {
// reqViewChange := req.GetViewchange()
// if reqViewChange.Sequence <= seq {
// overLWM++
// }
// for _, checkpoint := range reqViewChange.GetCheckpoints() {
// if checkpoint.Sequence == seq {
// d := checkpoint.Digest
// digests[string(d)]++
// if rep.oneThird(digests[string(d)]) {
// digest = d
// break
// }
// }
// }
// if rep.twoThirds(overLWM) && rep.oneThird(digests[string(digest)]) {
// summary = ToSummary(seq, digest)
// continue FOR_LOOP_1
// }
// }
// }
//
// if summary == nil {
// return
// }
//
// summaries = append(summaries, summary)
//
// start = summary.Sequence
// end = start + CheckPointPeriod*ConstantFactor
//
// // select summaries
// // TODO: optimize
//FOR_LOOP_2:
// for seq := start; seq <= end; seq++ {
//
// for _, REQ := range rep.pendingVC {
//
// sequence := REQ.GetViewchange().Sequence
//
// if sequence != seq {
// continue
// }
//
// var A1 []*pb.Request
// var A2 []*pb.Request
//
// view := REQ.GetViewchange().View
// digest := ReqDigest(REQ)
//
// FOR_LOOP_3:
// for _, req := range rep.pendingVC {
//
// reqViewChange := req.GetViewchange()
//
// if reqViewChange.Sequence < sequence {
// preps := reqViewChange.GetPreps()
// for _, prep := range preps {
// if prep.Sequence != sequence {
// continue
// }
// if prep.View > view || (prep.View == view && !EQ(prep.Digest, digest)) {
// continue FOR_LOOP_3
// }
// }
// A1 = append(A1, req)
// }
// prePreps := reqViewChange.GetPrepreps()
// for _, prePrep := range prePreps {
// if prePrep.Sequence != sequence {
// continue
// }
// if prePrep.View >= view && EQ(prePrep.Digest, digest) {
// A2 = append(A2, req)
// continue FOR_LOOP_3
// }
// }
// }
//
// if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
// summary = ToSummary(sequence, digest)
// summaries = append(summaries, summary)
// continue FOR_LOOP_2
// }
// }
// }
//
// request = ToRequestNewView(view, viewChanges, summaries, rep.ID)
// return
//}
//
//func (rep *Replica) requestNewView(view uint32) {
//
// req := rep.createNewView(view)
//
// if req == nil || rep.hasRequest(req) {
// return
// }
//
// // Process new view
//
// success := rep.processNewView(req)
//
// if !success {
// return
// }
//
// rep.logRequest(req)
//
// go func() {
// rep.requestChan <- req
// }()
//
//}
plugin/consensus/pbft/pbft_test.go
View file @ 6bf4628e
......@@ -150,3 +150,39 @@ func clearTestData() {
}
fmt.Println("test data clear successfully!")
}
func TestReplica(t *testing.T){
reply1, request1, isPrimary1:=NewReplica(1,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8891")
reply2, request2, isPrimary2:=NewReplica(2,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8892")
reply3, request3, isPrimary3:=NewReplica(3,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8893")
go func() {
for {
select {
case <-reply1:
t.Log("I'm have reply1 message")
case <-reply2:
t.Log("I'm have reply2 message")
case <-reply3:
t.Log("I'm have reply3 message")
}
}
}()
t.Log(isPrimary1)
t.Log(isPrimary2)
t.Log(isPrimary3)
go func() {
for i:=0;i<100;i++ {
op := &types.Operation{Value: &types.Block{}}
select {
case request1<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8891"):
t.Log("I'm have send request1 message")
case request2<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8892"):
t.Log("I'm have send request2 message")
case request3<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8893"):
t.Log("I'm have send request3 message")
}
}
}()
time.Sleep(5*time.Second)
}
\ No newline at end of file
plugin/consensus/raft/block.go
View file @ 6bf4628e
......@@ -5,6 +5,7 @@
package raft
import (
"context"
"fmt"
"sync"
"time"
......@@ -35,14 +36,15 @@ type Client struct {
errorC <-chan error
snapshotter *snap.Snapshotter
validatorC <-chan bool
stopC chan<- struct{}
ctx context.Context
cancel context.CancelFunc
once sync.Once
}
// NewBlockstore create Raft Client
func NewBlockstore(cfg *types.Consensus, snapshotter *snap.Snapshotter, proposeC chan<- *types.Block, commitC <-chan *types.Block, errorC <-chan error, validatorC <-chan bool, stopC chan<- struct{}) *Client {
func NewBlockstore(ctx context.Context, cfg *types.Consensus, snapshotter *snap.Snapshotter, proposeC chan<- *types.Block, commitC <-chan *types.Block, errorC <-chan error, validatorC <-chan bool, cancel context.CancelFunc) *Client {
c := drivers.NewBaseClient(cfg)
client := &Client{BaseClient: c, proposeC: proposeC, snapshotter: snapshotter, validatorC: validatorC, commitC: commitC, errorC: errorC, stopC: stopC}
client := &Client{BaseClient: c, proposeC: proposeC, snapshotter: snapshotter, validatorC: validatorC, commitC: commitC, errorC: errorC, ctx: ctx, cancel: cancel}
c.SetChild(client)
return client
}
......@@ -97,12 +99,12 @@ func (client *Client) SetQueueClient(c queue.Client) {
})
go client.EventLoop()
go client.readCommits(client.commitC, client.errorC)
go client.pollingTask(c)
go client.pollingTask()
}
// Close method
func (client *Client) Close() {
client.stopC <- struct{}{}
client.cancel()
rlog.Info("consensus raft closed")
}
......@@ -125,10 +127,14 @@ func (client *Client) CreateBlock() {
panic("This node encounter problem, exit.")
}
}
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-client.ctx.Done():
case <-ticker.C:
//如果leader节点突然挂了，不是打包节点，需要退出
if !isLeader {
if !mux.Load().(bool) {
rlog.Warn("I'm not the validator node anymore, exit.=============================")
break
}
......@@ -193,6 +199,8 @@ func (client *Client) CreateBlock() {
}
time.Sleep(time.Second * time.Duration(writeBlockSeconds))
}
}
}
// 向raft底层发送block
......@@ -219,17 +227,21 @@ func (client *Client) readCommits(commitC <-chan *types.Block, errorC <-chan err
if ok {
panic(err)
}
case <-client.ctx.Done():
return
}
}
}
//轮询任务，去检测本机器是否为validator节点，如果是，则执行打包任务
func (client *Client) pollingTask(c queue.Client) {
func (client *Client) pollingTask() {
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-client.ctx.Done():
return
case value, ok := <-client.validatorC:
//各个节点Block只初始化一次
client.once.Do(func() {
......@@ -237,9 +249,14 @@ func (client *Client) pollingTask(c queue.Client) {
})
if ok && !value {
rlog.Debug("================I'm not the validator node!=============")
leader := mux.Load().(bool)
if leader {
isLeader = false
} else if ok && !isLeader && value {
mux.Store(isLeader)
}
} else if ok && !mux.Load().(bool) && value {
isLeader = true
mux.Store(isLeader)
go client.CreateBlock()
} else if !ok {
break
......
plugin/consensus/raft/chain33.test.toml
View file @ 6bf4628e
......@@ -36,8 +36,8 @@ enableTxQuickIndex=true
[p2p]
seeds=["127.0.0.1:13802"]
enable=true
isSeed=true
enable=false
isSeed=false
serverStart=true
innerSeedEnable=false
useGithub=false
......@@ -107,8 +107,8 @@ peersURL="http://127.0.0.1:9021"
# raft共识用到，指示raft集群中只读节点的IP（只同步日志，不参与raft共识）
readOnlyPeersURL=""
addPeersURL=""
#raft共识用到，默认raft中多少条记录打包一个snapshot
defaultSnapCount=1000
#raft共识用到，默认raft中多少条记录打包一个snapshot(这里为了测试调整小一点）
defaultSnapCount=2
#raft共识用到，默认raft中写区块时间间隔
writeBlockSeconds=1
#raft共识用到，默认raft中leader发送心跳包时间间隔
......
plugin/consensus/raft/controller.go
View file @ 6bf4628e
......@@ -5,7 +5,9 @@
package raft
import (
"context"
"strings"
"sync/atomic"
log "github.com/33cn/chain33/common/log/log15"
"github.com/33cn/chain33/queue"
......@@ -22,6 +24,7 @@ var (
writeBlockSeconds int64 = 1
heartbeatTick = 1
isLeader = false
mux atomic.Value
confChangeC chan raftpb.ConfChange
)
......@@ -39,6 +42,10 @@ type subConfig struct {
HeartbeatTick int32 `json:"heartbeatTick"`
}
func init() {
mux.Store(isLeader)
}
// NewRaftCluster create raft cluster
func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
rlog.Info("Start to create raft cluster")
......@@ -70,10 +77,6 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
if subcfg.HeartbeatTick > 0 {
heartbeatTick = int(subcfg.HeartbeatTick)
}
// propose channel
proposeC := make(chan *types.Block)
confChangeC = make(chan raftpb.ConfChange)
var b *Client
getSnapshot := func() ([]byte, error) { return b.getSnapshot() }
// raft集群的建立,1. 初始化两条channel： propose channel用于客户端和raft底层交互, commit channel用于获取commit消息
......@@ -90,10 +93,15 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
if len(addPeers) == 1 && addPeers[0] == "" {
addPeers = []string{}
}
commitC, errorC, snapshotterReady, validatorC, stopC := NewRaftNode(int(subcfg.NodeID), subcfg.IsNewJoinNode, peers, readOnlyPeers, addPeers, getSnapshot, proposeC, confChangeC)
//采用context来统一管理所有服务
ctx, stop := context.WithCancel(context.Background())
// propose channel
proposeC := make(chan *types.Block)
confChangeC = make(chan raftpb.ConfChange)
commitC, errorC, snapshotterReady, validatorC := NewRaftNode(ctx, int(subcfg.NodeID), subcfg.IsNewJoinNode, peers, readOnlyPeers, addPeers, getSnapshot, proposeC, confChangeC)
//启动raft删除节点操作监听
go serveHTTPRaftAPI(int(subcfg.RaftAPIPort), confChangeC, errorC)
go serveHTTPRaftAPI(ctx, int(subcfg.RaftAPIPort), confChangeC, errorC)
// 监听commit channel,取block
b = NewBlockstore(cfg, <-snapshotterReady, proposeC, commitC, errorC, validatorC, stopC)
b = NewBlockstore(ctx, cfg, <-snapshotterReady, proposeC, commitC, errorC, validatorC, stop)
return b
}
plugin/consensus/raft/httpapi.go
View file @ 6bf4628e
......@@ -5,6 +5,7 @@
package raft
import (
"context"
"io/ioutil"
"net/http"
"strconv"
......@@ -66,8 +67,8 @@ func (h *httpRaftAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
func serveHTTPRaftAPI(port int, confChangeC chan<- raftpb.ConfChange, errorC <-chan error) {
srv := http.Server{
func serveHTTPRaftAPI(ctx context.Context, port int, confChangeC chan<- raftpb.ConfChange, errorC <-chan error) {
srv := &http.Server{
Addr: "localhost:" + strconv.Itoa(port),
Handler: &httpRaftAPI{
confChangeC: confChangeC,
......@@ -78,9 +79,11 @@ func serveHTTPRaftAPI(port int, confChangeC chan<- raftpb.ConfChange, errorC <-c
rlog.Error(fmt.Sprintf("ListenAndServe have a err: (%v)", err.Error()))
}
}()
// exit when raft goes down
if err, ok := <-errorC; ok {
select {
case <-ctx.Done():
srv.Close()
case err := <-errorC:
srv.Close()
rlog.Error(fmt.Sprintf("the errorC chan receive a err (%v)\n", err.Error()))
}
}
plugin/consensus/raft/listener.go
View file @ 6bf4628e
......@@ -5,6 +5,7 @@
package raft
import (
"context"
"errors"
"net"
"time"
......@@ -13,16 +14,16 @@ import (
// 设置TCP keep-alive超时，接收stopc
type stoppableListener struct {
*net.TCPListener
stopc <-chan struct{}
ctx context.Context
}
// 监听tcp连接
func newStoppableListener(addr string, stopc <-chan struct{}) (*stoppableListener, error) {
func newStoppableListener(addr string, ctx context.Context) (*stoppableListener, error) {
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, err
}
return &stoppableListener{ln.(*net.TCPListener), stopc}, nil
return &stoppableListener{ln.(*net.TCPListener), ctx}, nil
}
func (ln stoppableListener) Accept() (c net.Conn, err error) {
......@@ -37,7 +38,7 @@ func (ln stoppableListener) Accept() (c net.Conn, err error) {
connc <- tc
}()
select {
case <-ln.stopc:
case <-ln.ctx.Done():
return nil, errors.New("server stopped")
case err := <-errc:
return nil, err
......
plugin/consensus/raft/raft.go
View file @ 6bf4628e
......@@ -56,17 +56,18 @@ type raftNode struct {
snapCount uint64
transport *rafthttp.Transport
stopMu sync.RWMutex
stopc chan struct{}
httpstopc chan struct{}
httpdonec chan struct{}
ctx context.Context
//stopc chan struct{}
//httpstopc chan struct{}
//httpdonec chan struct{}
validatorC chan bool
//用于判断该节点是否重启过
restartC chan struct{}
}
// NewRaftNode create raft node
func NewRaftNode(id int, join bool, peers []string, readOnlyPeers []string, addPeers []string, getSnapshot func() ([]byte, error), proposeC <-chan *types.Block,
confChangeC <-chan raftpb.ConfChange) (<-chan *types.Block, <-chan error, <-chan *snap.Snapshotter, <-chan bool, chan<- struct{}) {
func NewRaftNode(ctx context.Context, id int, join bool, peers []string, readOnlyPeers []string, addPeers []string, getSnapshot func() ([]byte, error), proposeC <-chan *types.Block,
confChangeC <-chan raftpb.ConfChange) (<-chan *types.Block, <-chan error, <-chan *snap.Snapshotter, <-chan bool) {
rlog.Info("Enter consensus raft")
// commit channel
......@@ -86,16 +87,14 @@ func NewRaftNode(id int, join bool, peers []string, readOnlyPeers []string, addP
snapdir: fmt.Sprintf("chain33_raft-%d%ssnap", id, string(os.PathSeparator)),
getSnapshot: getSnapshot,
snapCount: defaultSnapCount,
stopc: make(chan struct{}),
httpstopc: make(chan struct{}),
httpdonec: make(chan struct{}),
validatorC: make(chan bool),
snapshotterReady: make(chan *snap.Snapshotter, 1),
restartC: make(chan struct{}, 1),
ctx: ctx,
}
go rc.startRaft()
return commitC, errorC, rc.snapshotterReady, rc.validatorC, rc.stopc
return commitC, errorC, rc.snapshotterReady, rc.validatorC
}
// 启动raft节点
......@@ -184,22 +183,22 @@ func (rc *raftNode) serveRaft() {
panic(err)
}
ln, err := newStoppableListener(nodeURL.Host, rc.httpstopc)
ln, err := newStoppableListener(nodeURL.Host, rc.ctx)
if err != nil {
rlog.Error(fmt.Sprintf("raft: Failed to listen rafthttp (%v)", err.Error()))
panic(err)
}
err = (&http.Server{Handler: rc.transport.Handler()}).Serve(ln)
raftSrv := &http.Server{Handler: rc.transport.Handler()}
err = raftSrv.Serve(ln)
if err != nil {
rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error()))
}
select {
case <-rc.httpstopc:
case <-rc.ctx.Done():
raftSrv.Close()
default:
rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error()))
}
close(rc.httpdonec)
}
func (rc *raftNode) serveChannels() {
......@@ -246,9 +245,11 @@ func (rc *raftNode) serveChannels() {
rlog.Error(fmt.Sprintf("rc.node.ProposeConfChange:%v", err.Error()))
}
}
case <-rc.ctx.Done():
rlog.Info("I have a exit message!")
return
}
}
close(rc.stopc)
}()
// 从Ready()中接收数据
for {
......@@ -275,7 +276,7 @@ func (rc *raftNode) serveChannels() {
rc.writeError(err)
return
case <-rc.stopc:
case <-rc.ctx.Done():
rc.stop()
return
}
......@@ -283,9 +284,9 @@ func (rc *raftNode) serveChannels() {
}
func (rc *raftNode) updateValidator() {
//TODO 这块监听后期需要根据场景进行优化?
time.Sleep(5 * time.Second)
//用于标记readOnlyPeers是否已经被添加到集群中了
flag := false
isRestart := false
......@@ -297,8 +298,12 @@ func (rc *raftNode) updateValidator() {
case <-ticker.C:
ticker.Stop()
}
ticker = time.NewTicker(time.Second)
for {
time.Sleep(time.Second)
select {
case <-rc.ctx.Done():
return
case <-ticker.C:
status := rc.Status()
if status.Lead == raft.None {
rlog.Debug(fmt.Sprintf("==============This is %s node!==============", status.RaftState.String()))
......@@ -317,6 +322,8 @@ func (rc *raftNode) updateValidator() {
flag = true
}
}
}
}
func (rc *raftNode) Status() raft.Status {
rc.stopMu.RLock()
......@@ -454,20 +461,18 @@ func (rc *raftNode) stop() {
rc.stopHTTP()
close(rc.commitC)
close(rc.errorC)
close(rc.stopc)
rc.node.Stop()
}
func (rc *raftNode) stopHTTP() {
rc.transport.Stop()
close(rc.httpstopc)
<-rc.httpdonec
//close(rc.httpstopc)
//<-rc.httpdonec
}
func (rc *raftNode) writeError(err error) {
rc.stopHTTP()
close(rc.commitC)
close(rc.stopc)
rc.errorC <- err
close(rc.errorC)
rc.node.Stop()
......@@ -488,7 +493,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
}
select {
case rc.commitC <- block:
case <-rc.stopc:
case <-rc.ctx.Done():
return false
}
......@@ -521,7 +526,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
if ents[i].Index == rc.lastIndex {
select {
case rc.commitC <- nil:
case <-rc.stopc:
case <-rc.ctx.Done():
return false
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment