Skip to content

P
plugin
Commit 6bf4628e authored by harrylee's avatar harrylee Committed by vipwzw
Browse files
Options

ajust code for raft and pbft

parent e89dbbfa
Show whitespace changes
Inline Side-by-side
Showing with 1070 additions and 1001 deletions
go.mod
View file @ 6bf4628e
...@@ -20,8 +20,6 @@ require ( ...@@ -20,8 +20,6 @@ require (
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/golang/protobuf v1.3.2 github.com/golang/protobuf v1.3.2
github.com/hashicorp/golang-lru v0.5.0 github.com/hashicorp/golang-lru v0.5.0
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.8.0 github.com/pkg/errors v0.8.0
github.com/prometheus/client_golang v1.1.0 // indirect github.com/prometheus/client_golang v1.1.0 // indirect
github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d
......
plugin/consensus/pbft/pbft.go
View file @ 6bf4628e
...@@ -277,12 +277,12 @@ func (rep *Replica) hasRequest(REQ *pb.Request) bool { ...@@ -277,12 +277,12 @@ func (rep *Replica) hasRequest(REQ *pb.Request) bool {
return rep.hasRequestPrepare(REQ) return rep.hasRequestPrepare(REQ)
case *pb.Request_Commit: case *pb.Request_Commit:
return rep.hasRequestCommit(REQ) return rep.hasRequestCommit(REQ)
case *pb.Request_Viewchange: //case *pb.Request_Viewchange:
return rep.hasRequestViewChange(REQ) // return rep.hasRequestViewChange(REQ)
case *pb.Request_Ack: //case *pb.Request_Ack:
return rep.hasRequestAck(REQ) // return rep.hasRequestAck(REQ)
case *pb.Request_Newview: //case *pb.Request_Newview:
return rep.hasRequestNewView(REQ) // return rep.hasRequestNewView(REQ)
default: default:
return false return false
} }
...@@ -335,160 +335,160 @@ func (rep *Replica) hasRequestCommit(REQ *pb.Request) bool { ...@@ -335,160 +335,160 @@ func (rep *Replica) hasRequestCommit(REQ *pb.Request) bool {
return false return false
} }
func (rep *Replica) hasRequestViewChange(REQ *pb.Request) bool { //func (rep *Replica) hasRequestViewChange(REQ *pb.Request) bool {
view := REQ.GetViewchange().View // view := REQ.GetViewchange().View
replica := REQ.GetViewchange().Replica // replica := REQ.GetViewchange().Replica
for _, req := range rep.requests["view-change"] { // for _, req := range rep.requests["view-change"] {
v := req.GetViewchange().View // v := req.GetViewchange().View
r := req.GetViewchange().Replica // r := req.GetViewchange().Replica
if v == view && r == replica { // if v == view && r == replica {
return true // return true
} // }
} // }
return false // return false
} //}
func (rep *Replica) hasRequestAck(REQ *pb.Request) bool { //func (rep *Replica) hasRequestAck(REQ *pb.Request) bool {
view := REQ.GetAck().View // view := REQ.GetAck().View
replica := REQ.GetAck().Replica // replica := REQ.GetAck().Replica
viewchanger := REQ.GetAck().Viewchanger // viewchanger := REQ.GetAck().Viewchanger
for _, req := range rep.requests["ack"] { // for _, req := range rep.requests["ack"] {
v := req.GetAck().View // v := req.GetAck().View
r := req.GetAck().Replica // r := req.GetAck().Replica
vc := req.GetAck().Viewchanger // vc := req.GetAck().Viewchanger
if v == view && r == replica && vc == viewchanger { // if v == view && r == replica && vc == viewchanger {
return true // return true
} // }
} // }
return false // return false
} //}
func (rep *Replica) hasRequestNewView(REQ *pb.Request) bool { //func (rep *Replica) hasRequestNewView(REQ *pb.Request) bool {
view := REQ.GetNewview().View // view := REQ.GetNewview().View
for _, req := range rep.requests["new-view"] { // for _, req := range rep.requests["new-view"] {
v := req.GetNewview().View // v := req.GetNewview().View
if v == view { // if v == view {
return true // return true
} // }
} // }
return false // return false
} //}
// Clear requests // Clear requests
func (rep *Replica) clearRequestsBySeq(sequence uint32) { //func (rep *Replica) clearRequestsBySeq(sequence uint32) {
rep.clearRequestClients() // rep.clearRequestClients()
rep.clearRequestPrepreparesBySeq(sequence) // rep.clearRequestPrepreparesBySeq(sequence)
rep.clearRequestPreparesBySeq(sequence) // rep.clearRequestPreparesBySeq(sequence)
rep.clearRequestCommitsBySeq(sequence) // rep.clearRequestCommitsBySeq(sequence)
//rep.clearRequestCheckpointsBySeq(sequence) // //rep.clearRequestCheckpointsBySeq(sequence)
} //}
func (rep *Replica) clearRequestClients() { //func (rep *Replica) clearRequestClients() {
clientReqs := rep.requests["client"] // clientReqs := rep.requests["client"]
lastTimestamp := rep.theLastReply().Timestamp // lastTimestamp := rep.theLastReply().Timestamp
for idx, req := range rep.requests["client"] { // for idx, req := range rep.requests["client"] {
timestamp := req.GetClient().Timestamp // timestamp := req.GetClient().Timestamp
if lastTimestamp >= timestamp && idx < (len(clientReqs)-1) { // if lastTimestamp >= timestamp && idx < (len(clientReqs)-1) {
clientReqs = append(clientReqs[:idx], clientReqs[idx+1:]...) // clientReqs = append(clientReqs[:idx], clientReqs[idx+1:]...)
} else { // } else {
clientReqs = clientReqs[:idx-1] // clientReqs = clientReqs[:idx-1]
} // }
} // }
rep.requests["client"] = clientReqs // rep.requests["client"] = clientReqs
} //}
//
func (rep *Replica) clearRequestPrepreparesBySeq(sequence uint32) { //func (rep *Replica) clearRequestPrepreparesBySeq(sequence uint32) {
prePrepares := rep.requests["pre-prepare"] // prePrepares := rep.requests["pre-prepare"]
for idx, req := range rep.requests["pre-prepare"] { // for idx, req := range rep.requests["pre-prepare"] {
s := req.GetPreprepare().Sequence // s := req.GetPreprepare().Sequence
if s <= sequence && idx < (len(prePrepares)-1) { // if s <= sequence && idx < (len(prePrepares)-1) {
prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...) // prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
} else { // } else {
prePrepares = prePrepares[:idx-1] // prePrepares = prePrepares[:idx-1]
} // }
} // }
rep.requests["pre-prepare"] = prePrepares // rep.requests["pre-prepare"] = prePrepares
} //}
//
func (rep *Replica) clearRequestPreparesBySeq(sequence uint32) { //func (rep *Replica) clearRequestPreparesBySeq(sequence uint32) {
prepares := rep.requests["prepare"] // prepares := rep.requests["prepare"]
for idx, req := range rep.requests["prepare"] { // for idx, req := range rep.requests["prepare"] {
s := req.GetPrepare().Sequence // s := req.GetPrepare().Sequence
if s <= sequence && idx < (len(prepares)-1) { // if s <= sequence && idx < (len(prepares)-1) {
prepares = append(prepares[:idx], prepares[idx+1:]...) // prepares = append(prepares[:idx], prepares[idx+1:]...)
} else { // } else {
prepares = prepares[:idx-1] // prepares = prepares[:idx-1]
} // }
} // }
rep.requests["prepare"] = prepares // rep.requests["prepare"] = prepares
} //}
//
func (rep *Replica) clearRequestCommitsBySeq(sequence uint32) { //func (rep *Replica) clearRequestCommitsBySeq(sequence uint32) {
commits := rep.requests["commit"] // commits := rep.requests["commit"]
for idx, req := range rep.requests["commit"] { // for idx, req := range rep.requests["commit"] {
s := req.GetCommit().Sequence // s := req.GetCommit().Sequence
if s <= sequence && idx < (len(commits)-1) { // if s <= sequence && idx < (len(commits)-1) {
commits = append(commits[:idx], commits[idx+1:]...) // commits = append(commits[:idx], commits[idx+1:]...)
} else { // } else {
commits = commits[:idx-1] // commits = commits[:idx-1]
} // }
} // }
rep.requests["commit"] = commits // rep.requests["commit"] = commits
} //}
//
func (rep *Replica) clearRequestCheckpointsBySeq(sequence uint32) { //func (rep *Replica) clearRequestCheckpointsBySeq(sequence uint32) {
checkpoints := rep.requests["checkpoint"] // checkpoints := rep.requests["checkpoint"]
for idx, req := range rep.requests["checkpoint"] { // for idx, req := range rep.requests["checkpoint"] {
s := req.GetCheckpoint().Sequence // s := req.GetCheckpoint().Sequence
if s <= sequence && idx < (len(checkpoints)-1) { // if s <= sequence && idx < (len(checkpoints)-1) {
checkpoints = append(checkpoints[:idx], checkpoints[idx+1:]...) // checkpoints = append(checkpoints[:idx], checkpoints[idx+1:]...)
} else { // } else {
checkpoints = checkpoints[:idx-1] // checkpoints = checkpoints[:idx-1]
} // }
} // }
rep.requests["checkpoint"] = checkpoints // rep.requests["checkpoint"] = checkpoints
} //}
//
func (rep *Replica) clearRequestsByView(view uint32) { //func (rep *Replica) clearRequestsByView(view uint32) {
rep.clearRequestPrepreparesByView(view) // rep.clearRequestPrepreparesByView(view)
rep.clearRequestPreparesByView(view) // rep.clearRequestPreparesByView(view)
rep.clearRequestCommitsByView(view) // rep.clearRequestCommitsByView(view)
//add others // //add others
} //}
//
func (rep *Replica) clearRequestPrepreparesByView(view uint32) { //func (rep *Replica) clearRequestPrepreparesByView(view uint32) {
prePrepares := rep.requests["pre-prepare"] // prePrepares := rep.requests["pre-prepare"]
for idx, req := range rep.requests["pre-prepare"] { // for idx, req := range rep.requests["pre-prepare"] {
v := req.GetPreprepare().View // v := req.GetPreprepare().View
if v < view { // if v < view {
prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...) // prePrepares = append(prePrepares[:idx], prePrepares[idx+1:]...)
} // }
} // }
rep.requests["pre-prepare"] = prePrepares // rep.requests["pre-prepare"] = prePrepares
} //}
//
func (rep *Replica) clearRequestPreparesByView(view uint32) { //func (rep *Replica) clearRequestPreparesByView(view uint32) {
prepares := rep.requests["prepare"] // prepares := rep.requests["prepare"]
for idx, req := range rep.requests["prepare"] { // for idx, req := range rep.requests["prepare"] {
v := req.GetPrepare().View // v := req.GetPrepare().View
if v < view { // if v < view {
prepares = append(prepares[:idx], prepares[idx+1:]...) // prepares = append(prepares[:idx], prepares[idx+1:]...)
} // }
} // }
rep.requests["prepare"] = prepares // rep.requests["prepare"] = prepares
} //}
//
func (rep *Replica) clearRequestCommitsByView(view uint32) { //func (rep *Replica) clearRequestCommitsByView(view uint32) {
commits := rep.requests["commit"] // commits := rep.requests["commit"]
for idx, req := range rep.requests["commit"] { // for idx, req := range rep.requests["commit"] {
v := req.GetCommit().View // v := req.GetCommit().View
if v < view { // if v < view {
commits = append(commits[:idx], commits[idx+1:]...) // commits = append(commits[:idx], commits[idx+1:]...)
} // }
} // }
rep.requests["commit"] = commits // rep.requests["commit"] = commits
} //}
// Handle requests // Handle requests
...@@ -511,17 +511,17 @@ func (rep *Replica) handleRequest(REQ *pb.Request) { ...@@ -511,17 +511,17 @@ func (rep *Replica) handleRequest(REQ *pb.Request) {
rep.handleRequestCommit(REQ) rep.handleRequestCommit(REQ)
case *pb.Request_Checkpoint: //case *pb.Request_Checkpoint:
//
rep.handleRequestCheckpoint(REQ) // rep.handleRequestCheckpoint(REQ)
//
case *pb.Request_Viewchange: //case *pb.Request_Viewchange:
//
rep.handleRequestViewChange(REQ) // rep.handleRequestViewChange(REQ)
//
case *pb.Request_Ack: //case *pb.Request_Ack:
//
rep.handleRequestAck(REQ) // rep.handleRequestAck(REQ)
default: default:
plog.Info("Replica %d received unrecognized request type\n", rep.ID) plog.Info("Replica %d received unrecognized request type\n", rep.ID)
...@@ -766,599 +766,599 @@ func (rep *Replica) handleRequestCommit(REQ *pb.Request) { ...@@ -766,599 +766,599 @@ func (rep *Replica) handleRequestCommit(REQ *pb.Request) {
} }
func (rep *Replica) handleRequestCheckpoint(REQ *pb.Request) { //func (rep *Replica) handleRequestCheckpoint(REQ *pb.Request) {
//
sequence := REQ.GetCheckpoint().Sequence // sequence := REQ.GetCheckpoint().Sequence
//
if !rep.sequenceInRange(sequence) { // if !rep.sequenceInRange(sequence) {
return // return
} // }
//
digest := REQ.GetCheckpoint().Digest // digest := REQ.GetCheckpoint().Digest
replica := REQ.GetCheckpoint().Replica // replica := REQ.GetCheckpoint().Replica
//
count := 0 // count := 0
for _, req := range rep.requests["checkpoint"] { // for _, req := range rep.requests["checkpoint"] {
s := req.GetCheckpoint().Sequence // s := req.GetCheckpoint().Sequence
d := req.GetCheckpoint().Digest // d := req.GetCheckpoint().Digest
r := req.GetCheckpoint().Replica // r := req.GetCheckpoint().Replica
if s != sequence || !EQ(d, digest) { // if s != sequence || !EQ(d, digest) {
continue // continue
} // }
if r == replica { // if r == replica {
plog.Info("Replica %d sent multiple checkpoint requests\n", replica) // plog.Info("Replica %d sent multiple checkpoint requests\n", replica)
//continue // //continue
} // }
count++ // count++
if !rep.overTwoThirds(count) { // if !rep.overTwoThirds(count) {
continue // continue
} // }
// rep.clearEntries(sequence) // // rep.clearEntries(sequence)
rep.clearRequestsBySeq(sequence) // rep.clearRequestsBySeq(sequence)
checkpoint := ToCheckpoint(sequence, digest) // checkpoint := ToCheckpoint(sequence, digest)
rep.addCheckpoint(checkpoint) // rep.addCheckpoint(checkpoint)
plog.Info("checkpoint and clear request done") // plog.Info("checkpoint and clear request done")
return // return
} // }
//
} //}
func (rep *Replica) handleRequestViewChange(REQ *pb.Request) { //func (rep *Replica) handleRequestViewChange(REQ *pb.Request) {
//
view := REQ.GetViewchange().View // view := REQ.GetViewchange().View
//
if view < rep.view { // if view < rep.view {
return // return
} // }
//
reqViewChange := REQ.GetViewchange() // reqViewChange := REQ.GetViewchange()
//
for _, prep := range reqViewChange.GetPreps() { // for _, prep := range reqViewChange.GetPreps() {
v := prep.View // v := prep.View
s := prep.Sequence // s := prep.Sequence
if v >= view || !rep.sequenceInRange(s) { // if v >= view || !rep.sequenceInRange(s) {
return // return
} // }
} // }
//
for _, prePrep := range reqViewChange.GetPrepreps() { // for _, prePrep := range reqViewChange.GetPrepreps() {
v := prePrep.View // v := prePrep.View
s := prePrep.Sequence // s := prePrep.Sequence
if v >= view || !rep.sequenceInRange(s) { // if v >= view || !rep.sequenceInRange(s) {
return // return
} // }
} // }
//
for _, checkpoint := range reqViewChange.GetCheckpoints() { // for _, checkpoint := range reqViewChange.GetCheckpoints() {
s := checkpoint.Sequence // s := checkpoint.Sequence
if !rep.sequenceInRange(s) { // if !rep.sequenceInRange(s) {
return // return
} // }
} // }
//
if rep.hasRequest(REQ) { // if rep.hasRequest(REQ) {
return // return
} // }
//
rep.logRequest(REQ) // rep.logRequest(REQ)
//
viewchanger := reqViewChange.Replica // viewchanger := reqViewChange.Replica
//
req := ToRequestAck( // req := ToRequestAck(
view, // view,
rep.ID, // rep.ID,
viewchanger, // viewchanger,
ReqDigest(REQ)) // ReqDigest(REQ))
//
go func() { // go func() {
rep.requestChan <- req // rep.requestChan <- req
}() // }()
} //}
func (rep *Replica) handleRequestAck(REQ *pb.Request) { //func (rep *Replica) handleRequestAck(REQ *pb.Request) {
//
view := REQ.GetAck().View // view := REQ.GetAck().View
primaryID := rep.newPrimary(view) // primaryID := rep.newPrimary(view)
//
if rep.ID != primaryID { // if rep.ID != primaryID {
return // return
} // }
//
if rep.hasRequest(REQ) { // if rep.hasRequest(REQ) {
return // return
} // }
//
rep.logRequest(REQ) // rep.logRequest(REQ)
//
replica := REQ.GetAck().Replica // replica := REQ.GetAck().Replica
viewchanger := REQ.GetAck().Viewchanger // viewchanger := REQ.GetAck().Viewchanger
digest := REQ.GetAck().Digest // digest := REQ.GetAck().Digest
//
reqViewChange := make(chan *pb.Request, 1) // reqViewChange := make(chan *pb.Request, 1)
twoThirds := make(chan bool, 1) // twoThirds := make(chan bool, 1)
//
go func() { // go func() {
for _, req := range rep.requests["view-change"] { // for _, req := range rep.requests["view-change"] {
v := req.GetViewchange().View // v := req.GetViewchange().View
vc := req.GetViewchange().Replica // vc := req.GetViewchange().Replica
if v == view && vc == viewchanger { // if v == view && vc == viewchanger {
reqViewChange <- req // reqViewChange <- req
} // }
} // }
reqViewChange <- nil // reqViewChange <- nil
}() // }()
//
go func() { // go func() {
count := 0 // count := 0
for _, req := range rep.requests["ack"] { // for _, req := range rep.requests["ack"] {
v := req.GetAck().View // v := req.GetAck().View
r := req.GetAck().Replica // r := req.GetAck().Replica
vc := req.GetAck().Viewchanger // vc := req.GetAck().Viewchanger
d := req.GetAck().Digest // d := req.GetAck().Digest
if v != view || vc != viewchanger || !EQ(d, digest) { // if v != view || vc != viewchanger || !EQ(d, digest) {
continue // continue
} // }
if r == replica { // if r == replica {
plog.Info("Replica %d sent multiple ack requests\n", replica) // plog.Info("Replica %d sent multiple ack requests\n", replica)
continue // continue
} // }
count++ // count++
if rep.twoThirds(count) { // if rep.twoThirds(count) {
twoThirds <- true // twoThirds <- true
return // return
} // }
} // }
twoThirds <- false // twoThirds <- false
}() // }()
//
req := <-reqViewChange // req := <-reqViewChange
//
if req == nil || !<-twoThirds { // if req == nil || !<-twoThirds {
return // return
} // }
//
rep.logPendingVC(req) // rep.logPendingVC(req)
//
// When to send new view? // // When to send new view?
rep.requestNewView(view) // rep.requestNewView(view)
} //}
func (rep *Replica) handleRequestNewView(REQ *pb.Request) { //func (rep *Replica) handleRequestNewView(REQ *pb.Request) {
//
view := REQ.GetNewview().View // view := REQ.GetNewview().View
//
if view == 0 || view < rep.view { // if view == 0 || view < rep.view {
return // return
} // }
//
replica := REQ.GetNewview().Replica // replica := REQ.GetNewview().Replica
primary := rep.newPrimary(view) // primary := rep.newPrimary(view)
//
if replica != primary { // if replica != primary {
return // return
} // }
//
if rep.hasRequest(REQ) { // if rep.hasRequest(REQ) {
return // return
} // }
//
rep.logRequest(REQ) // rep.logRequest(REQ)
//
rep.processNewView(REQ) // rep.processNewView(REQ)
} //}
func (rep *Replica) correctViewChanges(viewChanges []*pb.ViewChange) (requests []*pb.Request) { //func (rep *Replica) correctViewChanges(viewChanges []*pb.ViewChange) (requests []*pb.Request) {
//
// Returns requests if correct, else returns nil // // Returns requests if correct, else returns nil
//
valid := false // valid := false
for _, vc := range viewChanges { // for _, vc := range viewChanges {
for _, req := range rep.requests["view-change"] { // for _, req := range rep.requests["view-change"] {
d := ReqDigest(req) // d := ReqDigest(req)
if !EQ(d, vc.Digest) { // if !EQ(d, vc.Digest) {
continue // continue
} // }
requests = append(requests, req) // requests = append(requests, req)
v := req.GetViewchange().View // v := req.GetViewchange().View
// VIEW or rep.view?? // // VIEW or rep.view??
if v == rep.view { // if v == rep.view {
valid = true // valid = true
break // break
} // }
} // }
if !valid { // if !valid {
return nil // return nil
} // }
} // }
//
if rep.isPrimary(rep.ID) { // if rep.isPrimary(rep.ID) {
reps := make(map[uint32]int) // reps := make(map[uint32]int)
valid = false // valid = false
for _, req := range rep.requests["ack"] { // for _, req := range rep.requests["ack"] {
reqAck := req.GetAck() // reqAck := req.GetAck()
reps[reqAck.Replica]++ // reps[reqAck.Replica]++
if rep.twoThirds(reps[reqAck.Replica]) { //-2 // if rep.twoThirds(reps[reqAck.Replica]) { //-2
valid = true // valid = true
break // break
} // }
} // }
if !valid { // if !valid {
return nil // return nil
} // }
} // }
//
return // return
} //}
func (rep *Replica) correctSummaries(requests []*pb.Request, summaries []*pb.Summary) (correct bool) { //func (rep *Replica) correctSummaries(requests []*pb.Request, summaries []*pb.Summary) (correct bool) {
//
// Verify SUMMARIES // // Verify SUMMARIES
//
var start uint32 // var start uint32
var digest []byte // var digest []byte
digests := make(map[uint32][]byte) // digests := make(map[uint32][]byte)
//
for _, summary := range summaries { // for _, summary := range summaries {
s := summary.Sequence // s := summary.Sequence
d := summary.Digest // d := summary.Digest
if _d, ok := digests[s]; ok && !EQ(_d, d) { // if _d, ok := digests[s]; ok && !EQ(_d, d) {
return // return
} else if !ok { // } else if !ok {
digests[s] = d // digests[s] = d
} // }
if s < start || start == uint32(0) { // if s < start || start == uint32(0) {
start = s // start = s
digest = d // digest = d
} // }
} // }
//
var A1 []*pb.Request // var A1 []*pb.Request
var A2 []*pb.Request // var A2 []*pb.Request
//
valid := false // valid := false
for _, req := range requests { // for _, req := range requests {
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
s := reqViewChange.Sequence // s := reqViewChange.Sequence
if s <= start { // if s <= start {
A1 = append(A1, req) // A1 = append(A1, req)
} // }
checkpoints := reqViewChange.GetCheckpoints() // checkpoints := reqViewChange.GetCheckpoints()
for _, checkpoint := range checkpoints { // for _, checkpoint := range checkpoints {
if checkpoint.Sequence == start && EQ(checkpoint.Digest, digest) { // if checkpoint.Sequence == start && EQ(checkpoint.Digest, digest) {
A2 = append(A2, req) // A2 = append(A2, req)
break // break
} // }
} // }
if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) { // if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
valid = true // valid = true
break // break
} // }
} // }
//
if !valid { // if !valid {
return // return
} // }
//
end := start + CheckPointPeriod*ConstantFactor // end := start + CheckPointPeriod*ConstantFactor
//
for seq := start; seq <= end; seq++ { // for seq := start; seq <= end; seq++ {
//
valid = false // valid = false
//
for _, summary := range summaries { // for _, summary := range summaries {
//
if summary.Sequence != seq { // if summary.Sequence != seq {
continue // continue
} // }
//
if summary.Digest != nil { // if summary.Digest != nil {
//
var view uint32 // var view uint32
//
for _, req := range requests { // for _, req := range requests {
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
preps := reqViewChange.GetPreps() // preps := reqViewChange.GetPreps()
for _, prep := range preps { // for _, prep := range preps {
s := prep.Sequence // s := prep.Sequence
d := prep.Digest // d := prep.Digest
if s != summary.Sequence || !EQ(d, summary.Digest) { // if s != summary.Sequence || !EQ(d, summary.Digest) {
continue // continue
} // }
v := prep.View // v := prep.View
if v > view { // if v > view {
view = v // view = v
} // }
} // }
} // }
//
verifiedA1 := make(chan bool, 1) // verifiedA1 := make(chan bool, 1)
//
// Verify A1 // // Verify A1
go func() { // go func() {
//
var A1 []*pb.Request // var A1 []*pb.Request
//
FOR_LOOP: // FOR_LOOP:
for _, req := range requests { // for _, req := range requests {
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
s := reqViewChange.Sequence // s := reqViewChange.Sequence
if s >= summary.Sequence { // if s >= summary.Sequence {
continue // continue
} // }
preps := reqViewChange.GetPreps() // preps := reqViewChange.GetPreps()
for _, prep := range preps { // for _, prep := range preps {
s = prep.Sequence // s = prep.Sequence
if s != summary.Sequence { // if s != summary.Sequence {
continue // continue
} // }
d := prep.Digest // d := prep.Digest
v := prep.View // v := prep.View
if v > view || (v == view && !EQ(d, summary.Digest)) { // if v > view || (v == view && !EQ(d, summary.Digest)) {
continue FOR_LOOP // continue FOR_LOOP
} // }
} // }
A1 = append(A1, req) // A1 = append(A1, req)
if rep.twoThirds(len(A1)) { // if rep.twoThirds(len(A1)) {
verifiedA1 <- true // verifiedA1 <- true
return // return
} // }
} // }
verifiedA1 <- false // verifiedA1 <- false
}() // }()
//
verifiedA2 := make(chan bool, 1) // verifiedA2 := make(chan bool, 1)
//
// Verify A2 // // Verify A2
go func() { // go func() {
//
var A2 []*pb.Request // var A2 []*pb.Request
//
for _, req := range requests { // for _, req := range requests {
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
prePreps := reqViewChange.GetPrepreps() // prePreps := reqViewChange.GetPrepreps()
for _, prePrep := range prePreps { // for _, prePrep := range prePreps {
s := prePrep.Sequence // s := prePrep.Sequence
d := prePrep.Digest // d := prePrep.Digest
v := prePrep.View // v := prePrep.View
if s == summary.Sequence && EQ(d, summary.Digest) && v >= view { // if s == summary.Sequence && EQ(d, summary.Digest) && v >= view {
A2 = append(A2, req) // A2 = append(A2, req)
break // break
} // }
} // }
if rep.oneThird(len(A2)) { // if rep.oneThird(len(A2)) {
verifiedA2 <- true // verifiedA2 <- true
return // return
} // }
} // }
verifiedA2 <- false // verifiedA2 <- false
}() // }()
//
if !<-verifiedA1 || !<-verifiedA2 { // if !<-verifiedA1 || !<-verifiedA2 {
continue // continue
} // }
//
valid = true // valid = true
break // break
//
} else { // } else {
//
var A1 []*pb.Request // var A1 []*pb.Request
//
FOR_LOOP: // FOR_LOOP:
//
for _, req := range requests { // for _, req := range requests {
//
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
//
s := reqViewChange.Sequence // s := reqViewChange.Sequence
//
if s >= summary.Sequence { // if s >= summary.Sequence {
continue // continue
} // }
//
preps := reqViewChange.GetPreps() // preps := reqViewChange.GetPreps()
for _, prep := range preps { // for _, prep := range preps {
if prep.Sequence == summary.Sequence { // if prep.Sequence == summary.Sequence {
continue FOR_LOOP // continue FOR_LOOP
} // }
} // }
//
A1 = append(A1, req) // A1 = append(A1, req)
if rep.twoThirds(len(A1)) { // if rep.twoThirds(len(A1)) {
valid = true // valid = true
break // break
} // }
} // }
if valid { // if valid {
break // break
} // }
} // }
} // }
if !valid { // if !valid {
return // return
} // }
} // }
//
return true // return true
} //}
func (rep *Replica) processNewView(REQ *pb.Request) (success bool) { //func (rep *Replica) processNewView(REQ *pb.Request) (success bool) {
//
if rep.activeView { // if rep.activeView {
return // return
} // }
//
reqNewView := REQ.GetNewview() // reqNewView := REQ.GetNewview()
//
viewChanges := reqNewView.GetViewchanges() // viewChanges := reqNewView.GetViewchanges()
requests := rep.correctViewChanges(viewChanges) // requests := rep.correctViewChanges(viewChanges)
//
if requests == nil { // if requests == nil {
return // return
} // }
//
summaries := reqNewView.GetSummaries() // summaries := reqNewView.GetSummaries()
correct := rep.correctSummaries(requests, summaries) // correct := rep.correctSummaries(requests, summaries)
//
if !correct { // if !correct {
return // return
} // }
//
var h uint32 // var h uint32
for _, checkpoint := range rep.checkpoints { // for _, checkpoint := range rep.checkpoints {
if checkpoint.Sequence < h || h == uint32(0) { // if checkpoint.Sequence < h || h == uint32(0) {
h = checkpoint.Sequence // h = checkpoint.Sequence
} // }
} // }
//
var s uint32 // var s uint32
for _, summary := range summaries { // for _, summary := range summaries {
if summary.Sequence < s || s == uint32(0) { // if summary.Sequence < s || s == uint32(0) {
s = summary.Sequence // s = summary.Sequence
} // }
if summary.Sequence > h { // if summary.Sequence > h {
valid := false // valid := false
for _, req := range rep.requests["view-change"] { //in // for _, req := range rep.requests["view-change"] { //in
if EQ(ReqDigest(req), summary.Digest) { // if EQ(ReqDigest(req), summary.Digest) {
valid = true // valid = true
break // break
} // }
} // }
if !valid { // if !valid {
return // return
} // }
} // }
} // }
//
if h < s { // if h < s {
return // return
} // }
//
// Process new view // // Process new view
rep.activeView = true // rep.activeView = true
//
for _, summary := range summaries { // for _, summary := range summaries {
//
if rep.ID != reqNewView.Replica { // if rep.ID != reqNewView.Replica {
req := ToRequestPrepare( // req := ToRequestPrepare(
reqNewView.View, // reqNewView.View,
summary.Sequence, // summary.Sequence,
summary.Digest, // summary.Digest,
rep.ID) // the backup sends/logs prepare // rep.ID) // the backup sends/logs prepare
//
go func() { // go func() {
if !rep.hasRequest(req) { // if !rep.hasRequest(req) {
rep.requestChan <- req // rep.requestChan <- req
} // }
}() // }()
if summary.Sequence <= h { // if summary.Sequence <= h {
continue // continue
} // }
//
if !rep.hasRequest(req) { // if !rep.hasRequest(req) {
rep.logRequest(req) // rep.logRequest(req)
} // }
} else { // } else {
if summary.Sequence <= h { // if summary.Sequence <= h {
break // break
} // }
} // }
//
req := ToRequestPreprepare( // req := ToRequestPreprepare(
reqNewView.View, // reqNewView.View,
summary.Sequence, // summary.Sequence,
summary.Digest, // summary.Digest,
reqNewView.Replica) // new primary pre-prepares // reqNewView.Replica) // new primary pre-prepares
//
if !rep.hasRequest(req) { // if !rep.hasRequest(req) {
rep.logRequest(req) // rep.logRequest(req)
} // }
} // }
//
var maxSequence uint32 // var maxSequence uint32
for _, req := range rep.requests["pre-prepare"] { // for _, req := range rep.requests["pre-prepare"] {
reqPrePrepare := req.GetPreprepare() // reqPrePrepare := req.GetPreprepare()
if reqPrePrepare.Sequence > maxSequence { // if reqPrePrepare.Sequence > maxSequence {
maxSequence = reqPrePrepare.Sequence // maxSequence = reqPrePrepare.Sequence
} // }
} // }
rep.sequence = maxSequence // rep.sequence = maxSequence
return true // return true
} //}
func (rep *Replica) prePrepBySequence(sequence uint32) []*pb.Entry { //func (rep *Replica) prePrepBySequence(sequence uint32) []*pb.Entry {
var view uint32 // var view uint32
var requests []*pb.Request // var requests []*pb.Request
for _, req := range rep.requests["pre-prepare"] { // for _, req := range rep.requests["pre-prepare"] {
v := req.GetPreprepare().View // v := req.GetPreprepare().View
s := req.GetPreprepare().Sequence // s := req.GetPreprepare().Sequence
if v >= view && s == sequence { // if v >= view && s == sequence {
view = v // view = v
requests = append(requests, req) // requests = append(requests, req)
} // }
} // }
if requests == nil { // if requests == nil {
return nil // return nil
} // }
var prePreps []*pb.Entry // var prePreps []*pb.Entry
for _, req := range requests { // for _, req := range requests {
v := req.GetPreprepare().View // v := req.GetPreprepare().View
if v == view { // if v == view {
s := req.GetPreprepare().Sequence // s := req.GetPreprepare().Sequence
d := req.GetPreprepare().Digest // d := req.GetPreprepare().Digest
prePrep := ToEntry(s, d, v) // prePrep := ToEntry(s, d, v)
prePreps = append(prePreps, prePrep) // prePreps = append(prePreps, prePrep)
} // }
} // }
FOR_LOOP: //FOR_LOOP:
for _, prePrep := range prePreps { // for _, prePrep := range prePreps {
for _, req := range rep.allRequests() { //TODO: optimize // for _, req := range rep.allRequests() { //TODO: optimize
if EQ(ReqDigest(req), prePrep.Digest) { // if EQ(ReqDigest(req), prePrep.Digest) {
continue FOR_LOOP // continue FOR_LOOP
} // }
} // }
return nil // return nil
} // }
return prePreps // return prePreps
} //}
//
func (rep *Replica) prepBySequence(sequence uint32) ([]*pb.Entry, []*pb.Entry) { //func (rep *Replica) prepBySequence(sequence uint32) ([]*pb.Entry, []*pb.Entry) {
//
prePreps := rep.prePrepBySequence(sequence) // prePreps := rep.prePrepBySequence(sequence)
//
if prePreps == nil { // if prePreps == nil {
return nil, nil // return nil, nil
} // }
//
var preps []*pb.Entry // var preps []*pb.Entry
//
FOR_LOOP: //FOR_LOOP:
for _, prePrep := range prePreps { // for _, prePrep := range prePreps {
//
view := prePrep.View // view := prePrep.View
digest := prePrep.Digest // digest := prePrep.Digest
//
replicas := make(map[uint32]int) // replicas := make(map[uint32]int)
for _, req := range rep.requests["prepare"] { // for _, req := range rep.requests["prepare"] {
reqPrepare := req.GetPrepare() // reqPrepare := req.GetPrepare()
v := reqPrepare.View // v := reqPrepare.View
s := reqPrepare.Sequence // s := reqPrepare.Sequence
d := reqPrepare.Digest // d := reqPrepare.Digest
if v == view && s == sequence && EQ(d, digest) { // if v == view && s == sequence && EQ(d, digest) {
r := reqPrepare.Replica // r := reqPrepare.Replica
replicas[r]++ // replicas[r]++
if rep.twoThirds(replicas[r]) { // if rep.twoThirds(replicas[r]) {
prep := ToEntry(s, d, v) // prep := ToEntry(s, d, v)
preps = append(preps, prep) // preps = append(preps, prep)
continue FOR_LOOP // continue FOR_LOOP
} // }
} // }
} // }
return prePreps, nil // return prePreps, nil
} // }
//
return prePreps, preps // return prePreps, preps
} //}
/* /*
func (rep *Replica) prepBySequence(sequence uint32) *pb.Entry { func (rep *Replica) prepBySequence(sequence uint32) *pb.Entry {
...@@ -1420,187 +1420,187 @@ func (rep *Replica) prePrepBySequence(sequence uint32) *pb.Entry { ...@@ -1420,187 +1420,187 @@ func (rep *Replica) prePrepBySequence(sequence uint32) *pb.Entry {
} }
*/ */
func (rep *Replica) requestViewChange(view uint32) { //func (rep *Replica) requestViewChange(view uint32) {
//
if view != rep.view+1 { // if view != rep.view+1 {
return // return
} // }
rep.view = view // rep.view = view
rep.activeView = false // rep.activeView = false
//
var prePreps []*pb.Entry // var prePreps []*pb.Entry
var preps []*pb.Entry // var preps []*pb.Entry
//
start := rep.lowWaterMark() + 1 // start := rep.lowWaterMark() + 1
end := rep.highWaterMark() // end := rep.highWaterMark()
//
for s := start; s <= end; s++ { // for s := start; s <= end; s++ {
_prePreps, _preps := rep.prepBySequence(s) // _prePreps, _preps := rep.prepBySequence(s)
if _prePreps != nil { // if _prePreps != nil {
prePreps = append(prePreps, _prePreps...) // prePreps = append(prePreps, _prePreps...)
} // }
if _preps != nil { // if _preps != nil {
preps = append(preps, _preps...) // preps = append(preps, _preps...)
} // }
} // }
//
sequence := rep.lowWaterMark() // sequence := rep.lowWaterMark()
//
req := ToRequestViewChange( // req := ToRequestViewChange(
view, // view,
sequence, // sequence,
rep.checkpoints, // // rep.checkpoints, //
preps, // preps,
prePreps, // prePreps,
rep.ID) // rep.ID)
//
rep.logRequest(req) // rep.logRequest(req)
//
go func() { // go func() {
rep.requestChan <- req // rep.requestChan <- req
}() // }()
//
rep.clearRequestsByView(view) // rep.clearRequestsByView(view)
} //}
//
func (rep *Replica) createNewView(view uint32) (request *pb.Request) { //func (rep *Replica) createNewView(view uint32) (request *pb.Request) {
//
// Returns RequestNewView if successful, else returns nil // // Returns RequestNewView if successful, else returns nil
// create viewChanges // // create viewChanges
viewChanges := make([]*pb.ViewChange, len(rep.pendingVC)) // viewChanges := make([]*pb.ViewChange, len(rep.pendingVC))
//
for idx := range viewChanges { // for idx := range viewChanges {
req := rep.pendingVC[idx] // req := rep.pendingVC[idx]
viewchanger := req.GetViewchange().Replica // viewchanger := req.GetViewchange().Replica
vc := ToViewChange(viewchanger, ReqDigest(req)) // vc := ToViewChange(viewchanger, ReqDigest(req))
viewChanges[idx] = vc // viewChanges[idx] = vc
} // }
//
var summaries []*pb.Summary // var summaries []*pb.Summary
var summary *pb.Summary // var summary *pb.Summary
//
start := rep.lowWaterMark() + 1 // start := rep.lowWaterMark() + 1
end := rep.highWaterMark() // end := rep.highWaterMark()
//
// select starting checkpoint // // select starting checkpoint
FOR_LOOP_1: //FOR_LOOP_1:
for seq := start; seq <= end; seq++ { // for seq := start; seq <= end; seq++ {
//
overLWM := 0 // overLWM := 0
var digest []byte // var digest []byte
digests := make(map[string]int) // digests := make(map[string]int)
//
for _, req := range rep.pendingVC { // for _, req := range rep.pendingVC {
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
if reqViewChange.Sequence <= seq { // if reqViewChange.Sequence <= seq {
overLWM++ // overLWM++
} // }
for _, checkpoint := range reqViewChange.GetCheckpoints() { // for _, checkpoint := range reqViewChange.GetCheckpoints() {
if checkpoint.Sequence == seq { // if checkpoint.Sequence == seq {
d := checkpoint.Digest // d := checkpoint.Digest
digests[string(d)]++ // digests[string(d)]++
if rep.oneThird(digests[string(d)]) { // if rep.oneThird(digests[string(d)]) {
digest = d // digest = d
break // break
} // }
} // }
} // }
if rep.twoThirds(overLWM) && rep.oneThird(digests[string(digest)]) { // if rep.twoThirds(overLWM) && rep.oneThird(digests[string(digest)]) {
summary = ToSummary(seq, digest) // summary = ToSummary(seq, digest)
continue FOR_LOOP_1 // continue FOR_LOOP_1
} // }
} // }
} // }
//
if summary == nil { // if summary == nil {
return // return
} // }
//
summaries = append(summaries, summary) // summaries = append(summaries, summary)
//
start = summary.Sequence // start = summary.Sequence
end = start + CheckPointPeriod*ConstantFactor // end = start + CheckPointPeriod*ConstantFactor
//
// select summaries // // select summaries
// TODO: optimize // // TODO: optimize
FOR_LOOP_2: //FOR_LOOP_2:
for seq := start; seq <= end; seq++ { // for seq := start; seq <= end; seq++ {
//
for _, REQ := range rep.pendingVC { // for _, REQ := range rep.pendingVC {
//
sequence := REQ.GetViewchange().Sequence // sequence := REQ.GetViewchange().Sequence
//
if sequence != seq { // if sequence != seq {
continue // continue
} // }
//
var A1 []*pb.Request // var A1 []*pb.Request
var A2 []*pb.Request // var A2 []*pb.Request
//
view := REQ.GetViewchange().View // view := REQ.GetViewchange().View
digest := ReqDigest(REQ) // digest := ReqDigest(REQ)
//
FOR_LOOP_3: // FOR_LOOP_3:
for _, req := range rep.pendingVC { // for _, req := range rep.pendingVC {
//
reqViewChange := req.GetViewchange() // reqViewChange := req.GetViewchange()
//
if reqViewChange.Sequence < sequence { // if reqViewChange.Sequence < sequence {
preps := reqViewChange.GetPreps() // preps := reqViewChange.GetPreps()
for _, prep := range preps { // for _, prep := range preps {
if prep.Sequence != sequence { // if prep.Sequence != sequence {
continue // continue
} // }
if prep.View > view || (prep.View == view && !EQ(prep.Digest, digest)) { // if prep.View > view || (prep.View == view && !EQ(prep.Digest, digest)) {
continue FOR_LOOP_3 // continue FOR_LOOP_3
} // }
} // }
A1 = append(A1, req) // A1 = append(A1, req)
} // }
prePreps := reqViewChange.GetPrepreps() // prePreps := reqViewChange.GetPrepreps()
for _, prePrep := range prePreps { // for _, prePrep := range prePreps {
if prePrep.Sequence != sequence { // if prePrep.Sequence != sequence {
continue // continue
} // }
if prePrep.View >= view && EQ(prePrep.Digest, digest) { // if prePrep.View >= view && EQ(prePrep.Digest, digest) {
A2 = append(A2, req) // A2 = append(A2, req)
continue FOR_LOOP_3 // continue FOR_LOOP_3
} // }
} // }
} // }
//
if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) { // if rep.twoThirds(len(A1)) && rep.oneThird(len(A2)) {
summary = ToSummary(sequence, digest) // summary = ToSummary(sequence, digest)
summaries = append(summaries, summary) // summaries = append(summaries, summary)
continue FOR_LOOP_2 // continue FOR_LOOP_2
} // }
} // }
} // }
//
request = ToRequestNewView(view, viewChanges, summaries, rep.ID) // request = ToRequestNewView(view, viewChanges, summaries, rep.ID)
return // return
} //}
//
func (rep *Replica) requestNewView(view uint32) { //func (rep *Replica) requestNewView(view uint32) {
//
req := rep.createNewView(view) // req := rep.createNewView(view)
//
if req == nil || rep.hasRequest(req) { // if req == nil || rep.hasRequest(req) {
return // return
} // }
//
// Process new view // // Process new view
//
success := rep.processNewView(req) // success := rep.processNewView(req)
//
if !success { // if !success {
return // return
} // }
//
rep.logRequest(req) // rep.logRequest(req)
//
go func() { // go func() {
rep.requestChan <- req // rep.requestChan <- req
}() // }()
//
} //}
plugin/consensus/pbft/pbft_test.go
View file @ 6bf4628e
...@@ -150,3 +150,39 @@ func clearTestData() { ...@@ -150,3 +150,39 @@ func clearTestData() {
} }
fmt.Println("test data clear successfully!") fmt.Println("test data clear successfully!")
} }
func TestReplica(t *testing.T){
reply1, request1, isPrimary1:=NewReplica(1,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8891")
reply2, request2, isPrimary2:=NewReplica(2,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8892")
reply3, request3, isPrimary3:=NewReplica(3,"127.0.0.1:8891,127.0.0.1:8892,127.0.0.1:8893","127.0.0.1:8893")
go func() {
for {
select {
case <-reply1:
t.Log("I'm have reply1 message")
case <-reply2:
t.Log("I'm have reply2 message")
case <-reply3:
t.Log("I'm have reply3 message")
}
}
}()
t.Log(isPrimary1)
t.Log(isPrimary2)
t.Log(isPrimary3)
go func() {
for i:=0;i<100;i++ {
op := &types.Operation{Value: &types.Block{}}
select {
case request1<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8891"):
t.Log("I'm have send request1 message")
case request2<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8892"):
t.Log("I'm have send request2 message")
case request3<-ToRequestClient(op, types.Now().String(), "127.0.0.1:8893"):
t.Log("I'm have send request3 message")
}
}
}()
time.Sleep(5*time.Second)
}
\ No newline at end of file
plugin/consensus/raft/block.go
View file @ 6bf4628e
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
package raft package raft
import ( import (
"context"
"fmt" "fmt"
"sync" "sync"
"time" "time"
...@@ -35,14 +36,15 @@ type Client struct { ...@@ -35,14 +36,15 @@ type Client struct {
errorC <-chan error errorC <-chan error
snapshotter *snap.Snapshotter snapshotter *snap.Snapshotter
validatorC <-chan bool validatorC <-chan bool
stopC chan<- struct{} ctx context.Context
cancel context.CancelFunc
once sync.Once once sync.Once
} }
// NewBlockstore create Raft Client // NewBlockstore create Raft Client
func NewBlockstore(cfg *types.Consensus, snapshotter *snap.Snapshotter, proposeC chan<- *types.Block, commitC <-chan *types.Block, errorC <-chan error, validatorC <-chan bool, stopC chan<- struct{}) *Client { func NewBlockstore(ctx context.Context, cfg *types.Consensus, snapshotter *snap.Snapshotter, proposeC chan<- *types.Block, commitC <-chan *types.Block, errorC <-chan error, validatorC <-chan bool, cancel context.CancelFunc) *Client {
c := drivers.NewBaseClient(cfg) c := drivers.NewBaseClient(cfg)
client := &Client{BaseClient: c, proposeC: proposeC, snapshotter: snapshotter, validatorC: validatorC, commitC: commitC, errorC: errorC, stopC: stopC} client := &Client{BaseClient: c, proposeC: proposeC, snapshotter: snapshotter, validatorC: validatorC, commitC: commitC, errorC: errorC, ctx: ctx, cancel: cancel}
c.SetChild(client) c.SetChild(client)
return client return client
} }
...@@ -97,12 +99,12 @@ func (client *Client) SetQueueClient(c queue.Client) { ...@@ -97,12 +99,12 @@ func (client *Client) SetQueueClient(c queue.Client) {
}) })
go client.EventLoop() go client.EventLoop()
go client.readCommits(client.commitC, client.errorC) go client.readCommits(client.commitC, client.errorC)
go client.pollingTask(c) go client.pollingTask()
} }
// Close method // Close method
func (client *Client) Close() { func (client *Client) Close() {
client.stopC <- struct{}{} client.cancel()
rlog.Info("consensus raft closed") rlog.Info("consensus raft closed")
} }
...@@ -125,10 +127,14 @@ func (client *Client) CreateBlock() { ...@@ -125,10 +127,14 @@ func (client *Client) CreateBlock() {
panic("This node encounter problem, exit.") panic("This node encounter problem, exit.")
} }
} }
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for { for {
select {
case <-client.ctx.Done():
case <-ticker.C:
//如果leader节点突然挂了，不是打包节点，需要退出 //如果leader节点突然挂了，不是打包节点，需要退出
if !isLeader { if !mux.Load().(bool) {
rlog.Warn("I'm not the validator node anymore, exit.=============================") rlog.Warn("I'm not the validator node anymore, exit.=============================")
break break
} }
...@@ -193,6 +199,8 @@ func (client *Client) CreateBlock() { ...@@ -193,6 +199,8 @@ func (client *Client) CreateBlock() {
} }
time.Sleep(time.Second * time.Duration(writeBlockSeconds)) time.Sleep(time.Second * time.Duration(writeBlockSeconds))
} }
}
} }
// 向raft底层发送block // 向raft底层发送block
...@@ -219,17 +227,21 @@ func (client *Client) readCommits(commitC <-chan *types.Block, errorC <-chan err ...@@ -219,17 +227,21 @@ func (client *Client) readCommits(commitC <-chan *types.Block, errorC <-chan err
if ok { if ok {
panic(err) panic(err)
} }
case <-client.ctx.Done():
return
} }
} }
} }
//轮询任务，去检测本机器是否为validator节点，如果是，则执行打包任务 //轮询任务，去检测本机器是否为validator节点，如果是，则执行打包任务
func (client *Client) pollingTask(c queue.Client) { func (client *Client) pollingTask() {
ticker := time.NewTicker(100 * time.Millisecond) ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop() defer ticker.Stop()
for { for {
select { select {
case <-client.ctx.Done():
return
case value, ok := <-client.validatorC: case value, ok := <-client.validatorC:
//各个节点Block只初始化一次 //各个节点Block只初始化一次
client.once.Do(func() { client.once.Do(func() {
...@@ -237,9 +249,14 @@ func (client *Client) pollingTask(c queue.Client) { ...@@ -237,9 +249,14 @@ func (client *Client) pollingTask(c queue.Client) {
}) })
if ok && !value { if ok && !value {
rlog.Debug("================I'm not the validator node!=============") rlog.Debug("================I'm not the validator node!=============")
leader := mux.Load().(bool)
if leader {
isLeader = false isLeader = false
} else if ok && !isLeader && value { mux.Store(isLeader)
}
} else if ok && !mux.Load().(bool) && value {
isLeader = true isLeader = true
mux.Store(isLeader)
go client.CreateBlock() go client.CreateBlock()
} else if !ok { } else if !ok {
break break
......
plugin/consensus/raft/chain33.test.toml
View file @ 6bf4628e
...@@ -36,8 +36,8 @@ enableTxQuickIndex=true ...@@ -36,8 +36,8 @@ enableTxQuickIndex=true
[p2p] [p2p]
seeds=["127.0.0.1:13802"] seeds=["127.0.0.1:13802"]
enable=true enable=false
isSeed=true isSeed=false
serverStart=true serverStart=true
innerSeedEnable=false innerSeedEnable=false
useGithub=false useGithub=false
...@@ -107,8 +107,8 @@ peersURL="http://127.0.0.1:9021" ...@@ -107,8 +107,8 @@ peersURL="http://127.0.0.1:9021"
# raft共识用到，指示raft集群中只读节点的IP（只同步日志，不参与raft共识） # raft共识用到，指示raft集群中只读节点的IP（只同步日志，不参与raft共识）
readOnlyPeersURL="" readOnlyPeersURL=""
addPeersURL="" addPeersURL=""
#raft共识用到，默认raft中多少条记录打包一个snapshot #raft共识用到，默认raft中多少条记录打包一个snapshot(这里为了测试调整小一点）
defaultSnapCount=1000 defaultSnapCount=2
#raft共识用到，默认raft中写区块时间间隔 #raft共识用到，默认raft中写区块时间间隔
writeBlockSeconds=1 writeBlockSeconds=1
#raft共识用到，默认raft中leader发送心跳包时间间隔 #raft共识用到，默认raft中leader发送心跳包时间间隔
......
plugin/consensus/raft/controller.go
View file @ 6bf4628e
...@@ -5,7 +5,9 @@ ...@@ -5,7 +5,9 @@
package raft package raft
import ( import (
"context"
"strings" "strings"
"sync/atomic"
log "github.com/33cn/chain33/common/log/log15" log "github.com/33cn/chain33/common/log/log15"
"github.com/33cn/chain33/queue" "github.com/33cn/chain33/queue"
...@@ -22,6 +24,7 @@ var ( ...@@ -22,6 +24,7 @@ var (
writeBlockSeconds int64 = 1 writeBlockSeconds int64 = 1
heartbeatTick = 1 heartbeatTick = 1
isLeader = false isLeader = false
mux atomic.Value
confChangeC chan raftpb.ConfChange confChangeC chan raftpb.ConfChange
) )
...@@ -39,6 +42,10 @@ type subConfig struct { ...@@ -39,6 +42,10 @@ type subConfig struct {
HeartbeatTick int32 `json:"heartbeatTick"` HeartbeatTick int32 `json:"heartbeatTick"`
} }
func init() {
mux.Store(isLeader)
}
// NewRaftCluster create raft cluster // NewRaftCluster create raft cluster
func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module { func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
rlog.Info("Start to create raft cluster") rlog.Info("Start to create raft cluster")
...@@ -70,10 +77,6 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module { ...@@ -70,10 +77,6 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
if subcfg.HeartbeatTick > 0 { if subcfg.HeartbeatTick > 0 {
heartbeatTick = int(subcfg.HeartbeatTick) heartbeatTick = int(subcfg.HeartbeatTick)
} }
// propose channel
proposeC := make(chan *types.Block)
confChangeC = make(chan raftpb.ConfChange)
var b *Client var b *Client
getSnapshot := func() ([]byte, error) { return b.getSnapshot() } getSnapshot := func() ([]byte, error) { return b.getSnapshot() }
// raft集群的建立,1. 初始化两条channel： propose channel用于客户端和raft底层交互, commit channel用于获取commit消息 // raft集群的建立,1. 初始化两条channel： propose channel用于客户端和raft底层交互, commit channel用于获取commit消息
...@@ -90,10 +93,15 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module { ...@@ -90,10 +93,15 @@ func NewRaftCluster(cfg *types.Consensus, sub []byte) queue.Module {
if len(addPeers) == 1 && addPeers[0] == "" { if len(addPeers) == 1 && addPeers[0] == "" {
addPeers = []string{} addPeers = []string{}
} }
commitC, errorC, snapshotterReady, validatorC, stopC := NewRaftNode(int(subcfg.NodeID), subcfg.IsNewJoinNode, peers, readOnlyPeers, addPeers, getSnapshot, proposeC, confChangeC) //采用context来统一管理所有服务
ctx, stop := context.WithCancel(context.Background())
// propose channel
proposeC := make(chan *types.Block)
confChangeC = make(chan raftpb.ConfChange)
commitC, errorC, snapshotterReady, validatorC := NewRaftNode(ctx, int(subcfg.NodeID), subcfg.IsNewJoinNode, peers, readOnlyPeers, addPeers, getSnapshot, proposeC, confChangeC)
//启动raft删除节点操作监听 //启动raft删除节点操作监听
go serveHTTPRaftAPI(int(subcfg.RaftAPIPort), confChangeC, errorC) go serveHTTPRaftAPI(ctx, int(subcfg.RaftAPIPort), confChangeC, errorC)
// 监听commit channel,取block // 监听commit channel,取block
b = NewBlockstore(cfg, <-snapshotterReady, proposeC, commitC, errorC, validatorC, stopC) b = NewBlockstore(ctx, cfg, <-snapshotterReady, proposeC, commitC, errorC, validatorC, stop)
return b return b
} }
plugin/consensus/raft/httpapi.go
View file @ 6bf4628e
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
package raft package raft
import ( import (
"context"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"strconv" "strconv"
...@@ -66,8 +67,8 @@ func (h *httpRaftAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -66,8 +67,8 @@ func (h *httpRaftAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
} }
func serveHTTPRaftAPI(port int, confChangeC chan<- raftpb.ConfChange, errorC <-chan error) { func serveHTTPRaftAPI(ctx context.Context, port int, confChangeC chan<- raftpb.ConfChange, errorC <-chan error) {
srv := http.Server{ srv := &http.Server{
Addr: "localhost:" + strconv.Itoa(port), Addr: "localhost:" + strconv.Itoa(port),
Handler: &httpRaftAPI{ Handler: &httpRaftAPI{
confChangeC: confChangeC, confChangeC: confChangeC,
...@@ -78,9 +79,11 @@ func serveHTTPRaftAPI(port int, confChangeC chan<- raftpb.ConfChange, errorC <-c ...@@ -78,9 +79,11 @@ func serveHTTPRaftAPI(port int, confChangeC chan<- raftpb.ConfChange, errorC <-c
rlog.Error(fmt.Sprintf("ListenAndServe have a err: (%v)", err.Error())) rlog.Error(fmt.Sprintf("ListenAndServe have a err: (%v)", err.Error()))
} }
}() }()
select {
// exit when raft goes down case <-ctx.Done():
if err, ok := <-errorC; ok { srv.Close()
case err := <-errorC:
srv.Close()
rlog.Error(fmt.Sprintf("the errorC chan receive a err (%v)\n", err.Error())) rlog.Error(fmt.Sprintf("the errorC chan receive a err (%v)\n", err.Error()))
} }
} }
plugin/consensus/raft/listener.go
View file @ 6bf4628e
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
package raft package raft
import ( import (
"context"
"errors" "errors"
"net" "net"
"time" "time"
...@@ -13,16 +14,16 @@ import ( ...@@ -13,16 +14,16 @@ import (
// 设置TCP keep-alive超时，接收stopc // 设置TCP keep-alive超时，接收stopc
type stoppableListener struct { type stoppableListener struct {
*net.TCPListener *net.TCPListener
stopc <-chan struct{} ctx context.Context
} }
// 监听tcp连接 // 监听tcp连接
func newStoppableListener(addr string, stopc <-chan struct{}) (*stoppableListener, error) { func newStoppableListener(addr string, ctx context.Context) (*stoppableListener, error) {
ln, err := net.Listen("tcp", addr) ln, err := net.Listen("tcp", addr)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &stoppableListener{ln.(*net.TCPListener), stopc}, nil return &stoppableListener{ln.(*net.TCPListener), ctx}, nil
} }
func (ln stoppableListener) Accept() (c net.Conn, err error) { func (ln stoppableListener) Accept() (c net.Conn, err error) {
...@@ -37,7 +38,7 @@ func (ln stoppableListener) Accept() (c net.Conn, err error) { ...@@ -37,7 +38,7 @@ func (ln stoppableListener) Accept() (c net.Conn, err error) {
connc <- tc connc <- tc
}() }()
select { select {
case <-ln.stopc: case <-ln.ctx.Done():
return nil, errors.New("server stopped") return nil, errors.New("server stopped")
case err := <-errc: case err := <-errc:
return nil, err return nil, err
......
plugin/consensus/raft/raft.go
View file @ 6bf4628e
...@@ -56,17 +56,18 @@ type raftNode struct { ...@@ -56,17 +56,18 @@ type raftNode struct {
snapCount uint64 snapCount uint64
transport *rafthttp.Transport transport *rafthttp.Transport
stopMu sync.RWMutex stopMu sync.RWMutex
stopc chan struct{} ctx context.Context
httpstopc chan struct{} //stopc chan struct{}
httpdonec chan struct{} //httpstopc chan struct{}
//httpdonec chan struct{}
validatorC chan bool validatorC chan bool
//用于判断该节点是否重启过 //用于判断该节点是否重启过
restartC chan struct{} restartC chan struct{}
} }
// NewRaftNode create raft node // NewRaftNode create raft node
func NewRaftNode(id int, join bool, peers []string, readOnlyPeers []string, addPeers []string, getSnapshot func() ([]byte, error), proposeC <-chan *types.Block, func NewRaftNode(ctx context.Context, id int, join bool, peers []string, readOnlyPeers []string, addPeers []string, getSnapshot func() ([]byte, error), proposeC <-chan *types.Block,
confChangeC <-chan raftpb.ConfChange) (<-chan *types.Block, <-chan error, <-chan *snap.Snapshotter, <-chan bool, chan<- struct{}) { confChangeC <-chan raftpb.ConfChange) (<-chan *types.Block, <-chan error, <-chan *snap.Snapshotter, <-chan bool) {
rlog.Info("Enter consensus raft") rlog.Info("Enter consensus raft")
// commit channel // commit channel
...@@ -86,16 +87,14 @@ func NewRaftNode(id int, join bool, peers []string, readOnlyPeers []string, addP ...@@ -86,16 +87,14 @@ func NewRaftNode(id int, join bool, peers []string, readOnlyPeers []string, addP
snapdir: fmt.Sprintf("chain33_raft-%d%ssnap", id, string(os.PathSeparator)), snapdir: fmt.Sprintf("chain33_raft-%d%ssnap", id, string(os.PathSeparator)),
getSnapshot: getSnapshot, getSnapshot: getSnapshot,
snapCount: defaultSnapCount, snapCount: defaultSnapCount,
stopc: make(chan struct{}),
httpstopc: make(chan struct{}),
httpdonec: make(chan struct{}),
validatorC: make(chan bool), validatorC: make(chan bool),
snapshotterReady: make(chan *snap.Snapshotter, 1), snapshotterReady: make(chan *snap.Snapshotter, 1),
restartC: make(chan struct{}, 1), restartC: make(chan struct{}, 1),
ctx: ctx,
} }
go rc.startRaft() go rc.startRaft()
return commitC, errorC, rc.snapshotterReady, rc.validatorC, rc.stopc return commitC, errorC, rc.snapshotterReady, rc.validatorC
} }
// 启动raft节点 // 启动raft节点
...@@ -184,22 +183,22 @@ func (rc *raftNode) serveRaft() { ...@@ -184,22 +183,22 @@ func (rc *raftNode) serveRaft() {
panic(err) panic(err)
} }
ln, err := newStoppableListener(nodeURL.Host, rc.httpstopc) ln, err := newStoppableListener(nodeURL.Host, rc.ctx)
if err != nil { if err != nil {
rlog.Error(fmt.Sprintf("raft: Failed to listen rafthttp (%v)", err.Error())) rlog.Error(fmt.Sprintf("raft: Failed to listen rafthttp (%v)", err.Error()))
panic(err) panic(err)
} }
raftSrv := &http.Server{Handler: rc.transport.Handler()}
err = (&http.Server{Handler: rc.transport.Handler()}).Serve(ln) err = raftSrv.Serve(ln)
if err != nil { if err != nil {
rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error())) rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error()))
} }
select { select {
case <-rc.httpstopc: case <-rc.ctx.Done():
raftSrv.Close()
default: default:
rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error())) rlog.Error(fmt.Sprintf("raft: Failed to serve rafthttp (%v)", err.Error()))
} }
close(rc.httpdonec)
} }
func (rc *raftNode) serveChannels() { func (rc *raftNode) serveChannels() {
...@@ -246,9 +245,11 @@ func (rc *raftNode) serveChannels() { ...@@ -246,9 +245,11 @@ func (rc *raftNode) serveChannels() {
rlog.Error(fmt.Sprintf("rc.node.ProposeConfChange:%v", err.Error())) rlog.Error(fmt.Sprintf("rc.node.ProposeConfChange:%v", err.Error()))
} }
} }
case <-rc.ctx.Done():
rlog.Info("I have a exit message!")
return
} }
} }
close(rc.stopc)
}() }()
// 从Ready()中接收数据 // 从Ready()中接收数据
for { for {
...@@ -275,7 +276,7 @@ func (rc *raftNode) serveChannels() { ...@@ -275,7 +276,7 @@ func (rc *raftNode) serveChannels() {
rc.writeError(err) rc.writeError(err)
return return
case <-rc.stopc: case <-rc.ctx.Done():
rc.stop() rc.stop()
return return
} }
...@@ -283,9 +284,9 @@ func (rc *raftNode) serveChannels() { ...@@ -283,9 +284,9 @@ func (rc *raftNode) serveChannels() {
} }
func (rc *raftNode) updateValidator() { func (rc *raftNode) updateValidator() {
//TODO 这块监听后期需要根据场景进行优化? //TODO 这块监听后期需要根据场景进行优化?
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
//用于标记readOnlyPeers是否已经被添加到集群中了 //用于标记readOnlyPeers是否已经被添加到集群中了
flag := false flag := false
isRestart := false isRestart := false
...@@ -297,8 +298,12 @@ func (rc *raftNode) updateValidator() { ...@@ -297,8 +298,12 @@ func (rc *raftNode) updateValidator() {
case <-ticker.C: case <-ticker.C:
ticker.Stop() ticker.Stop()
} }
ticker = time.NewTicker(time.Second)
for { for {
time.Sleep(time.Second) select {
case <-rc.ctx.Done():
return
case <-ticker.C:
status := rc.Status() status := rc.Status()
if status.Lead == raft.None { if status.Lead == raft.None {
rlog.Debug(fmt.Sprintf("==============This is %s node!==============", status.RaftState.String())) rlog.Debug(fmt.Sprintf("==============This is %s node!==============", status.RaftState.String()))
...@@ -317,6 +322,8 @@ func (rc *raftNode) updateValidator() { ...@@ -317,6 +322,8 @@ func (rc *raftNode) updateValidator() {
flag = true flag = true
} }
} }
}
} }
func (rc *raftNode) Status() raft.Status { func (rc *raftNode) Status() raft.Status {
rc.stopMu.RLock() rc.stopMu.RLock()
...@@ -454,20 +461,18 @@ func (rc *raftNode) stop() { ...@@ -454,20 +461,18 @@ func (rc *raftNode) stop() {
rc.stopHTTP() rc.stopHTTP()
close(rc.commitC) close(rc.commitC)
close(rc.errorC) close(rc.errorC)
close(rc.stopc)
rc.node.Stop() rc.node.Stop()
} }
func (rc *raftNode) stopHTTP() { func (rc *raftNode) stopHTTP() {
rc.transport.Stop() rc.transport.Stop()
close(rc.httpstopc) //close(rc.httpstopc)
<-rc.httpdonec //<-rc.httpdonec
} }
func (rc *raftNode) writeError(err error) { func (rc *raftNode) writeError(err error) {
rc.stopHTTP() rc.stopHTTP()
close(rc.commitC) close(rc.commitC)
close(rc.stopc)
rc.errorC <- err rc.errorC <- err
close(rc.errorC) close(rc.errorC)
rc.node.Stop() rc.node.Stop()
...@@ -488,7 +493,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool { ...@@ -488,7 +493,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
} }
select { select {
case rc.commitC <- block: case rc.commitC <- block:
case <-rc.stopc: case <-rc.ctx.Done():
return false return false
} }
...@@ -521,7 +526,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool { ...@@ -521,7 +526,7 @@ func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
if ents[i].Index == rc.lastIndex { if ents[i].Index == rc.lastIndex {
select { select {
case rc.commitC <- nil: case rc.commitC <- nil:
case <-rc.stopc: case <-rc.ctx.Done():
return false return false
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment