prepareres res;
prop_t highest_n_a{0, ""};
for (auto i : nodes) {
- auto r = rpcc::bind_cached(i);
- if (!r)
+ auto cl = rpcc::bind_cached(i);
+ if (!cl)
continue;
- auto status = (paxos_protocol::status)r->call_timeout(
- paxos_protocol::preparereq, milliseconds(100), res, me, instance, proposal);
+ int status = cl->call_timeout(paxos_protocol::preparereq, milliseconds(100),
+ res, me, instance, proposal);
if (status == paxos_protocol::OK) {
- if (res.oldinstance) {
+ LOG << "preparereq response type=" << res.type << " n_a=(" << res.n_a.n
+ << ", " << res.n_a.m << ") " << "v_a=\"" << res.v_a << "\"";
+ if (res.type == prepareres::oldinstance) {
LOG << "commiting old instance!";
- commit(instance, res.v_a);
+ lock ml(acceptor_mutex);
+ commit(instance, res.v_a, ml);
return false;
- }
- LOG << "preparereq responded with oldinstance=" << res.oldinstance << " accept=" << res.accept << " n_a=(" << res.n_a.n << ", " << res.n_a.m << ") "
- << "v_a=\"" << res.v_a << "\"";
- if (res.accept) {
+ } else if (res.type == prepareres::accept) {
accepts.push_back(i);
if (res.n_a >= highest_n_a) {
LOG << "found a newer accepted proposal, \"" << res.v_a << "\", with number (" << res.n_a.n << ", " << res.n_a.m << ")";
void proposer_acceptor::accept(unsigned instance, nodes_t & accepts,
const nodes_t & nodes, const value_t & v) {
+ bool accept = false;
for (auto i : nodes) {
- auto r = rpcc::bind_cached(i);
- if (!r)
- continue;
- bool accept = false;
- int status = r->call_timeout(
- paxos_protocol::acceptreq, milliseconds(100), accept, me, instance, proposal, v);
- if (status == paxos_protocol::OK && accept)
- accepts.push_back(i);
+ if (auto cl = rpcc::bind_cached(i)) {
+ int status = cl->call_timeout(paxos_protocol::acceptreq, milliseconds(100),
+ accept, me, instance, proposal, v);
+ if (status == paxos_protocol::OK && accept)
+ accepts.push_back(i);
+ }
}
}
void proposer_acceptor::decide(unsigned instance, const nodes_t & accepts, const value_t & v) {
- for (auto i : accepts) {
- auto r = rpcc::bind_cached(i);
- if (!r)
- continue;
- int res = 0;
- r->call_timeout(paxos_protocol::decidereq, milliseconds(100), res, me, instance, v);
- }
+ int res = 0;
+ for (auto i : accepts)
+ if (auto cl = rpcc::bind_cached(i))
+ cl->call_timeout(paxos_protocol::decidereq, milliseconds(100), res, me, instance, v);
}
paxos_protocol::status
proposer_acceptor::preparereq(prepareres & r, const node_t &, unsigned instance, prop_t n) {
LOG << "instance " << instance << " proposal (" << n.n << ", " << n.m << ")";
lock ml(acceptor_mutex);
- r.oldinstance = false;
- r.accept = false;
- r.n_a = accepted;
- r.v_a = accepted_value;
if (instance <= instance_h) {
LOG << "old instance " << instance << " has value " << values[instance];
- r.oldinstance = true;
- r.v_a = values[instance];
+ r = prepareres{prepareres::oldinstance, accepted, values[instance]};
} else if (n > promise) {
LOG << "looks good to me";
promise = n;
l.logprop(promise);
- r.accept = true;
+ r = prepareres{prepareres::accept, accepted, accepted_value};
} else {
LOG << "I totally rejected this request. Ha.";
+ r = prepareres{prepareres::reject, accepted, accepted_value};
}
- LOG << "preparereq is responding with oldinstance=" << r.oldinstance << " accept=" << r.accept << " n_a=(" << r.n_a.n << ", " << r.n_a.m << ") "
- << "v_a=\"" << r.v_a << "\"";
+ LOG << "preparereq is responding with oldinstance=" << r.oldinstance << " accept=" << r.accept
+ << " n_a=(" << r.n_a.n << ", " << r.n_a.m << ") " << "v_a=\"" << r.v_a << "\"";
return paxos_protocol::OK;
}
proposer_acceptor::acceptreq(bool & r, const node_t &, unsigned instance, prop_t n, const value_t & v) {
lock ml(acceptor_mutex);
r = false;
- if (instance == instance_h + 1) {
- if (n >= promise) {
- accepted = n;
- accepted_value = v;
- l.logaccept(accepted, accepted_value);
- r = true;
- }
- return paxos_protocol::OK;
- } else {
+ if (instance != instance_h + 1)
return paxos_protocol::ERR;
+ if (n >= promise) {
+ accepted = n;
+ accepted_value = v;
+ l.logaccept(accepted, accepted_value);
+ r = true;
}
+ return paxos_protocol::OK;
}
paxos_protocol::status
LOG << "decidereq for accepted instance " << instance << " (my instance " << instance_h << ") v=" << accepted_value;
if (instance == instance_h + 1) {
VERIFY(accepted_value == v);
- commit(instance, accepted_value, ml);
+ commit(instance, v, ml);
} else if (instance <= instance_h) {
// we are ahead; ignore.
} else {
return paxos_protocol::OK;
}
-void proposer_acceptor::commit(unsigned instance, const value_t & value) {
- lock ml(acceptor_mutex);
- commit(instance, value, ml);
-}
-
-void proposer_acceptor::commit(unsigned instance, const value_t & value, lock & pxs_mutex_lock) {
+void proposer_acceptor::commit(unsigned instance, const value_t & value, lock & acceptor_mutex_lock) {
+ VERIFY(&value != &accepted_value); // eited by aliasing?
+ VERIFY(acceptor_mutex_lock);
LOG << "instance=" << instance << " has v=" << value;
if (instance > instance_h) {
LOG << "highestacceptedinstance = " << instance;
l.loginstance(instance, value);
instance_h = instance;
accepted = promise = {0, me};
- string v = value; // gaaahhh aliasing of value and accepted_value
- accepted_value.clear(); // this wipes out "value", too
+ accepted_value.clear();
if (delegate) {
- pxs_mutex_lock.unlock();
- delegate->paxos_commit(instance, v);
- pxs_mutex_lock.lock();
+ acceptor_mutex_lock.unlock();
+ delegate->paxos_commit(instance, value);
+ acceptor_mutex_lock.lock();
}
}
}
+++ /dev/null
-#!/usr/bin/perl -w
-
-use POSIX ":sys_wait_h";
-use Getopt::Std;
-use Time::HiRes (usleep);
-use strict;
-
-
-my @pid;
-my @logs = ();
-my @views = (); #expected views
-my %in_views; #the number of views a node is expected to be present
-my @p;
-my $t;
-my $always_kill = 0;
-
-use sigtrap 'handler' => \&killprocess, 'HUP', 'INT', 'ABRT', 'QUIT', 'TERM';
-
-sub paxos_log {
- my $port = shift;
- return "paxos-$port.log";
-}
-
-sub mydie {
- my ($s) = @_;
- killprocess() if ($always_kill);
- die $s;
-}
-
-sub killprocess {
- print "killprocess: forcestop all spawned processes...@pid \n";
- kill 9, @pid;
-}
-
-sub cleanup {
- kill 9, @pid;
- unlink(@logs);
- usleep 200000;
-}
-
-sub spawn {
- my ($p, @a) = @_;
- my $aa = join("-", @a);
- if (my $pid = fork) {
-# parent
- push( @logs, "$p-$aa.log" );
- if( $p =~ /lock_server/ ) {
- push( @logs, paxos_log($a[1]) );
- }
- return $pid;
- } elsif (defined $pid) {
-# child
- open(STDOUT, ">>$p-$aa.log")
- or mydie "Couln't redirect stout\n";
- open(STDERR, ">&STDOUT")
- or mydie "Couln't redirect stderr\n";
- $| = 1;
- print "$p @a\n";
- exec "$p @a"
- or mydie "Cannot start new $p @a $!\n";
- } else {
- mydie "Cannot fork: $!\n";
- }
-}
-
-sub randports {
-
- my $num = shift;
- my @p = ();
- for( my $i = 0; $i < $num; $i++ ) {
- push( @p, int(rand(54000/2))*2+10000 );
- }
- my @sp = sort { $a <=> $b } @p;
- return @sp;
-}
-
-sub print_config {
- my @ports = @_;
- open( CONFIG, ">config" ) or mydie( "Couldn't open config for writing" );
- foreach my $p (@ports) {
- printf CONFIG "%05d\n", $p;
- }
- close( CONFIG );
-}
-
-sub spawn_ls {
- my $master = shift;
- my $port = shift;
- return spawn( "./lock_server", $master, $port );
-}
-
-sub check_views {
-
- my $l = shift;
- my $v = shift;
- my $last_v = shift;
-
- open( LOG, "<$l" )
- or mydie( "Failed: couldn't read $l" );
- my @log = <LOG>;
- close(LOG);
-
- my @vs = @{$v};
-
- my $i = 0;
- my @last_view;
- foreach my $line (@log) {
- if( $line =~ /^done (\d+) ([\d\s]+)$/ ) {
-
- my $num = $1;
- my @view = split( /\s+/, $2 );
- @last_view = @view;
-
- if( $i > $#vs ) {
-# let there be extra views
- next;
- }
-
- my $e = $vs[$i];
- my @expected = @{$e};
-
- if( @expected != @view ) {
- mydie( "Failed: In log $l at view $num is (@view), but expected $i (@expected)" );
- }
-
- $i++;
- }
- }
-
- if( $i <= $#vs ) {
- mydie( "Failed: In log $l, not enough views seen!" );
- }
-
- if( defined $last_v ) {
- my @last_exp_v = @{$last_v};
- if( @last_exp_v != @last_view ) {
- mydie( "Failed: In log $l last view didn't match, got view @last_view, but expected @last_exp_v" );
- }
- }
-
-}
-
-sub get_num_views {
-
- my $log = shift;
- my $including = shift;
- my $nv = `grep "done " $log | grep "$including" | wc -l`;
- chomp $nv;
- return $nv;
-
-}
-
-sub wait_for_view_change {
-
- my $log = shift;
- my $num_views = shift;
- my $including = shift;
- my $timeout = shift;
-
- my $start = time();
- while( (get_num_views( $log, $including ) < $num_views) and
- ($start + $timeout > time()) ) {
- my $lastv = `grep done $log | tail -n 1`;
- chomp $lastv;
- print " Waiting for $including to be present in >=$num_views views in $log (Last view: $lastv)\n";
- usleep 100000;
- }
-
- if( get_num_views( $log, $including ) < $num_views) {
- mydie( "Failed: Timed out waiting for $including to be in >=$num_views in log $log" );
- }else{
- print " Done: $including is in >=$num_views views in $log\n";
- }
-}
-
-sub waitpid_to {
- my $pid = shift;
- my $to = shift;
-
- my $start = time();
- my $done_pid;
- do {
- usleep 100000;
- $done_pid = waitpid($pid, POSIX::WNOHANG);
- } while( $done_pid <= 0 and (time() - $start) < $to );
-
- if( $done_pid <= 0 ) {
- kill 9,$pid;
- mydie( "Failed: Timed out waiting for process $pid\n" );
- } else {
- return 1;
- }
-
-}
-
-sub wait_and_check_expected_view($) {
- my $v = shift;
- push @views, $v;
- for (my $i = 0; $i <=$#$v; $i++) {
- $in_views{$v->[$i]}++;
- }
- foreach my $port (@$v) {
- wait_for_view_change(paxos_log($port), $in_views{$port}, $port, 20);
- }
- foreach my $port (@$v) {
- my $log = paxos_log($port);
- check_views( $log, \@views );
- }
-}
-
-sub start_nodes ($$){
-
- @pid = ();
- @logs = ();
- @views = ();
- for (my $i = 0; $i <= $#p; $i++) {
- $in_views{$p[$i]} = 0;
- }
-
- my $n = shift;
- my $command = shift;
-
- for (my $i = 0; $i < $n; $i++) {
- if ($command eq "ls") {
- @pid = (@pid, spawn_ls($p[0],$p[$i]));
- print "Start lock_server on $p[$i]\n";
- }
- usleep 100000;
-
- my @vv = @p[0..$i];
- wait_and_check_expected_view(\@vv);
- }
-
-}
-
-my %options;
-getopts("s:k",\%options);
-if (defined($options{s})) {
- srand($options{s});
-}
-if (defined($options{k})) {
- $always_kill = 1;
-}
-
-#get a sorted list of random ports
-@p = randports(5);
-print_config( @p[0..4] );
-
-my @do_run = ();
-my $NUM_TESTS = 17;
-
-# see which tests are set
-if( $#ARGV > -1 ) {
- foreach my $t (@ARGV) {
- if( $t < $NUM_TESTS && $t >= 0 ) {
- $do_run[$t] = 1;
- }
- }
-} else {
-# turn on all tests
- for( my $i = 0; $i < $NUM_TESTS; $i++ ) {
- $do_run[$i] = 1;
- }
-}
-
-if ($do_run[0]) {
- print "test0: start 3-process lock server\n";
- start_nodes(3,"ls");
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[1]) {
- print "test1: start 3-process lock server, kill third server\n";
- start_nodes(3,"ls");
-
- print "Kill third server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 500000;
-
- # it should go through 4 views
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[2]) {
- print "test2: start 3-process lock server, kill first server\n";
- start_nodes(3,"ls");
-
- print "Kill first (PID: $pid[0]) on port $p[0]\n";
- kill "TERM", $pid[0];
-
- usleep 500000;
-
- # it should go through 4 views
- my @v4 = ($p[1], $p[2]);
- wait_and_check_expected_view(\@v4);
-
- cleanup();
- usleep 200000;
-}
-
-
-if ($do_run[3]) {
-
- print "test3: start 3-process lock_server, kill a server, restart a server\n";
- start_nodes(3,"ls");
-
- print "Kill server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 500000;
-
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- print "Restart killed server on port $p[2]\n";
- $pid[2] = spawn_ls ($p[0], $p[2]);
-
- usleep 500000;
-
- my @v5 = ($p[0], $p[1], $p[2]);
- wait_and_check_expected_view(\@v5);
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[4]) {
- print "test4: 3-process lock_server, kill third server, kill second server, restart third server, kill third server again, restart second server, re-restart third server, check logs\n";
- start_nodes(3,"ls");
-
- print "Kill server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 500000;
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- print "Kill server (PID: $pid[1]) on port $p[1]\n";
- kill "TERM", $pid[1];
-
- usleep 500000;
- #no view change can happen because of a lack of majority
-
- print "Restarting server on port $p[2]\n";
- $pid[2] = spawn_ls($p[0], $p[2]);
-
- usleep 500000;
-
- #no view change can happen because of a lack of majority
- foreach my $port (@p[0..2]) {
- my $num_v = get_num_views(paxos_log($port), $port);
- die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
- }
-
- # kill node 3 again,
- print "Kill server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 500000;
-
-
- print "Restarting server on port $p[1]\n";
- $pid[1] = spawn_ls($p[0], $p[1]);
-
- usleep 700000;
-
- foreach my $port (@p[0..1]) {
- $in_views{$port} = get_num_views( paxos_log($port), $port );
- print " Node $port is present in ", $in_views{$port}, " views in ", paxos_log($port), "\n";
- }
-
- print "Restarting server on port $p[2]\n";
- $pid[2] = spawn_ls($p[0], $p[2]);
-
- my @lastv = ($p[0],$p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
-# now check the paxos logs and make sure the logs go through the right
-# views
-
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv);
- }
-
- cleanup();
-
-}
-
-if ($do_run[5]) {
- print "test5: 3-process lock_server, send signal 1 to first server, kill third server, restart third server, check logs\n";
- start_nodes(3,"ls");
-
- print "Sending paxos breakpoint 1 to first server on port $p[0]\n";
- spawn("./rsm_tester", $p[0]+1, "breakpoint", 3);
-
- usleep 100000;
-
- print "Kill third server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 500000;
- foreach my $port (@p[0..2]) {
- my $num_v = get_num_views( paxos_log($port), $port );
- die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
- }
-
- print "Restarting third server on port $p[2]\n";
- $pid[2]= spawn_ls($p[0], $p[2]);
- my @lastv = ($p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
- usleep 1000000;
-
-# now check the paxos logs and make sure the logs go through the right
-# views
-
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv);
- }
-
- cleanup();
-
-}
-
-if ($do_run[6]) {
- print "test6: 4-process lock_server, send signal 2 to first server, kill fourth server, restart fourth server, check logs\n";
- start_nodes(4,"ls");
- print "Sending paxos breakpoint 2 to first server on port $p[0]\n";
- spawn("./rsm_tester", $p[0]+1, "breakpoint", 4);
-
- usleep 100000;
-
- print "Kill fourth server (PID: $pid[3]) on port $p[3]\n";
- kill "TERM", $pid[3];
-
- usleep 500000;
-
- foreach my $port ($p[1],$p[2]) {
- my $num_v = get_num_views( paxos_log($port), $port );
- die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
- }
-
- usleep 500000;
-
- print "Restarting fourth server on port $p[3]\n";
- $pid[3] = spawn_ls($p[1], $p[3]);
-
- usleep 500000;
-
- my @v5 = ($p[0],$p[1],$p[2]);
- foreach my $port (@v5) {
- $in_views{$port}++;
- }
- push @views, \@v5;
-
- usleep 1000000;
-
- # the 6th view will be (2,3) or (1,2,3,4)
- my @v6 = ($p[1],$p[2]);
- foreach my $port (@v6) {
- $in_views{$port}++;
- }
- foreach my $port (@v6) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 30);
- }
-
- # final will be (2,3,4)
- my @lastv = ($p[1],$p[2],$p[3]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv );
- }
- cleanup();
-
-}
-
-if ($do_run[7]) {
- print "test7: 4-process lock_server, send signal 2 to first server, kill fourth server, kill other servers, restart other servers, restart fourth server, check logs\n";
- start_nodes(4,"ls");
- print "Sending paxos breakpoint 2 to first server on port $p[0]\n";
- spawn("./rsm_tester", $p[0]+1, "breakpoint", 4);
- usleep 300000;
-
- print "Kill fourth server (PID: $pid[3]) on port $p[3]\n";
- kill "TERM", $pid[3];
-
- usleep 500000;
-
- print "Kill third server (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- print "Kill second server (PID: $pid[1]) on port $p[1]\n";
- kill "TERM", $pid[1];
-
- usleep 500000;
-
- print "Restarting second server on port $p[1]\n";
- $pid[1] = spawn_ls($p[0], $p[1]);
-
- usleep 500000;
-
- print "Restarting third server on port $p[2]\n";
- $pid[2] = spawn_ls($p[0], $p[2]);
-
- usleep 500000;
-
-#no view change is possible by now because there is no majority
- foreach my $port ($p[1],$p[2]) {
- my $num_v = get_num_views( paxos_log($port), $port );
- die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
- }
-
- print "Restarting fourth server on port $p[3]\n";
- $pid[3] = spawn_ls($p[1], $p[3]);
-
- usleep 500000;
-
- my @v5 = ($p[0], $p[1], $p[2]);
- push @views, \@v5;
- foreach my $port (@v5) {
- $in_views{$port}++;
- }
-
- usleep 1500000;
- my @lastv = ($p[1],$p[2],$p[3]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv);
- }
-
- cleanup();
-
-}
-
-if ($do_run[8]) {
- print "test8: start 3-process lock service\n";
- start_nodes(3,"ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 8" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[9]) {
-
- print "test9: start 3-process rsm, kill second slave while lock_tester is running\n";
- start_nodes(3,"ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- sleep int(rand(10)+1);
-
- print "Kill slave (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 300000;
-
- # it should go through 4 views
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 9" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[10]) {
-
- print "test10: start 3-process rsm, kill second slave and restarts it later while lock_tester is running\n";
- start_nodes(3,"ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- sleep int(rand(10)+1);
-
- print "Kill slave (PID: $pid[2]) on port $p[2]\n";
- kill "TERM", $pid[2];
-
- usleep 300000;
-
- # it should go through 4 views
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- usleep 300000;
-
- print "Restarting killed lock_server on port $p[2]\n";
- $pid[2] = spawn_ls($p[0], $p[2]);
- my @v5 = ($p[0],$p[1],$p[2]);
- wait_and_check_expected_view(\@v5);
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 10" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-
-if ($do_run[11]) {
-
- print "test11: start 3-process rsm, kill primary while lock_tester is running\n";
- start_nodes(3,"ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- sleep int(rand(10)+1);
-
- print "Kill primary (PID: $pid[0]) on port $p[0]\n";
- kill "TERM", $pid[0];
-
- usleep 300000;
-
- # it should go through 4 views
- my @v4 = ($p[1], $p[2]);
- wait_and_check_expected_view(\@v4);
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 11" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[12]) {
-
- print "test12: start 3-process rsm, kill master at break1 and restart it while lock_tester is running\n";
-
- start_nodes(3, "ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- usleep 100000;
-
- print "Kill master (PID: $pid[0]) on port $p[0] at breakpoint 1\n";
- spawn("./rsm_tester", $p[0]+1, "breakpoint", 1);
-
-
- usleep 100000;
-
- # it should go through 5 views
- my @v4 = ($p[1], $p[2]);
- wait_and_check_expected_view(\@v4);
-
- print "Restarting killed lock_server on port $p[0]\n";
- $pid[0] = spawn_ls($p[1], $p[0]);
-
- usleep 300000;
-
- # the last view should include all nodes
- my @lastv = ($p[0],$p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv);
- }
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 12" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[13]) {
-
- print "test13: start 3-process rsm, kill slave at break1 and restart it while lock_tester is running\n";
-
- start_nodes(3, "ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- usleep 100000;
-
- print "Kill slave (PID: $pid[2]) on port $p[2] at breakpoint 1\n";
- spawn("./rsm_tester", $p[2]+1, "breakpoint", 1);
-
- usleep 100000;
-
- # it should go through 4 views
- my @v4 = ($p[0], $p[1]);
- wait_and_check_expected_view(\@v4);
-
- print "Restarting killed lock_server on port $p[2]\n";
- $pid[2] = spawn_ls($p[0], $p[2]);
-
- usleep 300000;
-
- # the last view should include all nodes
- my @lastv = ($p[0],$p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- foreach my $port (@lastv) {
- check_views( paxos_log($port), \@views, \@lastv);
- }
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 13" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[14]) {
-
- print "test14: start 5-process rsm, kill slave break1, kill slave break2\n";
-
- start_nodes(5, "ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- usleep 100000;
-
- print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n";
- spawn("./rsm_tester", $p[4]+1, "breakpoint", 1);
-
-
- print "Kill slave (PID: $pid[3]) on port $p[3] at breakpoint 2\n";
- spawn("./rsm_tester", $p[3]+1, "breakpoint", 2);
-
-
- usleep 100000;
-
- # two view changes:
-
- print "first view change wait\n";
- my @lastv = ($p[0],$p[1],$p[2],$p[3]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- print "second view change wait\n";
-
- @lastv = ($p[0],$p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 14" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[15]) {
-
- print "test15: start 5-process rsm, kill slave break1, kill primary break2\n";
-
- start_nodes(5, "ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- usleep 100000;
-
- print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n";
- spawn("./rsm_tester", $p[4]+1, "breakpoint", 1);
-
-
- print "Kill primary (PID: $pid[0]) on port $p[0] at breakpoint 2\n";
- spawn("./rsm_tester", $p[0]+1, "breakpoint", 2);
-
- usleep 100000;
-
- # two view changes:
-
- print "first view change wait\n";
- my @lastv = ($p[0],$p[1],$p[2],$p[3]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- print "second view change wait\n";
-
- @lastv = ($p[1],$p[2],$p[3]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 15" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-if ($do_run[16]) {
-
- print "test16: start 3-process rsm, partition primary, heal it\n";
-
- start_nodes(3, "ls");
-
- print "Start lock_tester $p[0]\n";
- $t = spawn("./lock_tester", $p[0]);
-
- usleep 100000;
-
- print "Partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n";
-
- spawn("./rsm_tester", $p[0]+1, "partition", 0);
-
- usleep 300000;
-
- print "first view change wait\n";
- my @lastv = ($p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- usleep 100000;
-
- print "Heal partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n";
- spawn("./rsm_tester", $p[0]+1, "partition", 1);
-
- usleep 100000;
-
- # xxx it should test that this is the 5th view!
- print "second view change wait\n";
- @lastv = ($p[0], $p[1],$p[2]);
- foreach my $port (@lastv) {
- wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
- }
-
- print " Wait for lock_tester to finish (waitpid $t)\n";
- waitpid_to($t, 600);
-
- if( system( "grep \"passed all tests successfully\" lock_tester-$p[0].log" ) ) {
- mydie( "Failed lock tester for test 16" );
- }
-
- cleanup();
- usleep 200000;
-}
-
-print "tests done OK\n";
-
-unlink("config");
projects / invirt/third/libt4.git / commitdiff
