use POSIX ":sys_wait_h";
use Getopt::Std;
+use Time::HiRes (usleep);
use strict;
sub cleanup {
kill 9, @pid;
unlink(@logs);
- sleep 2;
+ usleep 200000;
}
sub spawn {
my $num = shift;
my @p = ();
for( my $i = 0; $i < $num; $i++ ) {
- push( @p, int(rand(54000))+10000 );
+ push( @p, int(rand(54000/2))*2+10000 );
}
my @sp = sort { $a <=> $b } @p;
return @sp;
my $start = time();
while( (get_num_views( $log, $including ) < $num_views) and
($start + $timeout > time()) ) {
- my $lastv = `grep done $log | tail -n 1`;
- chomp $lastv;
+ my $lastv = `grep done $log | tail -n 1`;
+ chomp $lastv;
print " Waiting for $including to be present in >=$num_views views in $log (Last view: $lastv)\n";
- sleep 1;
+ usleep 100000;
}
if( get_num_views( $log, $including ) < $num_views) {
my $start = time();
my $done_pid;
do {
- sleep 1;
+ usleep 100000;
$done_pid = waitpid($pid, POSIX::WNOHANG);
} while( $done_pid <= 0 and (time() - $start) < $to );
my $command = shift;
for (my $i = 0; $i < $n; $i++) {
- if ($command eq "ls") {
- @pid = (@pid, spawn_ls($p[0],$p[$i]));
- print "Start lock_server on $p[$i]\n";
- }
- sleep 1;
+ if ($command eq "ls") {
+ @pid = (@pid, spawn_ls($p[0],$p[$i]));
+ print "Start lock_server on $p[$i]\n";
+ }
+ usleep 100000;
my @vv = @p[0..$i];
wait_and_check_expected_view(\@vv);
print "test0: start 3-process lock server\n";
start_nodes(3,"ls");
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[1]) {
print "Kill third server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 5;
+ usleep 500000;
# it should go through 4 views
my @v4 = ($p[0], $p[1]);
wait_and_check_expected_view(\@v4);
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[2]) {
print "Kill first (PID: $pid[0]) on port $p[0]\n";
kill "TERM", $pid[0];
- sleep 5;
+ usleep 500000;
# it should go through 4 views
my @v4 = ($p[1], $p[2]);
wait_and_check_expected_view(\@v4);
cleanup();
- sleep 2;
+ usleep 200000;
}
print "Kill server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 5;
+ usleep 500000;
my @v4 = ($p[0], $p[1]);
wait_and_check_expected_view(\@v4);
print "Restart killed server on port $p[2]\n";
$pid[2] = spawn_ls ($p[0], $p[2]);
- sleep 5;
+ usleep 500000;
my @v5 = ($p[0], $p[1], $p[2]);
wait_and_check_expected_view(\@v5);
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[4]) {
print "Kill server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 5;
+ usleep 500000;
my @v4 = ($p[0], $p[1]);
wait_and_check_expected_view(\@v4);
print "Kill server (PID: $pid[1]) on port $p[1]\n";
kill "TERM", $pid[1];
- sleep 5;
+ usleep 500000;
#no view change can happen because of a lack of majority
print "Restarting server on port $p[2]\n";
$pid[2] = spawn_ls($p[0], $p[2]);
- sleep 5;
+ usleep 500000;
#no view change can happen because of a lack of majority
foreach my $port (@p[0..2]) {
print "Kill server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 5;
+ usleep 500000;
print "Restarting server on port $p[1]\n";
$pid[1] = spawn_ls($p[0], $p[1]);
- sleep 7;
+ usleep 700000;
foreach my $port (@p[0..1]) {
$in_views{$port} = get_num_views( paxos_log($port), $port );
print "Sending paxos breakpoint 1 to first server on port $p[0]\n";
spawn("./rsm_tester", $p[0]+1, "breakpoint", 3);
- sleep 1;
+ usleep 100000;
print "Kill third server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 5;
+ usleep 500000;
foreach my $port (@p[0..2]) {
my $num_v = get_num_views( paxos_log($port), $port );
die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
foreach my $port (@lastv) {
wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
}
- sleep 10;
+ usleep 1000000;
# now check the paxos logs and make sure the logs go through the right
# views
print "Sending paxos breakpoint 2 to first server on port $p[0]\n";
spawn("./rsm_tester", $p[0]+1, "breakpoint", 4);
- sleep 1;
+ usleep 100000;
print "Kill fourth server (PID: $pid[3]) on port $p[3]\n";
kill "TERM", $pid[3];
- sleep 5;
+ usleep 500000;
foreach my $port ($p[1],$p[2]) {
my $num_v = get_num_views( paxos_log($port), $port );
die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port});
}
- sleep 5;
+ usleep 500000;
print "Restarting fourth server on port $p[3]\n";
$pid[3] = spawn_ls($p[1], $p[3]);
- sleep 5;
+ usleep 500000;
my @v5 = ($p[0],$p[1],$p[2]);
foreach my $port (@v5) {
}
push @views, \@v5;
- sleep 10;
+ usleep 1000000;
# the 6th view will be (2,3) or (1,2,3,4)
my @v6 = ($p[1],$p[2]);
start_nodes(4,"ls");
print "Sending paxos breakpoint 2 to first server on port $p[0]\n";
spawn("./rsm_tester", $p[0]+1, "breakpoint", 4);
- sleep 3;
+ usleep 300000;
print "Kill fourth server (PID: $pid[3]) on port $p[3]\n";
kill "TERM", $pid[3];
- sleep 5;
+ usleep 500000;
print "Kill third server (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
print "Kill second server (PID: $pid[1]) on port $p[1]\n";
kill "TERM", $pid[1];
- sleep 5;
+ usleep 500000;
print "Restarting second server on port $p[1]\n";
$pid[1] = spawn_ls($p[0], $p[1]);
- sleep 5;
+ usleep 500000;
print "Restarting third server on port $p[2]\n";
$pid[2] = spawn_ls($p[0], $p[2]);
- sleep 5;
+ usleep 500000;
#no view change is possible by now because there is no majority
foreach my $port ($p[1],$p[2]) {
print "Restarting fourth server on port $p[3]\n";
$pid[3] = spawn_ls($p[1], $p[3]);
- sleep 5;
+ usleep 500000;
my @v5 = ($p[0], $p[1], $p[2]);
push @views, \@v5;
$in_views{$port}++;
}
- sleep 15;
+ usleep 1500000;
my @lastv = ($p[1],$p[2],$p[3]);
foreach my $port (@lastv) {
wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
if ($do_run[8]) {
print "test8: start 3-process lock service\n";
- start_nodes(3,"ls");
+ start_nodes(3,"ls");
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[9]) {
print "test9: start 3-process rsm, kill second slave while lock_tester is running\n";
- start_nodes(3,"ls");
+ start_nodes(3,"ls");
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
print "Kill slave (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 3;
+ usleep 300000;
# it should go through 4 views
my @v4 = ($p[0], $p[1]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[10]) {
print "test10: start 3-process rsm, kill second slave and restarts it later while lock_tester is running\n";
- start_nodes(3,"ls");
+ start_nodes(3,"ls");
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
print "Kill slave (PID: $pid[2]) on port $p[2]\n";
kill "TERM", $pid[2];
- sleep 3;
+ usleep 300000;
# it should go through 4 views
my @v4 = ($p[0], $p[1]);
wait_and_check_expected_view(\@v4);
- sleep 3;
+ usleep 300000;
print "Restarting killed lock_server on port $p[2]\n";
$pid[2] = spawn_ls($p[0], $p[2]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[11]) {
print "test11: start 3-process rsm, kill primary while lock_tester is running\n";
- start_nodes(3,"ls");
+ start_nodes(3,"ls");
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
print "Kill primary (PID: $pid[0]) on port $p[0]\n";
kill "TERM", $pid[0];
- sleep 3;
+ usleep 300000;
# it should go through 4 views
my @v4 = ($p[1], $p[2]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[12]) {
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
- sleep 1;
+ usleep 100000;
print "Kill master (PID: $pid[0]) on port $p[0] at breakpoint 1\n";
spawn("./rsm_tester", $p[0]+1, "breakpoint", 1);
- sleep 1;
+ usleep 100000;
# it should go through 5 views
my @v4 = ($p[1], $p[2]);
print "Restarting killed lock_server on port $p[0]\n";
$pid[0] = spawn_ls($p[1], $p[0]);
- sleep 3;
+ usleep 300000;
# the last view should include all nodes
my @lastv = ($p[0],$p[1],$p[2]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[13]) {
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
- sleep 1;
+ usleep 100000;
print "Kill slave (PID: $pid[2]) on port $p[2] at breakpoint 1\n";
spawn("./rsm_tester", $p[2]+1, "breakpoint", 1);
- sleep 1;
+ usleep 100000;
# it should go through 4 views
my @v4 = ($p[0], $p[1]);
print "Restarting killed lock_server on port $p[2]\n";
$pid[2] = spawn_ls($p[0], $p[2]);
- sleep 3;
+ usleep 300000;
# the last view should include all nodes
my @lastv = ($p[0],$p[1],$p[2]);
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[14]) {
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
- sleep 1;
+ usleep 100000;
print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n";
spawn("./rsm_tester", $p[4]+1, "breakpoint", 1);
spawn("./rsm_tester", $p[3]+1, "breakpoint", 2);
- sleep 1;
+ usleep 100000;
# two view changes:
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[15]) {
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
- sleep 1;
+ usleep 100000;
print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n";
spawn("./rsm_tester", $p[4]+1, "breakpoint", 1);
print "Kill primary (PID: $pid[0]) on port $p[0] at breakpoint 2\n";
spawn("./rsm_tester", $p[0]+1, "breakpoint", 2);
- sleep 1;
+ usleep 100000;
# two view changes:
}
cleanup();
- sleep 2;
+ usleep 200000;
}
if ($do_run[16]) {
print "Start lock_tester $p[0]\n";
$t = spawn("./lock_tester", $p[0]);
- sleep 1;
+ usleep 100000;
print "Partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n";
spawn("./rsm_tester", $p[0]+1, "partition", 0);
- sleep 3;
+ usleep 300000;
print "first view change wait\n";
my @lastv = ($p[1],$p[2]);
wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20);
}
- sleep 1;
+ usleep 100000;
print "Heal partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n";
spawn("./rsm_tester", $p[0]+1, "partition", 1);
- sleep 1;
+ usleep 100000;
# xxx it should test that this is the 5th view!
print "second view change wait\n";
}
cleanup();
- sleep 2;
+ usleep 200000;
}
print "tests done OK\n";
projects / invirt/third/libt4.git / blobdiff