X-Git-Url: http://xvm.mit.edu/gitweb/invirt/third/libt4.git/blobdiff_plain/16e7c282c6fcec8189425bd15ec9e8a4a0ee857d..f0dcb6b97d6d40f67698d1f71ac26970f1776f82:/rsm_tester.pl diff --git a/rsm_tester.pl b/rsm_tester.pl index c99c66f..43a0fb4 100755 --- a/rsm_tester.pl +++ b/rsm_tester.pl @@ -2,6 +2,7 @@ use POSIX ":sys_wait_h"; use Getopt::Std; +use Time::HiRes (usleep); use strict; @@ -34,7 +35,7 @@ sub killprocess { sub cleanup { kill 9, @pid; unlink(@logs); - sleep 2; + usleep 200000; } sub spawn { @@ -67,7 +68,7 @@ sub randports { my $num = shift; my @p = (); for( my $i = 0; $i < $num; $i++ ) { - push( @p, int(rand(54000))+10000 ); + push( @p, int(rand(54000/2))*2+10000 ); } my @sp = sort { $a <=> $b } @p; return @sp; @@ -162,7 +163,7 @@ sub wait_for_view_change { my $lastv = `grep done $log | tail -n 1`; chomp $lastv; print " Waiting for $including to be present in >=$num_views views in $log (Last view: $lastv)\n"; - sleep 1; + usleep 100000; } if( get_num_views( $log, $including ) < $num_views) { @@ -179,7 +180,7 @@ sub waitpid_to { my $start = time(); my $done_pid; do { - sleep 1; + usleep 100000; $done_pid = waitpid($pid, POSIX::WNOHANG); } while( $done_pid <= 0 and (time() - $start) < $to ); @@ -224,7 +225,7 @@ sub start_nodes ($$){ @pid = (@pid, spawn_ls($p[0],$p[$i])); print "Start lock_server on $p[$i]\n"; } - sleep 1; + usleep 100000; my @vv = @p[0..$i]; wait_and_check_expected_view(\@vv); @@ -266,7 +267,7 @@ if ($do_run[0]) { print "test0: start 3-process lock server\n"; start_nodes(3,"ls"); cleanup(); - sleep 2; + usleep 200000; } if ($do_run[1]) { @@ -276,14 +277,14 @@ if ($do_run[1]) { print "Kill third server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 5; + usleep 500000; # it should go through 4 views my @v4 = ($p[0], $p[1]); wait_and_check_expected_view(\@v4); cleanup(); - sleep 2; + usleep 200000; } if ($do_run[2]) { @@ -293,14 +294,14 @@ if ($do_run[2]) { print "Kill first (PID: $pid[0]) on port $p[0]\n"; kill "TERM", $pid[0]; - sleep 5; + usleep 500000; # it should go through 4 views my @v4 = ($p[1], $p[2]); wait_and_check_expected_view(\@v4); cleanup(); - sleep 2; + usleep 200000; } @@ -312,7 +313,7 @@ if ($do_run[3]) { print "Kill server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 5; + usleep 500000; my @v4 = ($p[0], $p[1]); wait_and_check_expected_view(\@v4); @@ -320,13 +321,13 @@ if ($do_run[3]) { print "Restart killed server on port $p[2]\n"; $pid[2] = spawn_ls ($p[0], $p[2]); - sleep 5; + usleep 500000; my @v5 = ($p[0], $p[1], $p[2]); wait_and_check_expected_view(\@v5); cleanup(); - sleep 2; + usleep 200000; } if ($do_run[4]) { @@ -336,20 +337,20 @@ if ($do_run[4]) { print "Kill server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 5; + usleep 500000; my @v4 = ($p[0], $p[1]); wait_and_check_expected_view(\@v4); print "Kill server (PID: $pid[1]) on port $p[1]\n"; kill "TERM", $pid[1]; - sleep 5; + usleep 500000; #no view change can happen because of a lack of majority print "Restarting server on port $p[2]\n"; $pid[2] = spawn_ls($p[0], $p[2]); - sleep 5; + usleep 500000; #no view change can happen because of a lack of majority foreach my $port (@p[0..2]) { @@ -361,13 +362,13 @@ if ($do_run[4]) { print "Kill server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 5; + usleep 500000; print "Restarting server on port $p[1]\n"; $pid[1] = spawn_ls($p[0], $p[1]); - sleep 7; + usleep 700000; foreach my $port (@p[0..1]) { $in_views{$port} = get_num_views( paxos_log($port), $port ); @@ -400,12 +401,12 @@ if ($do_run[5]) { print "Sending paxos breakpoint 1 to first server on port $p[0]\n"; spawn("./rsm_tester", $p[0]+1, "breakpoint", 3); - sleep 1; + usleep 100000; print "Kill third server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 5; + usleep 500000; foreach my $port (@p[0..2]) { my $num_v = get_num_views( paxos_log($port), $port ); die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port}); @@ -417,7 +418,7 @@ if ($do_run[5]) { foreach my $port (@lastv) { wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20); } - sleep 10; + usleep 1000000; # now check the paxos logs and make sure the logs go through the right # views @@ -436,24 +437,24 @@ if ($do_run[6]) { print "Sending paxos breakpoint 2 to first server on port $p[0]\n"; spawn("./rsm_tester", $p[0]+1, "breakpoint", 4); - sleep 1; + usleep 100000; print "Kill fourth server (PID: $pid[3]) on port $p[3]\n"; kill "TERM", $pid[3]; - sleep 5; + usleep 500000; foreach my $port ($p[1],$p[2]) { my $num_v = get_num_views( paxos_log($port), $port ); die "$num_v views in ", paxos_log($port), " : no new views should be formed due to the lack of majority\n" if ($num_v != $in_views{$port}); } - sleep 5; + usleep 500000; print "Restarting fourth server on port $p[3]\n"; $pid[3] = spawn_ls($p[1], $p[3]); - sleep 5; + usleep 500000; my @v5 = ($p[0],$p[1],$p[2]); foreach my $port (@v5) { @@ -461,7 +462,7 @@ if ($do_run[6]) { } push @views, \@v5; - sleep 10; + usleep 1000000; # the 6th view will be (2,3) or (1,2,3,4) my @v6 = ($p[1],$p[2]); @@ -489,12 +490,12 @@ if ($do_run[7]) { start_nodes(4,"ls"); print "Sending paxos breakpoint 2 to first server on port $p[0]\n"; spawn("./rsm_tester", $p[0]+1, "breakpoint", 4); - sleep 3; + usleep 300000; print "Kill fourth server (PID: $pid[3]) on port $p[3]\n"; kill "TERM", $pid[3]; - sleep 5; + usleep 500000; print "Kill third server (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; @@ -502,17 +503,17 @@ if ($do_run[7]) { print "Kill second server (PID: $pid[1]) on port $p[1]\n"; kill "TERM", $pid[1]; - sleep 5; + usleep 500000; print "Restarting second server on port $p[1]\n"; $pid[1] = spawn_ls($p[0], $p[1]); - sleep 5; + usleep 500000; print "Restarting third server on port $p[2]\n"; $pid[2] = spawn_ls($p[0], $p[2]); - sleep 5; + usleep 500000; #no view change is possible by now because there is no majority foreach my $port ($p[1],$p[2]) { @@ -523,7 +524,7 @@ if ($do_run[7]) { print "Restarting fourth server on port $p[3]\n"; $pid[3] = spawn_ls($p[1], $p[3]); - sleep 5; + usleep 500000; my @v5 = ($p[0], $p[1], $p[2]); push @views, \@v5; @@ -531,7 +532,7 @@ if ($do_run[7]) { $in_views{$port}++; } - sleep 15; + usleep 1500000; my @lastv = ($p[1],$p[2],$p[3]); foreach my $port (@lastv) { wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20); @@ -560,7 +561,7 @@ if ($do_run[8]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[9]) { @@ -576,7 +577,7 @@ if ($do_run[9]) { print "Kill slave (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 3; + usleep 300000; # it should go through 4 views my @v4 = ($p[0], $p[1]); @@ -590,7 +591,7 @@ if ($do_run[9]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[10]) { @@ -606,13 +607,13 @@ if ($do_run[10]) { print "Kill slave (PID: $pid[2]) on port $p[2]\n"; kill "TERM", $pid[2]; - sleep 3; + usleep 300000; # it should go through 4 views my @v4 = ($p[0], $p[1]); wait_and_check_expected_view(\@v4); - sleep 3; + usleep 300000; print "Restarting killed lock_server on port $p[2]\n"; $pid[2] = spawn_ls($p[0], $p[2]); @@ -627,7 +628,7 @@ if ($do_run[10]) { } cleanup(); - sleep 2; + usleep 200000; } @@ -644,7 +645,7 @@ if ($do_run[11]) { print "Kill primary (PID: $pid[0]) on port $p[0]\n"; kill "TERM", $pid[0]; - sleep 3; + usleep 300000; # it should go through 4 views my @v4 = ($p[1], $p[2]); @@ -658,7 +659,7 @@ if ($do_run[11]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[12]) { @@ -670,13 +671,13 @@ if ($do_run[12]) { print "Start lock_tester $p[0]\n"; $t = spawn("./lock_tester", $p[0]); - sleep 1; + usleep 100000; print "Kill master (PID: $pid[0]) on port $p[0] at breakpoint 1\n"; spawn("./rsm_tester", $p[0]+1, "breakpoint", 1); - sleep 1; + usleep 100000; # it should go through 5 views my @v4 = ($p[1], $p[2]); @@ -685,7 +686,7 @@ if ($do_run[12]) { print "Restarting killed lock_server on port $p[0]\n"; $pid[0] = spawn_ls($p[1], $p[0]); - sleep 3; + usleep 300000; # the last view should include all nodes my @lastv = ($p[0],$p[1],$p[2]); @@ -705,7 +706,7 @@ if ($do_run[12]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[13]) { @@ -717,12 +718,12 @@ if ($do_run[13]) { print "Start lock_tester $p[0]\n"; $t = spawn("./lock_tester", $p[0]); - sleep 1; + usleep 100000; print "Kill slave (PID: $pid[2]) on port $p[2] at breakpoint 1\n"; spawn("./rsm_tester", $p[2]+1, "breakpoint", 1); - sleep 1; + usleep 100000; # it should go through 4 views my @v4 = ($p[0], $p[1]); @@ -731,7 +732,7 @@ if ($do_run[13]) { print "Restarting killed lock_server on port $p[2]\n"; $pid[2] = spawn_ls($p[0], $p[2]); - sleep 3; + usleep 300000; # the last view should include all nodes my @lastv = ($p[0],$p[1],$p[2]); @@ -751,7 +752,7 @@ if ($do_run[13]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[14]) { @@ -763,7 +764,7 @@ if ($do_run[14]) { print "Start lock_tester $p[0]\n"; $t = spawn("./lock_tester", $p[0]); - sleep 1; + usleep 100000; print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n"; spawn("./rsm_tester", $p[4]+1, "breakpoint", 1); @@ -773,7 +774,7 @@ if ($do_run[14]) { spawn("./rsm_tester", $p[3]+1, "breakpoint", 2); - sleep 1; + usleep 100000; # two view changes: @@ -798,7 +799,7 @@ if ($do_run[14]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[15]) { @@ -810,7 +811,7 @@ if ($do_run[15]) { print "Start lock_tester $p[0]\n"; $t = spawn("./lock_tester", $p[0]); - sleep 1; + usleep 100000; print "Kill slave (PID: $pid[4]) on port $p[4] at breakpoint 1\n"; spawn("./rsm_tester", $p[4]+1, "breakpoint", 1); @@ -819,7 +820,7 @@ if ($do_run[15]) { print "Kill primary (PID: $pid[0]) on port $p[0] at breakpoint 2\n"; spawn("./rsm_tester", $p[0]+1, "breakpoint", 2); - sleep 1; + usleep 100000; # two view changes: @@ -844,7 +845,7 @@ if ($do_run[15]) { } cleanup(); - sleep 2; + usleep 200000; } if ($do_run[16]) { @@ -856,13 +857,13 @@ if ($do_run[16]) { print "Start lock_tester $p[0]\n"; $t = spawn("./lock_tester", $p[0]); - sleep 1; + usleep 100000; print "Partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n"; spawn("./rsm_tester", $p[0]+1, "partition", 0); - sleep 3; + usleep 300000; print "first view change wait\n"; my @lastv = ($p[1],$p[2]); @@ -870,12 +871,12 @@ if ($do_run[16]) { wait_for_view_change(paxos_log($port), $in_views{$port}+1, $port, 20); } - sleep 1; + usleep 100000; print "Heal partition primary (PID: $pid[0]) on port $p[0] at breakpoint\n"; spawn("./rsm_tester", $p[0]+1, "partition", 1); - sleep 1; + usleep 100000; # xxx it should test that this is the 5th view! print "second view change wait\n"; @@ -892,7 +893,7 @@ if ($do_run[16]) { } cleanup(); - sleep 2; + usleep 200000; } print "tests done OK\n";