This is an automated email from the ASF dual-hosted git repository. gaston pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit da69a034cf2a257ff2ef39f17396f3dc7810a7b1 Author: Meng Zhu <[email protected]> AuthorDate: Thu Sep 20 14:54:32 2018 -0700 Made authentication tests independent of wall clock time. This patch pauses and manipulates the clock for all authentication tests. This makes the test less flaky-prone and speeds up the test run time. Review: https://reviews.apache.org/r/68414/ --- src/tests/authentication_tests.cpp | 132 +++++++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 26 deletions(-) diff --git a/src/tests/authentication_tests.cpp b/src/tests/authentication_tests.cpp index f7a2cf1..bbb7e10 100644 --- a/src/tests/authentication_tests.cpp +++ b/src/tests/authentication_tests.cpp @@ -30,8 +30,6 @@ #include "master/detector/standalone.hpp" -#include "sched/constants.hpp" - #include "tests/mesos.hpp" #include "tests/utils.hpp" @@ -59,6 +57,8 @@ class AuthenticationTest : public MesosTest {}; // denied registration by the master. TEST_F(AuthenticationTest, UnauthenticatedFramework) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -84,6 +84,8 @@ TEST_F(AuthenticationTest, UnauthenticatedFramework) // denied registration by the master, but not shut down. TEST_F(AuthenticationTest, UnauthenticatedSlave) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -105,23 +107,21 @@ TEST_F(AuthenticationTest, UnauthenticatedSlave) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); + // Advance the clock to trigger the first registration attempt. + // The initial registration delay is between [0, registration_backoff_factor]. + Clock::advance(flags.registration_backoff_factor); + Clock::settle(); + AWAIT_READY(registerSlaveMessage1); Future<RegisterSlaveMessage> registerSlaveMessage2 = FUTURE_PROTOBUF(RegisterSlaveMessage(), _, _); // Advance the clock to trigger another registration attempt. - Clock::pause(); Clock::advance(slave::REGISTER_RETRY_INTERVAL_MAX); Clock::settle(); - Clock::resume(); AWAIT_READY(registerSlaveMessage2); - - // Settle to make sure neither `SlaveRegisteredMessage` nor - // `ShutdownMessage` are sent. - Clock::pause(); - Clock::settle(); } @@ -129,6 +129,8 @@ TEST_F(AuthenticationTest, UnauthenticatedSlave) // authentication disabled, it registers unauthenticated frameworks. TEST_F(AuthenticationTest, DisableFrameworkAuthentication) { + Clock::pause(); + master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; // Disable authentication. @@ -158,6 +160,8 @@ TEST_F(AuthenticationTest, DisableFrameworkAuthentication) // authentication disabled, it registers unauthenticated slaves. TEST_F(AuthenticationTest, DisableSlaveAuthentication) { + Clock::pause(); + master::Flags flags = CreateMasterFlags(); flags.authenticate_agents = false; // Disable authentication. @@ -175,6 +179,11 @@ TEST_F(AuthenticationTest, DisableSlaveAuthentication) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags); ASSERT_SOME(slave); + // Advance the clock to trigger registration attempt. + // The initial registration delay is between [0, registration_backoff_factor]. + Clock::advance(slaveFlags.registration_backoff_factor); + Clock::settle(); + // Slave should be able to get registered. AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); @@ -186,6 +195,8 @@ TEST_F(AuthenticationTest, DisableSlaveAuthentication) // FrameworkInfo.principal than Credential.principal. TEST_F(AuthenticationTest, MismatchedFrameworkInfoPrincipal) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -216,6 +227,8 @@ TEST_F(AuthenticationTest, MismatchedFrameworkInfoPrincipal) // when authentication is not required. TEST_F(AuthenticationTest, DisabledFrameworkAuthenticationPrincipalMismatch) { + Clock::pause(); + master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; // Authentication not required. @@ -248,6 +261,8 @@ TEST_F(AuthenticationTest, DisabledFrameworkAuthenticationPrincipalMismatch) // register. TEST_F(AuthenticationTest, UnspecifiedFrameworkInfoPrincipal) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -276,6 +291,8 @@ TEST_F(AuthenticationTest, UnspecifiedFrameworkInfoPrincipal) // authentication disabled, it registers authenticated frameworks. TEST_F(AuthenticationTest, AuthenticatedFramework) { + Clock::pause(); + master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; // Disable authentication. @@ -305,6 +322,8 @@ TEST_F(AuthenticationTest, AuthenticatedFramework) // authentication disabled, it registers authenticated slaves. TEST_F(AuthenticationTest, AuthenticatedSlave) { + Clock::pause(); + master::Flags flags = CreateMasterFlags(); flags.authenticate_agents = false; // Disable authentication. @@ -320,6 +339,12 @@ TEST_F(AuthenticationTest, AuthenticatedSlave) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get()); ASSERT_SOME(slave); + // Advance the clock to trigger authentication and registration attempt. + // The initial authentication/registration delay is between + // [0, registration_backoff_factor]. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + // Slave should be able to get registered. AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); @@ -330,6 +355,8 @@ TEST_F(AuthenticationTest, AuthenticatedSlave) // authentication when authenticate message is lost. TEST_F(AuthenticationTest, RetryFrameworkAuthentication) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -350,10 +377,9 @@ TEST_F(AuthenticationTest, RetryFrameworkAuthentication) .WillOnce(FutureSatisfy(®istered)); // Advance the clock for the scheduler to retry. - Clock::pause(); - Clock::advance(Seconds(5)); + Clock::advance( + mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Scheduler should be able to get registered. AWAIT_READY(registered); @@ -367,6 +393,8 @@ TEST_F(AuthenticationTest, RetryFrameworkAuthentication) // authentication when authenticate message is lost. TEST_F(AuthenticationTest, RetrySlaveAuthentication) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -378,16 +406,20 @@ TEST_F(AuthenticationTest, RetrySlaveAuthentication) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get()); ASSERT_SOME(slave); + // Advance the clock to trigger authentication attempt. + // Currently the initial backoff is [0, registration_backoff_factor] + // for agents both with or without authentication. See MESOS-9173. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + AWAIT_READY(authenticateMessage); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); // Advance the clock for the slave to retry. - Clock::pause(); Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Slave should be able to get registered. AWAIT_READY(slaveRegisteredMessage); @@ -466,6 +498,8 @@ TEST_F(AuthenticationTest, MasterRetriedAuthenticationHandling) // is lost. TEST_F(AuthenticationTest, DropIntermediateSASLMessage) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -489,10 +523,9 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessage) .WillOnce(FutureSatisfy(®istered)); // Advance the clock for the scheduler to retry. - Clock::pause(); - Clock::advance(Seconds(5)); + Clock::advance( + mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Ensure another authentication attempt was made. AWAIT_READY(authenticationCompletedMessage); @@ -510,6 +543,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessage) // is lost. TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -521,6 +556,12 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get()); ASSERT_SOME(slave); + // Advance the clock to trigger authentication attempt. + // Currently the initial backoff is [0, registration_backoff_factor] + // for agents both with or without authentication. See MESOS-9173. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + AWAIT_READY(authenticationStepMessage); Future<AuthenticationCompletedMessage> authenticationCompletedMessage = @@ -530,10 +571,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave) FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); // Advance the clock for the slave to retry. - Clock::pause(); - Clock::advance(Seconds(5)); + Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Ensure another authentication attempt was made. AWAIT_READY(authenticationCompletedMessage); @@ -552,6 +591,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave) // eventually register. TEST_F(AuthenticationTest, DropFinalSASLMessage) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -575,10 +616,9 @@ TEST_F(AuthenticationTest, DropFinalSASLMessage) .WillOnce(FutureSatisfy(®istered)); // Advance the clock for the scheduler to retry. - Clock::pause(); - Clock::advance(Seconds(5)); + Clock::advance( + mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Ensure another authentication attempt was made. AWAIT_READY(authenticationCompletedMessage); @@ -599,6 +639,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessage) // eventually register. TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -610,6 +652,12 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave) Try<Owned<cluster::Slave>> slave = StartSlave(detector.get()); ASSERT_SOME(slave); + // Advance the clock to trigger authentication attempt. + // Currently the initial backoff is [0, registration_backoff_factor] + // for agents both with or without authentication. See MESOS-9173. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + AWAIT_READY(authenticationCompletedMessage); authenticationCompletedMessage = @@ -619,10 +667,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave) FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); // Advance the clock for the scheduler to retry. - Clock::pause(); - Clock::advance(Seconds(5)); + Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); - Clock::resume(); // Ensure another authentication attempt was made. AWAIT_READY(authenticationCompletedMessage); @@ -638,6 +684,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave) // authenticates. TEST_F(AuthenticationTest, MasterFailover) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -680,6 +728,8 @@ TEST_F(AuthenticationTest, MasterFailover) // authenticates. TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -692,6 +742,12 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication) Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags); ASSERT_SOME(slave); + // Advance the clock to trigger authentication attempt. + // Currently the initial backoff is [0, registration_backoff_factor] + // for agents both with or without authentication. See MESOS-9173. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + AWAIT_READY(authenticateMessage); // While the authentication is in progress simulate a failed over @@ -706,6 +762,11 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication) // Appoint a new master and inform the slave about it. detector.appoint(master.get()->pid); + // Advance the clock to trigger authentication and registration attempt. + // The initial registration delay is between [0, registration_backoff_factor]. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + // Slave should be able to get registered. AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); @@ -717,6 +778,8 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication) // detected due to leader election), it is handled properly. TEST_F(AuthenticationTest, LeaderElection) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -754,6 +817,8 @@ TEST_F(AuthenticationTest, LeaderElection) // detected due to leader election), it is handled properly. TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -766,6 +831,12 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication) Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags); ASSERT_SOME(slave); + // Advance the clock to trigger authentication attempt. + // Currently the initial backoff is [0, registration_backoff_factor] + // for agents both with or without authentication. See MESOS-9173. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + // Drop the intermediate SASL message so that authentication fails. AWAIT_READY(authenticationStepMessage); @@ -775,6 +846,11 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication) // Appoint a new master and inform the slave about it. detector.appoint(master.get()->pid); + // Advance the clock to trigger authentication and registration attempt. + // The initial registration delay is between [0, registration_backoff_factor]. + Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR); + Clock::settle(); + // Slave should be able to get registered. AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); @@ -786,6 +862,8 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication) // with the master when it comes back up. TEST_F(AuthenticationTest, SchedulerFailover) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); @@ -853,6 +931,8 @@ TEST_F(AuthenticationTest, SchedulerFailover) // authentication. TEST_F(AuthenticationTest, RejectedSchedulerFailover) { + Clock::pause(); + Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master);
