[
https://issues.apache.org/jira/browse/MESOS-6724?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16008142#comment-16008142
]
Alexander Rukletsov commented on MESOS-6724:
--------------------------------------------
Observed it again on Apache CI for 1.1.2 release.
{noformat}
[ RUN ] HTTPCommandExecutorTest.TerminateWithACK
I0504 15:43:05.341382 32064 cluster.cpp:158] Creating default 'local' authorizer
I0504 15:43:05.345090 32064 leveldb.cpp:174] Opened db in 3.444533ms
I0504 15:43:05.345728 32064 leveldb.cpp:181] Compacted db in 603462ns
I0504 15:43:05.345772 32064 leveldb.cpp:196] Created db iterator in 16838ns
I0504 15:43:05.345788 32064 leveldb.cpp:202] Seeked to beginning of db in 1987ns
I0504 15:43:05.345799 32064 leveldb.cpp:271] Iterated through 0 keys in the db
in 269ns
I0504 15:43:05.345834 32064 replica.cpp:776] Replica recovered with log
positions 0 -> 0 with 1 holes and 0 unlearned
I0504 15:43:05.346590 32091 recover.cpp:451] Starting replica recovery
I0504 15:43:05.346793 32091 recover.cpp:477] Replica is in EMPTY status
I0504 15:43:05.347823 32098 replica.cpp:673] Replica in EMPTY status received a
broadcasted recover request from __req_res__(168)@172.17.0.3:41866
I0504 15:43:05.348352 32090 recover.cpp:197] Received a recover response from a
replica in EMPTY status
I0504 15:43:05.348784 32098 recover.cpp:568] Updating replica status to STARTING
I0504 15:43:05.349874 32095 leveldb.cpp:304] Persisting metadata (8 bytes) to
leveldb took 840720ns
I0504 15:43:05.349900 32095 replica.cpp:320] Persisted replica status to
STARTING
I0504 15:43:05.350070 32088 recover.cpp:477] Replica is in STARTING status
I0504 15:43:05.350971 32102 master.cpp:380] Master
2075640b-b7dc-44f0-89b5-b0f9af99be7e (41c61dc99119) started on 172.17.0.3:41866
I0504 15:43:05.351112 32088 replica.cpp:673] Replica in STARTING status
received a broadcasted recover request from __req_res__(169)@172.17.0.3:41866
I0504 15:43:05.350991 32102 master.cpp:382] Flags at startup: --acls=""
--agent_ping_timeout="15secs" --agent_reregister_timeout="10mins"
--allocation_interval="1secs" --allocator="HierarchicalDRF"
--authenticate_agents="true" --authenticate_frameworks="true"
--authenticate_http_frameworks="true" --authenticate_http_readonly="true"
--authenticate_http_readwrite="true" --authenticators="crammd5"
--authorizers="local" --credentials="/tmp/t7Ea9P/credentials"
--framework_sorter="drf" --help="false" --hostname_lookup="true"
--http_authenticators="basic" --http_framework_authenticators="basic"
--initialize_driver_logging="true" --log_auto_initialize="true"
--logbufsecs="0" --logging_level="INFO" --max_agent_ping_timeouts="5"
--max_completed_frameworks="50" --max_completed_tasks_per_framework="1000"
--quiet="false" --recovery_agent_removal_limit="100%"
--registry="replicated_log" --registry_fetch_timeout="1mins"
--registry_gc_interval="15mins" --registry_max_agent_age="2weeks"
--registry_max_agent_count="102400" --registry_store_timeout="100secs"
--registry_strict="false" --root_submissions="true" --user_sorter="drf"
--version="false" --webui_dir="/mesos/mesos-1.1.2/_inst/share/mesos/webui"
--work_dir="/tmp/t7Ea9P/master" --zk_session_timeout="10secs"
I0504 15:43:05.351322 32102 master.cpp:432] Master only allowing authenticated
frameworks to register
I0504 15:43:05.351335 32102 master.cpp:446] Master only allowing authenticated
agents to register
I0504 15:43:05.351341 32102 master.cpp:459] Master only allowing authenticated
HTTP frameworks to register
I0504 15:43:05.351348 32102 credentials.hpp:37] Loading credentials for
authentication from '/tmp/t7Ea9P/credentials'
I0504 15:43:05.351394 32094 recover.cpp:197] Received a recover response from a
replica in STARTING status
I0504 15:43:05.351594 32102 master.cpp:504] Using default 'crammd5'
authenticator
I0504 15:43:05.351850 32102 http.cpp:887] Using default 'basic' HTTP
authenticator for realm 'mesos-master-readonly'
I0504 15:43:05.352252 32102 http.cpp:887] Using default 'basic' HTTP
authenticator for realm 'mesos-master-readwrite'
I0504 15:43:05.352270 32090 recover.cpp:568] Updating replica status to VOTING
I0504 15:43:05.352500 32102 http.cpp:887] Using default 'basic' HTTP
authenticator for realm 'mesos-master-scheduler'
I0504 15:43:05.352635 32092 leveldb.cpp:304] Persisting metadata (8 bytes) to
leveldb took 225076ns
I0504 15:43:05.352660 32092 replica.cpp:320] Persisted replica status to VOTING
I0504 15:43:05.352707 32102 master.cpp:584] Authorization enabled
I0504 15:43:05.352778 32087 recover.cpp:582] Successfully joined the Paxos group
I0504 15:43:05.352880 32091 hierarchical.cpp:149] Initialized hierarchical
allocator process
I0504 15:43:05.352883 32089 whitelist_watcher.cpp:77] No whitelist given
I0504 15:43:05.353144 32087 recover.cpp:466] Recover process terminated
I0504 15:43:05.355403 32085 master.cpp:2017] Elected as the leading master!
I0504 15:43:05.355437 32085 master.cpp:1560] Recovering from registrar
I0504 15:43:05.355631 32097 registrar.cpp:329] Recovering registrar
I0504 15:43:05.356300 32094 log.cpp:553] Attempting to start the writer
I0504 15:43:05.357868 32094 replica.cpp:493] Replica received implicit promise
request from __req_res__(170)@172.17.0.3:41866 with proposal 1
I0504 15:43:05.358140 32094 leveldb.cpp:304] Persisting metadata (8 bytes) to
leveldb took 238736ns
I0504 15:43:05.358163 32094 replica.cpp:342] Persisted promised to 1
I0504 15:43:05.358783 32106 coordinator.cpp:238] Coordinator attempting to fill
missing positions
I0504 15:43:05.360201 32094 replica.cpp:388] Replica received explicit promise
request from __req_res__(171)@172.17.0.3:41866 for position 0 with proposal 2
I0504 15:43:05.360452 32094 leveldb.cpp:341] Persisting action (8 bytes) to
leveldb took 209201ns
I0504 15:43:05.360476 32094 replica.cpp:708] Persisted action NOP at position 0
I0504 15:43:05.361809 32091 replica.cpp:537] Replica received write request for
position 0 from __req_res__(172)@172.17.0.3:41866
I0504 15:43:05.361876 32091 leveldb.cpp:436] Reading position from leveldb took
29524ns
I0504 15:43:05.362116 32091 leveldb.cpp:341] Persisting action (14 bytes) to
leveldb took 194035ns
I0504 15:43:05.362143 32091 replica.cpp:708] Persisted action NOP at position 0
I0504 15:43:05.362979 32103 replica.cpp:691] Replica received learned notice
for position 0 from @0.0.0.0:0
I0504 15:43:05.363215 32103 leveldb.cpp:341] Persisting action (16 bytes) to
leveldb took 207499ns
I0504 15:43:05.363240 32103 replica.cpp:708] Persisted action NOP at position 0
I0504 15:43:05.363785 32105 log.cpp:569] Writer started with ending position 0
I0504 15:43:05.365002 32089 leveldb.cpp:436] Reading position from leveldb took
26769ns
I0504 15:43:05.365936 32094 registrar.cpp:362] Successfully fetched the
registry (0B) in 10.246912ms
I0504 15:43:05.366044 32094 registrar.cpp:461] Applied 1 operations in 19811ns;
attempting to update the registry
I0504 15:43:05.366716 32092 log.cpp:577] Attempting to append 168 bytes to the
log
I0504 15:43:05.366830 32087 coordinator.cpp:348] Coordinator attempting to
write APPEND action at position 1
I0504 15:43:05.367691 32095 replica.cpp:537] Replica received write request for
position 1 from __req_res__(173)@172.17.0.3:41866
I0504 15:43:05.375273 32095 leveldb.cpp:341] Persisting action (187 bytes) to
leveldb took 7.546417ms
I0504 15:43:05.375305 32095 replica.cpp:708] Persisted action APPEND at
position 1
I0504 15:43:05.376070 32089 replica.cpp:691] Replica received learned notice
for position 1 from @0.0.0.0:0
I0504 15:43:05.376334 32089 leveldb.cpp:341] Persisting action (189 bytes) to
leveldb took 223609ns
I0504 15:43:05.376359 32089 replica.cpp:708] Persisted action APPEND at
position 1
I0504 15:43:05.377385 32093 registrar.cpp:506] Successfully updated the
registry in 11.236864ms
I0504 15:43:05.377521 32093 registrar.cpp:392] Successfully recovered registrar
I0504 15:43:05.377568 32098 log.cpp:596] Attempting to truncate the log to 1
I0504 15:43:05.377693 32085 coordinator.cpp:348] Coordinator attempting to
write TRUNCATE action at position 2
I0504 15:43:05.378213 32105 master.cpp:1676] Recovered 0 agents from the
registry (129B); allowing 10mins for agents to re-register
I0504 15:43:05.378229 32100 hierarchical.cpp:176] Skipping recovery of
hierarchical allocator: nothing to recover
I0504 15:43:05.378677 32086 replica.cpp:537] Replica received write request for
position 2 from __req_res__(174)@172.17.0.3:41866
I0504 15:43:05.378937 32086 leveldb.cpp:341] Persisting action (16 bytes) to
leveldb took 223698ns
I0504 15:43:05.378960 32086 replica.cpp:708] Persisted action TRUNCATE at
position 2
I0504 15:43:05.379722 32102 replica.cpp:691] Replica received learned notice
for position 2 from @0.0.0.0:0
I0504 15:43:05.383610 32102 leveldb.cpp:341] Persisting action (18 bytes) to
leveldb took 3.858422ms
I0504 15:43:05.383677 32102 leveldb.cpp:399] Deleting ~1 keys from leveldb took
35851ns
I0504 15:43:05.383697 32102 replica.cpp:708] Persisted action TRUNCATE at
position 2
I0504 15:43:05.386698 32064 containerizer.cpp:200] Using isolation:
posix/cpu,posix/mem,filesystem/posix,network/cni
W0504 15:43:05.387332 32064 backend.cpp:76] Failed to create 'aufs' backend:
AufsBackend requires root privileges, but is running as user mesos
W0504 15:43:05.387490 32064 backend.cpp:76] Failed to create 'bind' backend:
BindBackend requires root privileges
I0504 15:43:05.391284 32096 slave.cpp:208] Mesos agent started on
72.17.0.3:41866
I0504 15:43:05.391306 32096 slave.cpp:209] Flags at startup: --acls=""
--appc_simple_discovery_uri_prefix="http://";
--appc_store_dir="/tmp/mesos/store/appc" --authenticate_http_readonly="true"
--authenticate_http_readwrite="true" --authenticatee="crammd5"
--authentication_backoff_factor="1secs" --authorizer="local"
--cgroups_cpu_enable_pids_and_tids_count="false" --cgroups_enable_cfs="false"
--cgroups_hierarchy="/sys/fs/cgroup" --cgroups_limit_swap="false"
--cgroups_root="mesos" --container_disk_watch_interval="15secs"
--containerizers="mesos"
--credential="/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/credential"
--default_role="*" --disk_watch_interval="1mins" --docker="docker"
--docker_kill_orphans="true" --docker_registry="https://registry-1.docker.io";
--docker_remove_delay="6hrs" --docker_socket="/var/run/docker.sock"
--docker_stop_timeout="0ns" --docker_store_dir="/tmp/mesos/store/docker"
--docker_volume_checkpoint_dir="/var/run/mesos/isolators/docker/volume"
--enforce_container_disk_quota="false" --executor_registration_timeout="1mins"
--executor_shutdown_grace_period="5secs"
--fetcher_cache_dir="/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/fetch"
--fetcher_cache_size="2GB" --frameworks_home="" --gc_delay="1weeks"
--gc_disk_headroom="0.1" --hadoop_home="" --help="false"
--hostname_lookup="true" --http_authenticators="basic"
--http_command_executor="true"
--http_credentials="/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/http_credentials"
--image_provisioner_backend="copy" --initialize_driver_logging="true"
--isolation="posix/cpu,posix/mem" --launcher="posix"
--launcher_dir="/mesos/mesos-1.1.2/_build/src" --logbufsecs="0"
--logging_level="INFO" --max_completed_executors_per_framework="150"
--oversubscribed_resources_interval="15secs" --perf_duration="10secs"
--perf_interval="1mins" --qos_correction_interval_min="0ns" --quiet="false"
--recover="reconnect" --recovery_timeout="15mins"
--registration_backoff_factor="10ms"
--resources="cpus:2;gpus:0;mem:1024;disk:1024;ports:[31000-32000]"
--revocable_cpu_low_priority="true"
--runtime_dir="/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8"
--sandbox_directory="/mnt/mesos/sandbox" --strict="true" --switch_user="true"
--systemd_enable_support="true"
--systemd_runtime_directory="/run/systemd/system" --version="false"
--work_dir="/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu"
I0504 15:43:05.391834 32096 credentials.hpp:86] Loading credential for
authentication from
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/credential'
I0504 15:43:05.391991 32096 slave.cpp:346] Agent using credential for:
test-principal
I0504 15:43:05.392012 32096 credentials.hpp:37] Loading credentials for
authentication from
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/http_credentials'
I0504 15:43:05.392226 32096 http.cpp:887] Using default 'basic' HTTP
authenticator for realm 'mesos-agent-readonly'
I0504 15:43:05.392369 32096 http.cpp:887] Using default 'basic' HTTP
authenticator for realm 'mesos-agent-readwrite'
I0504 15:43:05.392388 32064 sched.cpp:226] Version: 1.1.2
I0504 15:43:05.392886 32098 sched.cpp:330] New master detected at
master@172.17.0.3:41866
I0504 15:43:05.392966 32098 sched.cpp:396] Authenticating with master
master@172.17.0.3:41866
I0504 15:43:05.392990 32098 sched.cpp:403] Using default CRAM-MD5 authenticatee
I0504 15:43:05.393358 32095 authenticatee.cpp:121] Creating new client SASL
connection
I0504 15:43:05.393451 32096 slave.cpp:533] Agent resources: cpus(*):2;
mem(*):1024; disk(*):1024; ports(*):[31000-32000]
I0504 15:43:05.393534 32096 slave.cpp:541] Agent attributes: [ ]
I0504 15:43:05.393548 32096 slave.cpp:546] Agent hostname: 41c61dc99119
I0504 15:43:05.393612 32086 master.cpp:6680] Authenticating
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:05.393725 32102 authenticator.cpp:414] Starting authentication
session for crammd5-authenticatee(32)@172.17.0.3:41866
I0504 15:43:05.394050 32089 authenticator.cpp:98] Creating new server SASL
connection
I0504 15:43:05.394294 32101 authenticatee.cpp:213] Received SASL authentication
mechanisms: CRAM-MD5
I0504 15:43:05.394325 32101 authenticatee.cpp:239] Attempting to authenticate
with mechanism 'CRAM-MD5'
I0504 15:43:05.394428 32088 authenticator.cpp:204] Received SASL authentication
start
I0504 15:43:05.394482 32088 authenticator.cpp:326] Authentication requires more
steps
I0504 15:43:05.394671 32083 authenticatee.cpp:259] Received SASL authentication
step
I0504 15:43:05.394803 32101 authenticator.cpp:232] Received SASL authentication
step
I0504 15:43:05.394848 32101 auxprop.cpp:109] Request to lookup properties for
user: 'test-principal' realm: '41c61dc99119' server FQDN: '41c61dc99119'
SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
SASL_AUXPROP_AUTHZID: false
I0504 15:43:05.394871 32101 auxprop.cpp:181] Looking up auxiliary property
'*userPassword'
I0504 15:43:05.394943 32101 auxprop.cpp:181] Looking up auxiliary property
'*cmusaslsecretCRAM-MD5'
I0504 15:43:05.394985 32101 auxprop.cpp:109] Request to lookup properties for
user: 'test-principal' realm: '41c61dc99119' server FQDN: '41c61dc99119'
SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
SASL_AUXPROP_AUTHZID: true
I0504 15:43:05.395009 32101 auxprop.cpp:131] Skipping auxiliary property
'*userPassword' since SASL_AUXPROP_AUTHZID == true
I0504 15:43:05.395025 32101 auxprop.cpp:131] Skipping auxiliary property
'*cmusaslsecretCRAM-MD5' since SASL_AUXPROP_AUTHZID == true
I0504 15:43:05.395052 32101 authenticator.cpp:318] Authentication success
I0504 15:43:05.395088 32084 state.cpp:57] Recovering state from
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/meta'
I0504 15:43:05.395241 32102 master.cpp:6710] Successfully authenticated
principal 'test-principal' at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:05.395313 32097 authenticator.cpp:432] Authentication session
cleanup for crammd5-authenticatee(32)@172.17.0.3:41866
I0504 15:43:05.395408 32098 authenticatee.cpp:299] Authentication success
I0504 15:43:05.395480 32103 status_update_manager.cpp:203] Recovering status
update manager
I0504 15:43:05.395692 32087 containerizer.cpp:555] Recovering containerizer
I0504 15:43:05.395896 32090 sched.cpp:502] Successfully authenticated with
master master@172.17.0.3:41866
I0504 15:43:05.395917 32090 sched.cpp:820] Sending SUBSCRIBE call to
master@172.17.0.3:41866
I0504 15:43:05.396005 32090 sched.cpp:853] Will retry registration in
1.712040262secs if necessary
I0504 15:43:05.396121 32102 master.cpp:2596] Received SUBSCRIBE call for
framework 'default' at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:05.396178 32102 master.cpp:2053] Authorizing framework principal
'test-principal' to receive offers for role '*'
I0504 15:43:05.396824 32088 master.cpp:2672] Subscribing framework default with
checkpointing disabled and capabilities [ ]
I0504 15:43:05.397313 32098 provisioner.cpp:253] Provisioner recovery complete
I0504 15:43:05.397626 32096 sched.cpp:743] Framework registered with
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.397678 32096 sched.cpp:757] Scheduler::registered took 26397ns
I0504 15:43:05.397708 32104 hierarchical.cpp:275] Added framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.397784 32104 hierarchical.cpp:1694] No allocations performed
I0504 15:43:05.397799 32100 slave.cpp:5281] Finished recovery
I0504 15:43:05.397812 32104 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:05.397878 32104 hierarchical.cpp:1286] Performed allocation for 0
agents in 140315ns
I0504 15:43:05.398274 32100 slave.cpp:5455] Querying resource estimator for
oversubscribable resources
I0504 15:43:05.398495 32086 status_update_manager.cpp:177] Pausing sending
status updates
I0504 15:43:05.398500 32100 slave.cpp:915] New master detected at
master@172.17.0.3:41866
I0504 15:43:05.398524 32100 slave.cpp:974] Authenticating with master
master@172.17.0.3:41866
I0504 15:43:05.398576 32100 slave.cpp:985] Using default CRAM-MD5 authenticatee
I0504 15:43:05.398720 32100 slave.cpp:947] Detecting new master
I0504 15:43:05.398767 32083 authenticatee.cpp:121] Creating new client SASL
connection
I0504 15:43:05.399022 32091 master.cpp:6680] Authenticating (1)@172.17.0.3:41866
I0504 15:43:05.399214 32097 authenticator.cpp:414] Starting authentication
session for crammd5-authenticatee(33)@172.17.0.3:41866
I0504 15:43:05.399416 32087 authenticator.cpp:98] Creating new server SASL
connection
I0504 15:43:05.399605 32095 authenticatee.cpp:213] Received SASL authentication
mechanisms: CRAM-MD5
I0504 15:43:05.399631 32095 authenticatee.cpp:239] Attempting to authenticate
with mechanism 'CRAM-MD5'
I0504 15:43:05.399798 32087 authenticator.cpp:204] Received SASL authentication
start
I0504 15:43:05.399850 32087 authenticator.cpp:326] Authentication requires more
steps
I0504 15:43:05.399952 32090 authenticatee.cpp:259] Received SASL authentication
step
I0504 15:43:05.400058 32095 authenticator.cpp:232] Received SASL authentication
step
I0504 15:43:05.400089 32095 auxprop.cpp:109] Request to lookup properties for
user: 'test-principal' realm: '41c61dc99119' server FQDN: '41c61dc99119'
SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
SASL_AUXPROP_AUTHZID: false
I0504 15:43:05.400108 32095 auxprop.cpp:181] Looking up auxiliary property
'*userPassword'
I0504 15:43:05.400138 32095 auxprop.cpp:181] Looking up auxiliary property
'*cmusaslsecretCRAM-MD5'
I0504 15:43:05.400166 32095 auxprop.cpp:109] Request to lookup properties for
user: 'test-principal' realm: '41c61dc99119' server FQDN: '41c61dc99119'
SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
SASL_AUXPROP_AUTHZID: true
I0504 15:43:05.400176 32095 auxprop.cpp:131] Skipping auxiliary property
'*userPassword' since SASL_AUXPROP_AUTHZID == true
I0504 15:43:05.400184 32095 auxprop.cpp:131] Skipping auxiliary property
'*cmusaslsecretCRAM-MD5' since SASL_AUXPROP_AUTHZID == true
I0504 15:43:05.400198 32095 authenticator.cpp:318] Authentication success
I0504 15:43:05.400274 32090 authenticatee.cpp:299] Authentication success
I0504 15:43:05.400344 32102 master.cpp:6710] Successfully authenticated
principal 'test-principal' at (1)@172.17.0.3:41866
I0504 15:43:05.400382 32104 authenticator.cpp:432] Authentication session
cleanup for crammd5-authenticatee(33)@172.17.0.3:41866
I0504 15:43:05.400687 32089 slave.cpp:1069] Successfully authenticated with
master master@172.17.0.3:41866
I0504 15:43:05.400862 32089 slave.cpp:1475] Will retry registration in
4.615564ms if necessary
I0504 15:43:05.401051 32088 master.cpp:5103] Registering agent at
(1)@172.17.0.3:41866 (41c61dc99119) with id
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0
I0504 15:43:05.401487 32083 registrar.cpp:461] Applied 1 operations in 49606ns;
attempting to update the registry
I0504 15:43:05.403597 32097 log.cpp:577] Attempting to append 337 bytes to the
log
I0504 15:43:05.403753 32085 coordinator.cpp:348] Coordinator attempting to
write APPEND action at position 3
I0504 15:43:05.404583 32086 replica.cpp:537] Replica received write request for
position 3 from __req_res__(175)@172.17.0.3:41866
I0504 15:43:05.404911 32086 leveldb.cpp:341] Persisting action (356 bytes) to
leveldb took 269746ns
I0504 15:43:05.404937 32086 replica.cpp:708] Persisted action APPEND at
position 3
I0504 15:43:05.405844 32106 replica.cpp:691] Replica received learned notice
for position 3 from @0.0.0.0:0
I0504 15:43:05.406158 32106 leveldb.cpp:341] Persisting action (358 bytes) to
leveldb took 282857ns
I0504 15:43:05.406189 32106 replica.cpp:708] Persisted action APPEND at
position 3
I0504 15:43:05.406263 32089 slave.cpp:1475] Will retry registration in
13.993213ms if necessary
I0504 15:43:05.406363 32093 master.cpp:5091] Ignoring register agent message
from (1)@172.17.0.3:41866 (41c61dc99119) as admission is already in progress
I0504 15:43:05.408088 32099 registrar.cpp:506] Successfully updated the
registry in 6.545152ms
I0504 15:43:05.408306 32093 log.cpp:596] Attempting to truncate the log to 3
I0504 15:43:05.408526 32099 coordinator.cpp:348] Coordinator attempting to
write TRUNCATE action at position 4
I0504 15:43:05.408797 32088 slave.cpp:4133] Received ping from
slave-observer(8)@172.17.0.3:41866
I0504 15:43:05.408907 32098 master.cpp:5174] Registered agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 at (1)@172.17.0.3:41866 (41c61dc99119)
with cpus(*):2; mem(*):1024; disk(*):1024; ports(*):[31000-32000]
I0504 15:43:05.409015 32088 slave.cpp:1115] Registered with master
master@172.17.0.3:41866; given agent ID 2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0
I0504 15:43:05.409046 32088 fetcher.cpp:90] Clearing fetcher cache
I0504 15:43:05.409303 32084 hierarchical.cpp:485] Added agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 (41c61dc99119) with cpus(*):2;
mem(*):1024; disk(*):1024; ports(*):[31000-32000] (allocated: {})
I0504 15:43:05.409330 32098 status_update_manager.cpp:184] Resuming sending
status updates
I0504 15:43:05.409369 32099 replica.cpp:537] Replica received write request for
position 4 from __req_res__(176)@172.17.0.3:41866
I0504 15:43:05.409466 32088 slave.cpp:1138] Checkpointing SlaveInfo to
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/meta/slaves/2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0/slave.info'
I0504 15:43:05.409611 32099 leveldb.cpp:341] Persisting action (16 bytes) to
leveldb took 207020ns
I0504 15:43:05.409636 32099 replica.cpp:708] Persisted action TRUNCATE at
position 4
I0504 15:43:05.410361 32100 replica.cpp:691] Replica received learned notice
for position 4 from @0.0.0.0:0
I0504 15:43:05.410599 32100 leveldb.cpp:341] Persisting action (18 bytes) to
leveldb took 211227ns
I0504 15:43:05.410648 32084 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:05.410667 32100 leveldb.cpp:399] Deleting ~2 keys from leveldb took
40031ns
I0504 15:43:05.410691 32100 replica.cpp:708] Persisted action TRUNCATE at
position 4
I0504 15:43:05.410740 32084 hierarchical.cpp:1309] Performed allocation for
agent 2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 in 1.395222ms
I0504 15:43:05.411295 32104 master.cpp:6509] Sending 1 offers to framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:05.411969 32099 sched.cpp:917] Scheduler::resourceOffers took
122658ns
I0504 15:43:05.414469 32102 master.cpp:3554] Processing ACCEPT call for offers:
[ 2075640b-b7dc-44f0-89b5-b0f9af99be7e-O0 ] on agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 at (1)@172.17.0.3:41866 (41c61dc99119)
for framework 2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:05.414572 32102 master.cpp:3159] Authorizing framework principal
'test-principal' to launch task 600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a
I0504 15:43:05.417814 32091 master.cpp:8262] Adding task
600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a with resources cpus(*):2; mem(*):1024;
disk(*):1024; ports(*):[31000-32000] on agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 (41c61dc99119)
I0504 15:43:05.418177 32091 master.cpp:4192] Launching task
600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866 with resources
cpus(*):2; mem(*):1024; disk(*):1024; ports(*):[31000-32000] on agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 at (1)@172.17.0.3:41866 (41c61dc99119)
I0504 15:43:05.418993 32099 slave.cpp:1539] Got assigned task
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' for framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.420351 32099 slave.cpp:1701] Launching task
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' for framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.421084 32099 paths.cpp:536] Trying to chown
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/slaves/2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0/frameworks/2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000/executors/600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a/runs/0817f9bd-8e43-4b6f-b6b3-88805365c80c'
to user 'mesos'
I0504 15:43:05.438580 32099 slave.cpp:6179] Launching executor
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 with resources cpus(*):0.1; mem(*):32
in work directory
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/slaves/2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0/frameworks/2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000/executors/600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a/runs/0817f9bd-8e43-4b6f-b6b3-88805365c80c'
I0504 15:43:05.439748 32097 containerizer.cpp:938] Starting container
0817f9bd-8e43-4b6f-b6b3-88805365c80c for executor
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.440160 32099 slave.cpp:1987] Queued task
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' for executor
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:05.440285 32099 slave.cpp:868] Successfully attached file
'/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/slaves/2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0/frameworks/2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000/executors/600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a/runs/0817f9bd-8e43-4b6f-b6b3-88805365c80c'
I0504 15:43:05.447082 32097 containerizer.cpp:1463] Launching
'mesos-containerizer' with flags
'--command="{"arguments":["mesos-executor","--launcher_dir=\/mesos\/mesos-1.1.2\/_build\/src"],"shell":false,"value":"\/mesos\/mesos-1.1.2\/_build\/src\/mesos-executor"}"
--help="false" --pipe_read="9" --pipe_write="12" --pre_exec_commands="[]"
--runtime_directory="/tmp/HTTPCommandExecutorTest_TerminateWithACK_3LgMk8/containers/0817f9bd-8e43-4b6f-b6b3-88805365c80c"
--unshare_namespace_mnt="false" --user="mesos"
--working_directory="/tmp/HTTPCommandExecutorTest_TerminateWithACK_BRTXeu/slaves/2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0/frameworks/2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000/executors/600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a/runs/0817f9bd-8e43-4b6f-b6b3-88805365c80c"'
I0504 15:43:05.449122 32097 launcher.cpp:127] Forked child with pid '32121' for
container '0817f9bd-8e43-4b6f-b6b3-88805365c80c'
I0504 15:43:06.354243 32087 hierarchical.cpp:1694] No allocations performed
I0504 15:43:06.354343 32087 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:06.354439 32087 hierarchical.cpp:1286] Performed allocation for 1
agents in 488945ns
I0504 15:43:07.355734 32101 hierarchical.cpp:1694] No allocations performed
I0504 15:43:07.355855 32101 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:07.355985 32101 hierarchical.cpp:1286] Performed allocation for 1
agents in 567052ns
I0504 15:43:08.356890 32100 hierarchical.cpp:1694] No allocations performed
I0504 15:43:08.356999 32100 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:08.357092 32100 hierarchical.cpp:1286] Performed allocation for 1
agents in 462937ns
I0504 15:43:09.358674 32088 hierarchical.cpp:1694] No allocations performed
I0504 15:43:09.358762 32088 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:09.358856 32088 hierarchical.cpp:1286] Performed allocation for 1
agents in 460517ns
I0504 15:43:10.360016 32105 hierarchical.cpp:1694] No allocations performed
I0504 15:43:10.360115 32105 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:10.360208 32105 hierarchical.cpp:1286] Performed allocation for 1
agents in 468579ns
I0504 15:43:11.361495 32092 hierarchical.cpp:1694] No allocations performed
I0504 15:43:11.361596 32092 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:11.361706 32092 hierarchical.cpp:1286] Performed allocation for 1
agents in 456795ns
I0504 15:43:12.362313 32089 hierarchical.cpp:1694] No allocations performed
I0504 15:43:12.362409 32089 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:12.362490 32089 hierarchical.cpp:1286] Performed allocation for 1
agents in 421777ns
I0504 15:43:13.363725 32090 hierarchical.cpp:1694] No allocations performed
I0504 15:43:13.363808 32090 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:13.363903 32090 hierarchical.cpp:1286] Performed allocation for 1
agents in 446183ns
I0504 15:43:14.365000 32104 hierarchical.cpp:1694] No allocations performed
I0504 15:43:14.365072 32104 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:14.365156 32104 hierarchical.cpp:1286] Performed allocation for 1
agents in 349292ns
I0504 15:43:15.366421 32103 hierarchical.cpp:1694] No allocations performed
I0504 15:43:15.366508 32103 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:15.366601 32103 hierarchical.cpp:1286] Performed allocation for 1
agents in 396177ns
I0504 15:43:16.367655 32085 hierarchical.cpp:1694] No allocations performed
I0504 15:43:16.367746 32085 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:16.367839 32085 hierarchical.cpp:1286] Performed allocation for 1
agents in 442656ns
I0504 15:43:17.369190 32103 hierarchical.cpp:1694] No allocations performed
I0504 15:43:17.369278 32103 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:17.369369 32103 hierarchical.cpp:1286] Performed allocation for 1
agents in 409472ns
I0504 15:43:18.370553 32096 hierarchical.cpp:1694] No allocations performed
I0504 15:43:18.370645 32096 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:18.370726 32096 hierarchical.cpp:1286] Performed allocation for 1
agents in 369732ns
I0504 15:43:19.371739 32087 hierarchical.cpp:1694] No allocations performed
I0504 15:43:19.371831 32087 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:19.371904 32087 hierarchical.cpp:1286] Performed allocation for 1
agents in 373744ns
I0504 15:43:20.373103 32096 hierarchical.cpp:1694] No allocations performed
I0504 15:43:20.373210 32096 hierarchical.cpp:1789] No inverse offers to send
out!
I0504 15:43:20.373299 32096 hierarchical.cpp:1286] Performed allocation for 1
agents in 410026ns
I0504 15:43:20.409860 32088 slave.cpp:4133] Received ping from
slave-observer(8)@172.17.0.3:41866
../../src/tests/command_executor_tests.cpp:280: Failure
Failed to wait 15secs for statusRunning
I0504 15:43:20.414811 32086 master.cpp:1297] Framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866 disconnected
I0504 15:43:20.414896 32086 master.cpp:2904] Disconnecting framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:20.414968 32086 master.cpp:2928] Deactivating framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:20.415062 32086 master.cpp:1310] Giving framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866 0ns to failover
I0504 15:43:20.415182 32087 hierarchical.cpp:386] Deactivated framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
../../src/tests/command_executor_tests.cpp:269: Failure
Actual function call count doesn't match EXPECT_CALL(sched, statusUpdate(_,
_))...
Expected: to be called twice
Actual: never called - unsatisfied and active
I0504 15:43:20.415565 32085 master.cpp:6361] Framework failover timeout,
removing framework 2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:20.415604 32085 master.cpp:7095] Removing framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (default) at
scheduler-16d6a594-fabf-46b8-8d65-3d174b8e2daa@172.17.0.3:41866
I0504 15:43:20.415940 32104 slave.cpp:2511] Asked to shut down framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 by master@172.17.0.3:41866
I0504 15:43:20.415990 32104 slave.cpp:2536] Shutting down framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
I0504 15:43:20.416119 32085 master.cpp:7640] Updating the state of task
600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 (latest state: TASK_KILLED, status
update state: TASK_KILLED)
I0504 15:43:20.416194 32104 slave.cpp:4860] Shutting down executor
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
W0504 15:43:20.416224 32104 slave.hpp:863] Unable to send event to executor
'600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a' of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000: unknown connection type
../../src/tests/command_executor_tests.cpp:274: Failure
Actual function call count doesn't match EXPECT_CALL(slave,
executorTerminated(_, _, _))...
Expected: to be called once
Actual: never called - unsatisfied and active
I0504 15:43:20.417469 32085 master.cpp:7736] Removing task
600b509d-caf9-4c9e-a7ec-d0cb49ec2f7a with resources cpus(*):2; mem(*):1024;
disk(*):1024; ports(*):[31000-32000] of framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000 on agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 at (1)@172.17.0.3:41866 (41c61dc99119)
I0504 15:43:20.417841 32097 hierarchical.cpp:1018] Recovered cpus(*):2;
mem(*):1024; disk(*):1024; ports(*):[31000-32000] (total: cpus(*):2;
mem(*):1024; disk(*):1024; ports(*):[31000-32000], allocated: {}) on agent
2075640b-b7dc-44f0-89b5-b0f9af99be7e-S0 from framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
*** Aborted at 1493912600 (unix time) try "date -d @1493912600" if you are
using GNU date ***
I0504 15:43:20.418335 32096 hierarchical.cpp:337] Removed framework
2075640b-b7dc-44f0-89b5-b0f9af99be7e-0000
PC: @ 0x0 (unknown)
*** SIGSEGV (@0x0) received by PID 32064 (TID 0x7fd8a6934700) from PID 0; stack
trace: ***
@ 0x7fd8b3f2f370 (unknown)
{noformat}
> The test "HTTPCommandExecutorTest.TerminateWithACK" is flaky
> ------------------------------------------------------------
>
> Key: MESOS-6724
> URL: https://issues.apache.org/jira/browse/MESOS-6724
> Project: Mesos
> Issue Type: Bug
> Components: test
> Reporter: Qian Zhang
> Labels: flaky-test, mesosphere
>
> It seems the test "HTTPCommandExecutorTest.TerminateWithACK" may fail when
> the machine is under heavy load (e.g., using the “stress” utility to generate
> load on the machine, like {{stress --cpu 4 --io 4 --timeout 120}}).
> {code}
> I1104 21:43:47.768609 31812 authenticator.cpp:98] Creating new server SASL
> connection
> I1104 21:43:47.768844 31812 authenticatee.cpp:213] Received SASL
> authentication mechanisms: CRAM-MD5
> I1104 21:43:47.768874 31812 authenticatee.cpp:239] Attempting to authenticate
> with mechanism 'CRAM-MD5'
> I1104 21:43:47.768960 31812 authenticator.cpp:204] Received SASL
> authentication start
> I1104 21:43:47.769021 31812 authenticator.cpp:326] Authentication requires
> more steps
> I1104 21:43:47.769300 31812 authenticatee.cpp:259] Received SASL
> authentication step
> I1104 21:43:47.770079 31821 authenticator.cpp:232] Received SASL
> authentication step
> I1104 21:43:47.770108 31817 state.cpp:57] Recovering state from
> '/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/meta'
> I1104 21:43:47.770146 31821 auxprop.cpp:109] Request to lookup properties for
> user: 'test-principal' realm: 'b7fb1902101b' server FQDN: 'b7fb1902101b'
> SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
> SASL_AUXPROP_AUTHZID: false
> I1104 21:43:47.770166 31821 auxprop.cpp:181] Looking up auxiliary property
> '*userPassword'
> I1104 21:43:47.770213 31821 auxprop.cpp:181] Looking up auxiliary property
> '*cmusaslsecretCRAM-MD5'
> I1104 21:43:47.770244 31821 auxprop.cpp:109] Request to lookup properties for
> user: 'test-principal' realm: 'b7fb1902101b' server FQDN: 'b7fb1902101b'
> SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
> SASL_AUXPROP_AUTHZID: true
> I1104 21:43:47.770258 31821 auxprop.cpp:131] Skipping auxiliary property
> '*userPassword' since SASL_AUXPROP_AUTHZID == true
> I1104 21:43:47.770268 31821 auxprop.cpp:131] Skipping auxiliary property
> '*cmusaslsecretCRAM-MD5' since SASL_AUXPROP_AUTHZID == true
> I1104 21:43:47.770290 31821 authenticator.cpp:318] Authentication success
> I1104 21:43:47.770388 31826 authenticatee.cpp:299] Authentication success
> I1104 21:43:47.770429 31823 status_update_manager.cpp:203] Recovering status
> update manager
> I1104 21:43:47.770494 31820 master.cpp:6775] Successfully authenticated
> principal 'test-principal' at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:43:47.770536 31815 authenticator.cpp:432] Authentication session
> cleanup for crammd5-authenticatee(32)@172.17.0.3:50719
> I1104 21:43:47.771021 31814 sched.cpp:502] Successfully authenticated with
> master master@172.17.0.3:50719
> I1104 21:43:47.771066 31814 sched.cpp:820] Sending SUBSCRIBE call to
> master@172.17.0.3:50719
> I1104 21:43:47.771214 31814 sched.cpp:853] Will retry registration in
> 384.083867ms if necessary
> I1104 21:43:47.771524 31827 master.cpp:2612] Received SUBSCRIBE call for
> framework 'default' at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:43:47.771775 31827 master.cpp:2069] Authorizing framework principal
> 'test-principal' to receive offers for role '*'
> I1104 21:43:47.772073 31822 containerizer.cpp:557] Recovering containerizer
> I1104 21:43:47.772267 31827 master.cpp:2688] Subscribing framework default
> with checkpointing disabled and capabilities [ ]
> I1104 21:43:47.772873 31818 sched.cpp:743] Framework registered with
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:47.772953 31824 hierarchical.cpp:275] Added framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:47.773032 31818 sched.cpp:757] Scheduler::registered took 22098ns
> I1104 21:43:47.773084 31824 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:47.773125 31824 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:47.773246 31824 hierarchical.cpp:1286] Performed allocation for 0
> agents in 226006ns
> I1104 21:43:47.773977 31821 provisioner.cpp:253] Provisioner recovery complete
> I1104 21:43:47.774346 31814 slave.cpp:5399] Finished recovery
> I1104 21:43:47.788095 31814 slave.cpp:5573] Querying resource estimator for
> oversubscribable resources
> I1104 21:43:47.788635 31821 status_update_manager.cpp:177] Pausing sending
> status updates
> I1104 21:43:47.788642 31814 slave.cpp:915] New master detected at
> master@172.17.0.3:50719
> I1104 21:43:47.788817 31814 slave.cpp:974] Authenticating with master
> master@172.17.0.3:50719
> I1104 21:43:47.788998 31814 slave.cpp:985] Using default CRAM-MD5
> authenticatee
> I1104 21:43:47.789353 31821 authenticatee.cpp:121] Creating new client SASL
> connection
> I1104 21:43:47.789775 31821 master.cpp:6745] Authenticating
> (1)@172.17.0.3:50719
> I1104 21:43:47.789908 31818 authenticator.cpp:414] Starting authentication
> session for crammd5-authenticatee(33)@172.17.0.3:50719
> I1104 21:43:47.790071 31814 slave.cpp:947] Detecting new master
> I1104 21:43:47.790207 31827 authenticator.cpp:98] Creating new server SASL
> connection
> I1104 21:43:47.790566 31812 authenticatee.cpp:213] Received SASL
> authentication mechanisms: CRAM-MD5
> I1104 21:43:47.790602 31812 authenticatee.cpp:239] Attempting to authenticate
> with mechanism 'CRAM-MD5'
> I1104 21:43:47.790733 31825 authenticator.cpp:204] Received SASL
> authentication start
> I1104 21:43:47.790802 31825 authenticator.cpp:326] Authentication requires
> more steps
> I1104 21:43:47.790940 31827 authenticatee.cpp:259] Received SASL
> authentication step
> I1104 21:43:47.791082 31827 authenticator.cpp:232] Received SASL
> authentication step
> I1104 21:43:47.791121 31827 auxprop.cpp:109] Request to lookup properties for
> user: 'test-principal' realm: 'b7fb1902101b' server FQDN: 'b7fb1902101b'
> SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
> SASL_AUXPROP_AUTHZID: false
> I1104 21:43:47.791146 31827 auxprop.cpp:181] Looking up auxiliary property
> '*userPassword'
> I1104 21:43:47.791210 31827 auxprop.cpp:181] Looking up auxiliary property
> '*cmusaslsecretCRAM-MD5'
> I1104 21:43:47.791240 31827 auxprop.cpp:109] Request to lookup properties for
> user: 'test-principal' realm: 'b7fb1902101b' server FQDN: 'b7fb1902101b'
> SASL_AUXPROP_VERIFY_AGAINST_HASH: false SASL_AUXPROP_OVERRIDE: false
> SASL_AUXPROP_AUTHZID: true
> I1104 21:43:47.791256 31827 auxprop.cpp:131] Skipping auxiliary property
> '*userPassword' since SASL_AUXPROP_AUTHZID == true
> I1104 21:43:47.791270 31827 auxprop.cpp:131] Skipping auxiliary property
> '*cmusaslsecretCRAM-MD5' since SASL_AUXPROP_AUTHZID == true
> I1104 21:43:47.791292 31827 authenticator.cpp:318] Authentication success
> I1104 21:43:47.791380 31825 authenticatee.cpp:299] Authentication success
> I1104 21:43:47.791461 31818 master.cpp:6775] Successfully authenticated
> principal 'test-principal' at (1)@172.17.0.3:50719
> I1104 21:43:47.791535 31817 authenticator.cpp:432] Authentication session
> cleanup for crammd5-authenticatee(33)@172.17.0.3:50719
> I1104 21:43:47.792150 31825 slave.cpp:1069] Successfully authenticated with
> master master@172.17.0.3:50719
> I1104 21:43:47.792346 31825 slave.cpp:1483] Will retry registration in
> 15.654413ms if necessary
> I1104 21:43:47.792618 31827 master.cpp:5154] Registering agent at
> (1)@172.17.0.3:50719 (b7fb1902101b) with id
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0
> I1104 21:43:47.793177 31821 registrar.cpp:461] Applied 1 operations in
> 81809ns; attempting to update the registry
> I1104 21:43:47.794234 31820 log.cpp:577] Attempting to append 337 bytes to
> the log
> I1104 21:43:47.794380 31813 coordinator.cpp:348] Coordinator attempting to
> write APPEND action at position 3
> I1104 21:43:47.795359 31821 replica.cpp:537] Replica received write request
> for position 3 from __req_res__(175)@172.17.0.3:50719
> I1104 21:43:47.809108 31812 slave.cpp:1483] Will retry registration in
> 23.319674ms if necessary
> I1104 21:43:47.809288 31826 master.cpp:5142] Ignoring register agent message
> from (1)@172.17.0.3:50719 (b7fb1902101b) as admission is already in progress
> I1104 21:43:47.834007 31815 slave.cpp:1483] Will retry registration in
> 60.150271ms if necessary
> I1104 21:43:47.834194 31813 master.cpp:5142] Ignoring register agent message
> from (1)@172.17.0.3:50719 (b7fb1902101b) as admission is already in progress
> I1104 21:43:47.870666 31821 leveldb.cpp:341] Persisting action (356 bytes) to
> leveldb took 75.23855ms
> I1104 21:43:47.870769 31821 replica.cpp:708] Persisted action APPEND at
> position 3
> I1104 21:43:47.872097 31820 replica.cpp:691] Replica received learned notice
> for position 3 from @0.0.0.0:0
> I1104 21:43:47.895557 31824 slave.cpp:1483] Will retry registration in
> 144.206303ms if necessary
> I1104 21:43:47.896121 31824 master.cpp:5142] Ignoring register agent message
> from (1)@172.17.0.3:50719 (b7fb1902101b) as admission is already in progress
> I1104 21:43:47.989075 31820 leveldb.cpp:341] Persisting action (358 bytes) to
> leveldb took 116.91704ms
> I1104 21:43:47.989181 31820 replica.cpp:708] Persisted action APPEND at
> position 3
> I1104 21:43:47.992252 31820 registrar.cpp:506] Successfully updated the
> registry in 198.990848ms
> I1104 21:43:47.992590 31820 log.cpp:596] Attempting to truncate the log to 3
> I1104 21:43:47.993695 31820 master.cpp:5225] Registered agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 at (1)@172.17.0.3:50719
> (b7fb1902101b) with cpus(*):2; mem(*):1024; disk(*):1024;
> ports(*):[31000-32000]
> I1104 21:43:47.994426 31817 slave.cpp:1115] Registered with master
> master@172.17.0.3:50719; given agent ID
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0
> I1104 21:43:47.994458 31817 fetcher.cpp:86] Clearing fetcher cache
> I1104 21:43:47.994853 31825 hierarchical.cpp:485] Added agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 (b7fb1902101b) with cpus(*):2;
> mem(*):1024; disk(*):1024; ports(*):[31000-32000] (allocated: {})
> I1104 21:43:47.994938 31813 status_update_manager.cpp:184] Resuming sending
> status updates
> I1104 21:43:47.994881 31817 slave.cpp:1138] Checkpointing SlaveInfo to
> '/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/meta/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/slave.info'
> I1104 21:43:47.995440 31817 slave.cpp:4251] Received ping from
> slave-observer(8)@172.17.0.3:50719
> I1104 21:43:47.996817 31825 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:47.996932 31825 hierarchical.cpp:1309] Performed allocation for
> agent 266cf569-3b26-40cb-be6f-080699ef02a1-S0 in 2.024174ms
> I1104 21:43:47.997050 31820 coordinator.cpp:348] Coordinator attempting to
> write TRUNCATE action at position 4
> I1104 21:43:47.997628 31817 master.cpp:6574] Sending 1 offers to framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:43:47.998433 31817 sched.cpp:917] Scheduler::resourceOffers took
> 135585ns
> I1104 21:43:47.998826 31817 replica.cpp:537] Replica received write request
> for position 4 from __req_res__(176)@172.17.0.3:50719
> I1104 21:43:48.007855 31821 master.cpp:3581] Processing ACCEPT call for
> offers: [ 266cf569-3b26-40cb-be6f-080699ef02a1-O0 ] on agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 at (1)@172.17.0.3:50719
> (b7fb1902101b) for framework 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> (default) at scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:43:48.008427 31821 master.cpp:3173] Authorizing framework principal
> 'test-principal' to launch task 19aea987-3513-4860-8cdf-d298f1c6c207
> I1104 21:43:48.012418 31821 master.cpp:8337] Adding task
> 19aea987-3513-4860-8cdf-d298f1c6c207 with resources cpus(*):2; mem(*):1024;
> disk(*):1024; ports(*):[31000-32000] on agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 (b7fb1902101b)
> I1104 21:43:48.013135 31821 master.cpp:4233] Launching task
> 19aea987-3513-4860-8cdf-d298f1c6c207 of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719 with
> resources cpus(*):2; mem(*):1024; disk(*):1024; ports(*):[31000-32000] on
> agent 266cf569-3b26-40cb-be6f-080699ef02a1-S0 at (1)@172.17.0.3:50719
> (b7fb1902101b)
> I1104 21:43:48.014135 31813 slave.cpp:1547] Got assigned task
> '19aea987-3513-4860-8cdf-d298f1c6c207' for framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:48.015380 31813 slave.cpp:1709] Launching task
> '19aea987-3513-4860-8cdf-d298f1c6c207' for framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:48.051807 31813 paths.cpp:536] Trying to chown
> '/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/frameworks/266cf569-3b26-40cb-be6f-080699ef02a1-0000/executors/19aea987-3513-4860-8cdf-d298f1c6c207/runs/bcf25a3b-79a9-46af-b6b3-f687355a4e82'
> to user 'mesos'
> I1104 21:43:48.071727 31813 slave.cpp:6307] Launching executor
> '19aea987-3513-4860-8cdf-d298f1c6c207' of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 with resources cpus(*):0.1;
> mem(*):32 in work directory
> '/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/frameworks/266cf569-3b26-40cb-be6f-080699ef02a1-0000/executors/19aea987-3513-4860-8cdf-d298f1c6c207/runs/bcf25a3b-79a9-46af-b6b3-f687355a4e82'
> I1104 21:43:48.073207 31817 leveldb.cpp:341] Persisting action (16 bytes) to
> leveldb took 74.308207ms
> I1104 21:43:48.073351 31817 replica.cpp:708] Persisted action TRUNCATE at
> position 4
> I1104 21:43:48.073992 31816 containerizer.cpp:940] Starting container
> bcf25a3b-79a9-46af-b6b3-f687355a4e82 for executor
> '19aea987-3513-4860-8cdf-d298f1c6c207' of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:48.074343 31824 replica.cpp:691] Replica received learned notice
> for position 4 from @0.0.0.0:0
> I1104 21:43:48.074465 31813 slave.cpp:2031] Queued task
> '19aea987-3513-4860-8cdf-d298f1c6c207' for executor
> '19aea987-3513-4860-8cdf-d298f1c6c207' of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:43:48.074615 31813 slave.cpp:868] Successfully attached file
> '/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/frameworks/266cf569-3b26-40cb-be6f-080699ef02a1-0000/executors/19aea987-3513-4860-8cdf-d298f1c6c207/runs/bcf25a3b-79a9-46af-b6b3-f687355a4e82'
> I1104 21:43:48.080924 31822 containerizer.cpp:1469] Launching
> 'mesos-containerizer' with flags
> '--command="{"arguments":["mesos-executor","--launcher_dir=\/mesos\/mesos-1.2.0\/_build\/src"],"shell":false,"value":"\/mesos\/mesos-1.2.0\/_build\/src\/mesos-executor"}"
>
> --environment="{"LIBPROCESS_PORT":"0","MESOS_AGENT_ENDPOINT":"172.17.0.3:50719","MESOS_CHECKPOINT":"0","MESOS_DIRECTORY":"\/tmp\/HTTPCommandExecutorTest_TerminateWithACK_zuye4X\/slaves\/266cf569-3b26-40cb-be6f-080699ef02a1-S0\/frameworks\/266cf569-3b26-40cb-be6f-080699ef02a1-0000\/executors\/19aea987-3513-4860-8cdf-d298f1c6c207\/runs\/bcf25a3b-79a9-46af-b6b3-f687355a4e82","MESOS_EXECUTOR_ID":"19aea987-3513-4860-8cdf-d298f1c6c207","MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD":"5secs","MESOS_FRAMEWORK_ID":"266cf569-3b26-40cb-be6f-080699ef02a1-0000","MESOS_HTTP_COMMAND_EXECUTOR":"1","MESOS_SANDBOX":"\/tmp\/HTTPCommandExecutorTest_TerminateWithACK_zuye4X\/slaves\/266cf569-3b26-40cb-be6f-080699ef02a1-S0\/frameworks\/266cf569-3b26-40cb-be6f-080699ef02a1-0000\/executors\/19aea987-3513-4860-8cdf-d298f1c6c207\/runs\/bcf25a3b-79a9-46af-b6b3-f687355a4e82","MESOS_SLAVE_ID":"266cf569-3b26-40cb-be6f-080699ef02a1-S0","MESOS_SLAVE_PID":"(1)@172.17.0.3:50719","PATH":"\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin"}"
> --help="false" --pipe_read="9" --pipe_write="12" --pre_exec_commands="[]"
> --runtime_directory="/tmp/HTTPCommandExecutorTest_TerminateWithACK_4fCwxB/containers/bcf25a3b-79a9-46af-b6b3-f687355a4e82"
> --unshare_namespace_mnt="false" --user="mesos"
> --working_directory="/tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/frameworks/266cf569-3b26-40cb-be6f-080699ef02a1-0000/executors/19aea987-3513-4860-8cdf-d298f1c6c207/runs/bcf25a3b-79a9-46af-b6b3-f687355a4e82"'
> I1104 21:43:48.109117 31822 launcher.cpp:127] Forked child with pid '31842'
> for container 'bcf25a3b-79a9-46af-b6b3-f687355a4e82'
> I1104 21:43:48.114708 31827 fetcher.cpp:345] Starting to fetch URIs for
> container: bcf25a3b-79a9-46af-b6b3-f687355a4e82, directory:
> /tmp/HTTPCommandExecutorTest_TerminateWithACK_zuye4X/slaves/266cf569-3b26-40cb-be6f-080699ef02a1-S0/frameworks/266cf569-3b26-40cb-be6f-080699ef02a1-0000/executors/19aea987-3513-4860-8cdf-d298f1c6c207/runs/bcf25a3b-79a9-46af-b6b3-f687355a4e82
> I1104 21:43:48.120514 31824 leveldb.cpp:341] Persisting action (18 bytes) to
> leveldb took 46.104261ms
> I1104 21:43:48.121068 31824 leveldb.cpp:399] Deleting ~2 keys from leveldb
> took 90072ns
> I1104 21:43:48.121314 31824 replica.cpp:708] Persisted action TRUNCATE at
> position 4
> I1104 21:43:48.527518 31827 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:48.527603 31827 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:48.527698 31827 hierarchical.cpp:1286] Performed allocation for 1
> agents in 560672ns
> I1104 21:43:49.528602 31818 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:49.528686 31818 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:49.528774 31818 hierarchical.cpp:1286] Performed allocation for 1
> agents in 406807ns
> I1104 21:43:50.530165 31817 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:50.530411 31817 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:50.530570 31817 hierarchical.cpp:1286] Performed allocation for 1
> agents in 629648ns
> I1104 21:43:51.531759 31818 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:51.532184 31818 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:51.532444 31818 hierarchical.cpp:1286] Performed allocation for 1
> agents in 934089ns
> I1104 21:43:52.538225 31814 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:52.538314 31814 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:52.538415 31814 hierarchical.cpp:1286] Performed allocation for 1
> agents in 446541ns
> I1104 21:43:53.539400 31825 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:53.539484 31825 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:53.539607 31825 hierarchical.cpp:1286] Performed allocation for 1
> agents in 446476ns
> I1104 21:43:54.540755 31822 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:54.540838 31822 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:54.540920 31822 hierarchical.cpp:1286] Performed allocation for 1
> agents in 390633ns
> I1104 21:43:55.542132 31818 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:55.542212 31818 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:55.542296 31818 hierarchical.cpp:1286] Performed allocation for 1
> agents in 498341ns
> I1104 21:43:56.545979 31819 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:56.546061 31819 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:56.546146 31819 hierarchical.cpp:1286] Performed allocation for 1
> agents in 421659ns
> I1104 21:43:57.546845 31826 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:57.546926 31826 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:57.547003 31826 hierarchical.cpp:1286] Performed allocation for 1
> agents in 415968ns
> I1104 21:43:58.547880 31816 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:58.548008 31816 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:58.548120 31816 hierarchical.cpp:1286] Performed allocation for 1
> agents in 521955ns
> I1104 21:43:59.548861 31825 hierarchical.cpp:1694] No allocations performed
> I1104 21:43:59.548948 31825 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:43:59.549033 31825 hierarchical.cpp:1286] Performed allocation for 1
> agents in 399053ns
> I1104 21:44:00.550261 31823 hierarchical.cpp:1694] No allocations performed
> I1104 21:44:00.550832 31823 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:44:00.551102 31823 hierarchical.cpp:1286] Performed allocation for 1
> agents in 1.104644ms
> I1104 21:44:01.552731 31822 hierarchical.cpp:1694] No allocations performed
> I1104 21:44:01.552816 31822 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:44:01.552906 31822 hierarchical.cpp:1286] Performed allocation for 1
> agents in 394875ns
> I1104 21:44:02.553762 31817 hierarchical.cpp:1694] No allocations performed
> I1104 21:44:02.554481 31817 hierarchical.cpp:1789] No inverse offers to send
> out!
> I1104 21:44:02.554841 31817 hierarchical.cpp:1286] Performed allocation for 1
> agents in 1.283909ms
> I1104 21:44:02.998473 31818 slave.cpp:4251] Received ping from
> slave-observer(8)@172.17.0.3:50719
> ../../src/tests/command_executor_tests.cpp:392: Failure
> Failed to wait 15secs for statusRunning
> I1104 21:44:03.015902 31817 master.cpp:1297] Framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719 disconnected
> I1104 21:44:03.015971 31817 master.cpp:2918] Disconnecting framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:44:03.016018 31817 master.cpp:2942] Deactivating framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:44:03.016219 31821 hierarchical.cpp:386] Deactivated framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> ../../src/tests/command_executor_tests.cpp:381: Failure
> Actual function call count doesn't match EXPECT_CALL(sched, statusUpdate(_,
> _))...
> Expected: to be called twice
> Actual: never called - unsatisfied and active
> I1104 21:44:03.016443 31817 master.cpp:1310] Giving framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719 0ns to
> failover
> I1104 21:44:03.016952 31826 master.cpp:6426] Framework failover timeout,
> removing framework 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> I1104 21:44:03.016983 31826 master.cpp:7170] Removing framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (default) at
> scheduler-3a4e1df5-e0b3-4373-937e-daf6914ba47d@172.17.0.3:50719
> ../../src/tests/command_executor_tests.cpp:386: Failure
> Actual function call count doesn't match EXPECT_CALL(slave,
> executorTerminated(_, _, _))...
> Expected: to be called once
> Actual: never called - unsatisfied and active
> I1104 21:44:03.017590 31827 slave.cpp:2575] Asked to shut down framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 by master@172.17.0.3:50719
> I1104 21:44:03.017659 31827 slave.cpp:2600] Shutting down framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:44:03.017762 31826 master.cpp:7715] Updating the state of task
> 19aea987-3513-4860-8cdf-d298f1c6c207 of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 (latest state: TASK_KILLED, status
> update state: TASK_KILLED)
> I1104 21:44:03.017792 31827 slave.cpp:4852] Cleaning up framework üÑ %
> @ üÑ x üÑ agent
> F1104 21:44:03.017873 31827 slave.cpp:4855] Check failed: framework->state ==
> Framework::RUNNING || framework->state == Framework::TERMINATING
> *** Check failure stack trace: ***
> I1104 21:44:03.018919 31826 master.cpp:7811] Removing task
> 19aea987-3513-4860-8cdf-d298f1c6c207 with resources cpus(*):2; mem(*):1024;
> disk(*):1024; ports(*):[31000-32000] of framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000 on agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 at (1)@172.17.0.3:50719 (b7fb1902101b)
> I1104 21:44:03.019129 31823 hierarchical.cpp:1018] Recovered cpus(*):2;
> mem(*):1024; disk(*):1024; ports(*):[31000-32000] (total: cpus(*):2;
> mem(*):1024; disk(*):1024; ports(*):[31000-32000], allocated: {}) on agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0 from framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:44:03.175541 31826 master.cpp:1097] Master terminating
> I1104 21:44:03.176000 31816 hierarchical.cpp:337] Removed framework
> 266cf569-3b26-40cb-be6f-080699ef02a1-0000
> I1104 21:44:03.177108 31816 hierarchical.cpp:517] Removed agent
> 266cf569-3b26-40cb-be6f-080699ef02a1-S0
> @ 0x7fd220cbf56a google::LogMessage::Fail()
> [ FAILED ] HTTPCommandExecutorTest.TerminateWithACK (16901 ms)
> [ RUN ] HTTPCommandExecutorTest.ExplicitAcknowledgements
> I1104 21:44:03.206542 31793 cluster.cpp:158] Creating default 'local'
> authorizer
> @ 0x7fd220cbf4c9 google::LogMessage::SendToLog()
> @ 0x7fd220cbeeda google::LogMessage::Flush()
> @ 0x7fd220cc1c0e google::LogMessageFatal::~LogMessageFatal()
> @ 0x7fd21ffc68d5 mesos::internal::slave::Slave::removeFramework()
> @ 0x7fd21ffb616e mesos::internal::slave::Slave::shutdownFramework()
> @ 0x7fd21fffdc99 ProtobufProcess<>::handler1<>()
> @ 0x7fd22005622d
> _ZNSt5_BindIFPFvPN5mesos8internal5slave5SlaveEMS3_FvRKN7process4UPIDERKNS0_11FrameworkIDEEMNS1_24ShutdownFrameworkMessageEKFSB_vES8_RKSsES4_SD_SG_St12_PlaceholderILi1EESL_ILi2EEEE6__callIvJS8_SI_EJLm0ELm1ELm2ELm3ELm4EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
> @ 0x7fd220045846
> _ZNSt5_BindIFPFvPN5mesos8internal5slave5SlaveEMS3_FvRKN7process4UPIDERKNS0_11FrameworkIDEEMNS1_24ShutdownFrameworkMessageEKFSB_vES8_RKSsES4_SD_SG_St12_PlaceholderILi1EESL_ILi2EEEEclIJS8_SI_EvEET0_DpOT_
> @ 0x7fd220032a42 std::_Function_handler<>::_M_invoke()
> @ 0x1111c29 std::function<>::operator()()
> @ 0x13c0416 ProtobufProcess<>::visit()
> @ 0x7fd21fc347a4 process::MessageEvent::visit()
> @ 0xa67a86 process::ProcessBase::serve()
> @ 0x7fd220bef00b process::ProcessManager::resume()
> @ 0x7fd220bebb9b _ZZN7process14ProcessManager12init_threadsEvENKUt_clEv
> @ 0x7fd220bfa66e
> _ZNSt12_Bind_simpleIFZN7process14ProcessManager12init_threadsEvEUt_vEE9_M_invokeIIEEEvSt12_Index_tupleIIXspT_EEE
> @ 0x7fd220bfa5c5
> _ZNSt12_Bind_simpleIFZN7process14ProcessManager12init_threadsEvEUt_vEEclEv
> @ 0x7fd220bfa55e
> _ZNSt6thread5_ImplISt12_Bind_simpleIFZN7process14ProcessManager12init_threadsEvEUt_vEEE6_M_runEv
> @ 0x7fd21b0b0220 (unknown)
> @ 0x7fd21b716dc5 start_thread
> @ 0x7fd21a817ced __clone
> make[4]: *** [check-local] Aborted
> make[4]: Leaving directory `/mesos/mesos-1.2.0/_build/src'
> make[3]: *** [check-am] Error 2
> make[3]: Leaving directory `/mesos/mesos-1.2.0/_build/src'
> make[2]: *** [check] Error 2
> make[2]: Leaving directory `/mesos/mesos-1.2.0/_build/src'
> make[1]: *** [check-recursive] Error 1
> make[1]: Leaving directory `/mesos/mesos-1.2.0/_build'
> make: *** [distcheck] Error 1
> + docker rmi mesos-1478292489-26858
> Untagged: mesos-1478292489-26858:latest
> Deleted:
> sha256:70ba295d0c7694d9c5fa660260f7802366e3934ea63a50dc39b0d96e7d0e317d
> Deleted:
> sha256:6ea5c23c10290f93176de6ee6c15f768a5ffb50b9fce575d5f2ba913ffcb8082
> Deleted:
> sha256:ecb31a4c4466faeab60ba71d5bc05190e306e2a1154e8a090c19e686fa946de9
> Deleted:
> sha256:204ca9080801dec3f2328a8a038fd02a7adae5da70ccc70ca988b5b8ef416a5d
> Deleted:
> sha256:7976202fea0a5ef2c846a1a3e29d951aa323e6f96dfff856ec9615c4de238c5e
> Deleted:
> sha256:baacec71af85abafca8e478204e63e5e237323e53583fb87a126deac388841f6
> Deleted:
> sha256:b6e4783eb3fefb656acadbb01db7e702d4a1b7898a901c4b447fd92cce7e3919
> Deleted:
> sha256:3cdcf01e0f0aae0894ce554f249b2d2d9dcc029be4d172fccaa2217b3ba35598
> Deleted:
> sha256:43aea5cc78365b286ef353743bfac52df46ca69b0188ffbf6d8ecf4741f3b463
> Deleted:
> sha256:582242ccadcb3e14e90fc429e9547a9cd2c49e13e05ee1d3e311c5624b6498ae
> Deleted:
> sha256:a830017303ba9bbd431b5dcbf6c8b3991b5f2d625ac2f411a19cbcb16beeaf9b
> Deleted:
> sha256:e86101d26ff0d8d16ad38517aaedaf2bda08415328714f36c2b03dceff185bfd
> Deleted:
> sha256:b5acd89fc60ab98a8757371860d3446567593748c3c9a25dd5042e76a1123809
> Deleted:
> sha256:6f0c88f47cefa553cade2f445f024b680daa2b515dce0bcc8fa03c241faa1b33
> Deleted:
> sha256:a692bab5031e10a12ba20ef5bb060e4161e317cefbeb8de40e43f2fd7aec9e7d
> Deleted:
> sha256:2e727b6d1554957fd994b0b2cb3a27894a0419272dec177d87c2de9f28649fa1
> Deleted:
> sha256:b9372aef649a8c003fc527adc2242ccfbbbc63221d41be0f32d36556dee3d197
> Deleted:
> sha256:7c3801033b029edeef89012b2bac19fe034fa1944d74d1366311b0ce1a78922e
> Deleted:
> sha256:3f466225b26a44f9ec7614afefd27634e5e9b300a534be15434b090b5579eb06
> Deleted:
> sha256:fdbe438f9f2caf785a78eb2e3f6b573105a37664b158b5c4c2a67e087e50ab4f
> Deleted:
> sha256:132f6c9016cc05e1025c7ae464e6d9ed74aa2cc5568e4b744aa491af58435385
> Deleted:
> sha256:cec6b3051662f8805fcbc80604f47850d48317ef98a8b7fc4c3724d379c51724
> Deleted:
> sha256:f4ef5fc6a72d7f36595b7356d2b15a0accab3c0f1f3703fe95ab0816605651a5
> Deleted:
> sha256:b7cc8689244c7ea617b79f9d96a66caa40269005f590c4ab3d92dceeec960672
> Deleted:
> sha256:f9ed4f83503abf45080b1a63a2a054dac341865f08f45c739df99c84e2b4f6a3
> Build step 'Execute shell' marked build as failure
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
