[
https://issues.apache.org/jira/browse/MESOS-7831?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Yan Xu updated MESOS-7831:
--------------------------
Description:
When an agent reregisters, the master [doesn't apply refinement to
completed_frameworks|https://github.pie.apple.com/pie/mesos/blob/cd3380c4e9521b4b26f9030658816eee7a4b89a1/src/master/master.cpp#L8600]
the agents sends. This then would violate CHECKs that could be triggered by
querying the /state endpoint.
A sample CHECK failure backtrace:
{noformat:title=}
Program terminated with signal SIGABRT, Aborted.
#0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
#0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
#1 0x00007fae8a752105 in abort () from /lib64/libc.so.6
#2 0x00007fae8d6bccf4 in DumpStackTraceAndExit () at src/utilities.cc:147
#3 0x00007fae8d6b5aaa in Fail () at src/logging.cc:1458
#4 0x00007fae8d6b5a06 in SendToLog () at src/logging.cc:1412
#5 0x00007fae8d6b53fc in Flush () at src/logging.cc:1281
#6 0x00007fae8d6b81b6 in ~LogMessageFatal () at src/logging.cc:1984
#7 0x00007fae8c6d3c1a in mesos::Resources::isEmpty (resource=...) at
../../src/common/resources.cpp:1052
#8 0x00007fae8c6d3d16 in mesos::Resources::Resource_::isEmpty
(this=this@entry=0x7fae787e6910) at ../../src/common/resources.cpp:1174
#9 0x00007fae8c6d3d43 in mesos::Resources::add (this=0x7fae787e6a40, that=...)
at ../../src/common/resources.cpp:1994
#10 0x00007fae8c6d53f0 in mesos::Resources::operator+=
(this=this@entry=0x7fae787e6a40, that=...) at
../../src/common/resources.cpp:2017
#11 0x00007fae8c6d54a8 in mesos::Resources::operator+=
(this=this@entry=0x7fae787e6a40, that=...) at
../../src/common/resources.cpp:2026
#12 0x00007fae8c6d55fb in mesos::Resources::Resources (this=0x7fae787e6a40,
_resources=...) at ../../src/common/resources.cpp:1278
#13 0x00007fae8c6ade1e in mesos::json (writer=writer@entry=0x7fae787e6b30,
task=...) at ../../src/common/http.cpp:675
#14 0x00007fae8c85d62f in operator() (stream=<optimized out>,
__closure=<optimized out>) at ../../3rdparty/stout/include/stout/jsonify.hpp:771
#15 std::_Function_handler<void (std::ostream*), std::function<void
(std::ostream*)> JSON::internal::jsonify<mesos::Task>(mesos::Task const&,
JSON::internal::LessPrefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
const&, std::ostream*) (__functor=..., __args#0=<optimized out>) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#16 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
this=0x7fae787e6bd0) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#17 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#18 0x00007fae8c85f3f9 in JSON::ArrayWriter::element<mesos::Task>
(this=0x7fae787e6cf0, value=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:406
#19 0x00007fae8c86a378 in
mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::{lambda(JSON::ArrayWriter*)#3}::operator()(JSON::ArrayWriter*) const (
__closure=0x7fae787e6db0, writer=writer@entry=0x7fae787e6cf0) at
../../src/master/http.cpp:348
#20 0x00007fae8c86a5ad in operator() (stream=<optimized out>,
__closure=0x7fad9a48af68) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#21 std::_Function_handler<void (std::ostream*), std::function<void
(std::ostream*)>
JSON::internal::jsonify<mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::{lambda(JSON::ArrayWriter*)#3},
void>(mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::{lambda(JSON::ArrayWriter*)#3} const&,
JSON::internal::Prefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
const&, std::ostream*) (__functor=..., __args#0=<optimized out>) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#22 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
this=0x7fae787e6e00) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#23 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#24 0x00007fae8c8b9331 in
field<mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::<lambda(JSON::ArrayWriter*)> > (value=..., key="completed_tasks",
this=0x7fae787e6eb0)
at ../../3rdparty/stout/include/stout/jsonify.hpp:440
#25 mesos::internal::master::FullFrameworkWriter::operator()
(this=0x7fae787e6f70, writer=writer@entry=0x7fae787e6eb0) at
../../src/master/http.cpp:350
#26 0x00007fae8c8b9bfd in operator() (stream=<optimized out>,
__closure=0x7fad9a49b080) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#27 std::_Function_handler<void (std::ostream*), std::function<void
(std::ostream*)>
JSON::internal::jsonify<mesos::internal::master::FullFrameworkWriter,
void>(mesos::internal::master::FullFrameworkWriter const&,
JSON::internal::Prefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
const&, std::ostream*) (__functor=..., __args#0=<optimized out>)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#28 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
this=0x7fae787e6f90) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#29 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#30 0x00007fae8c81792f in element<mesos::internal::master::FullFrameworkWriter>
(value=..., this=0x7fae787e6fe0) at
../../3rdparty/stout/include/stout/jsonify.hpp:406
#31 operator() (writer=0x7fae787e6fe0, __closure=0x7fae787e7310) at
../../src/master/http.cpp:2931
#32 operator() (stream=<optimized out>, __closure=<optimized out>) at
../../3rdparty/stout/include/stout/jsonify.hpp:759
#33 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>
>&)>::<lambda(JSON::ObjectWriter*)>::<lambda(JSON::ArrayWriter*)>;
<template-parameter-1-2> = void]::<lambda(std::ostream*)> >::_M_invoke(const
std::_Any_data &, std::basic_ostream<char, std::char_traits<char> > *)
(__functor=..., __args#0=<optimized out>)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#34 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
this=0x7fae787e76a0) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#35 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#36 0x00007fae8c84b24d in
mesos::internal::master::Master::Http::state(process::http::Request const&,
Option<process::http::authentication::Principal> const&)
const::{lambda(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >
const&)#1}::operator()(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> > const&)
const::{lambda(JSON::ObjectWriter*)#1}::operator()(JSON::ObjectWriter) const ()
at ../../3rdparty/stout/include/stout/jsonify.hpp:440
#37 0x00007fae8c84c39d in operator() (stream=<optimized out>,
__closure=0x7fadbaef9b08) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#38 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>
>&)>::<lambda(JSON::ObjectWriter*)>; <template-parameter-1-2> =
void]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
std::basic_ostream<char, std::char_traits<char> > *) (__functor=...,
__args#0=<optimized out>)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#39 0x00007fae8d57ea4c in operator() (__args#0=0x7fae787e7820,
this=0x7fae787e7a10) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#40 operator<< (that=..., stream=...) at
../../../3rdparty/stout/include/stout/jsonify.hpp:172
#41 process::http::OK::OK(JSON::Proxy&&, Option<std::string> const&)
(this=0x7fae787e7ad0, value=..., jsonp=...) at
../../../3rdparty/libprocess/src/http.cpp:690
#42 0x00007fae8c81dcda in operator() (acceptors=..., __closure=<optimized out>)
at ../../src/master/http.cpp:2967
#43 operator() (__closure=<optimized out>) at
../../3rdparty/libprocess/include/process/deferred.hpp:225
#44 std::_Function_handler<process::Future<process::http::Response>(),
process::_Deferred<G>::operator std::function<R(P0)>() const::<lambda(P0)>
[with R = process::Future<process::http::Response>; P0 = const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >&; F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >&)>]::<lambda()>
>::_M_invoke(const std::_Any_data &) (__functor=...) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2025
#45 0x00007fae8c6797c2 in operator() (this=0x7fadb2c971d0) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#46 operator() (__closure=0x7fadb2c971c0) at
../../3rdparty/libprocess/include/process/dispatch.hpp:112
#47 std::_Function_handler<void (process::ProcessBase*),
process::Future<process::http::Response>
process::internal::Dispatch<process::Future<process::http::Response>
>::operator()<std::function<process::Future<process::http::Response>
()>&>(process::UPID const&,
std::function<process::Future<process::http::Response>
()>&)::{lambda(process::ProcessBase*)#1}>::_M_invoke(std::_Any_data const&,
process::ProcessBase*) (__functor=..., __args#0=<optimized out>) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#48 0x00007fae8d60cd1c in serve (event=..., this=0x7fae84098058) at
../../../3rdparty/libprocess/include/process/process.hpp:84
#49 process::ProcessManager::resume (this=0x7fae840a2a80,
process=0x7fae84098058) at ../../../3rdparty/libprocess/src/process.cpp:3242
#50 0x00007fae8d6147a6 in operator() (__closure=0x7fae8401cf38) at
../../../3rdparty/libprocess/src/process.cpp:2835
#51 _M_invoke<> (this=0x7fae8401cf38) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1700
#52 operator() (this=0x7fae8401cf38) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1688
#53
std::thread::_Impl<std::_Bind_simple<process::ProcessManager::init_threads()::<lambda()>()>
>::_M_run(void) (this=0x7fae8401cf20) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/thread:115
#54 0x00007fae8b002470 in ?? () from /usr/lib64/libstdc++.so.6
#55 0x00007fae8b2599d1 in start_thread () from /lib64/libpthread.so.0
#56 0x00007fae8a806b5d in clone () from /lib64/libc.so.6
{noformat}
was:
When an agent reregisters, the master [doesn't apply refinement to
completed_frameworks|https://github.pie.apple.com/pie/mesos/blob/cd3380c4e9521b4b26f9030658816eee7a4b89a1/src/master/master.cpp#L8600]
the agents sends. This then would violate CHECKs that could be triggered by
querying the /state endpoint.
A sample CHECK failure backtrace:
{noformat:title=}
Program terminated with signal 6, Aborted.
#0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
#0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
#1 0x00007fae8a752105 in abort () from /lib64/libc.so.6
#2 0x00007fae8d6bccf4 in DumpStackTraceAndExit () at src/utilities.cc:147
#3 0x00007fae8d6b5aaa in Fail () at src/logging.cc:1458
#4 0x00007fae8d6b5a06 in SendToLog () at src/logging.cc:1412
#5 0x00007fae8d6b53fc in Flush () at src/logging.cc:1281
#6 0x00007fae8d6b81b6 in ~LogMessageFatal () at src/logging.cc:1984
#7 0x00007fae8c6d3c1a in mesos::Resources::isEmpty (resource=...) at
../../src/common/resources.cpp:1052
#8 0x00007fae8c6d3d43 in mesos::Resources::add (this=0x7fae787e6a40, that=...)
at ../../src/common/resources.cpp:1994
#9 0x00007fae8c6d53f0 in mesos::Resources::operator+= (this=0x7fae787e6a40,
that=...) at ../../src/common/resources.cpp:2017
#10 0x00007fae8c6d54a8 in mesos::Resources::operator+= (this=0x7fae787e6a40,
that=Unhandled dwarf expression opcode 0xf3
) at ../../src/common/resources.cpp:2026
#11 0x00007fae8c6d55fb in mesos::Resources::Resources (this=0x7fae787e6a40,
_resources=Unhandled dwarf expression opcode 0xf3
) at ../../src/common/resources.cpp:1278
#12 0x00007fae8c6ade1e in mesos::json (writer=0x7fae787e6b30, task=...) at
../../src/common/http.cpp:675
#13 0x00007fae8c85d62f in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:771
#14 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const T&, JSON::internal::LessPrefer) [with T =
mesos::Task]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
std::basic_ostream<char, std::char_traits<char> > *) (__functor=Unhandled dwarf
expression opcode 0xf3
) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#15 0x00007fae8c69af73 in operator() (stream=..., that=...) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#16 JSON::operator<< (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#17 0x00007fae8c85f3f9 in JSON::ArrayWriter::element<mesos::Task>
(this=0x7fae787e6cf0, value=Unhandled dwarf expression opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:406
#18 0x00007fae8c86a378 in
mesos::internal::master::FullFrameworkWriter::<lambda(JSON::ArrayWriter*)>::operator()(JSON::ArrayWriter
*) const (__closure=0x7fae787e6db0, writer=0x7fae787e6cf0)
at ../../src/master/http.cpp:348
#19 0x00007fae8c86a5ad in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#20 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::<lambda(JSON::ArrayWriter*)>; <template-parameter-1-2> =
void]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
std::basic_ostream<char, std::char_traits<char> > *) (__functor=Unhandled dwarf
expression opcode 0xf3
) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#21 0x00007fae8c69af73 in operator() (stream=..., that=...) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#22 JSON::operator<< (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#23 0x00007fae8c8b9331 in
field<mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
const::<lambda(JSON::ArrayWriter*)> > (this=0x7fae787e6f70,
writer=0x7fae787e6eb0)
at ../../3rdparty/stout/include/stout/jsonify.hpp:440
#24 mesos::internal::master::FullFrameworkWriter::operator()
(this=0x7fae787e6f70, writer=0x7fae787e6eb0) at ../../src/master/http.cpp:350
#25 0x00007fae8c8b9bfd in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#26 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::FullFrameworkWriter; <template-parameter-1-2> =
void]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
std::basic_ostream<char, std::char_traits<char> > *) (__functor=Unhandled dwarf
expression opcode 0xf3
)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#27 0x00007fae8c69af73 in operator() (stream=..., that=...) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#28 JSON::operator<< (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#29 0x00007fae8c81792f in element<mesos::internal::master::FullFrameworkWriter>
(__functor=Unhandled dwarf expression opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:406
#30 operator() (__functor=Unhandled dwarf expression opcode 0xf3
) at ../../src/master/http.cpp:2931
#31 operator() (__functor=Unhandled dwarf expression opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#32 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>
>&)>::<lambda(JSON::ObjectWriter*)>::<lambda(JSON::ArrayWriter*)>;
<template-parameter-1-2> = void]::<lambda(std::ostream*)> >::_M_invoke(const
std::_Any_data &, std::basic_ostream<char, std::char_traits<char> > *)
(__functor=Unhandled dwarf expression opcode 0xf3
)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#33 0x00007fae8c69af73 in operator() (stream=..., that=...) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#34 JSON::operator<< (stream=..., that=...) at
../../3rdparty/stout/include/stout/jsonify.hpp:172
#35 0x00007fae8c84b24d in
mesos::internal::master::Master::Http::state(process::http::Request const&,
Option<process::http::authentication::Principal> const&)
const::{lambda(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >
const&)#1}::operator()(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> > const&)
const::{lambda(JSON::ObjectWriter*)#1}::operator()(JSON::ObjectWriter) const ()
at ../../3rdparty/stout/include/stout/jsonify.hpp:440
#36 0x00007fae8c84c39d in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at ../../3rdparty/stout/include/stout/jsonify.hpp:759
#37 std::_Function_handler<void(std::basic_ostream<char, std::char_traits<char>
>*), JSON::internal::jsonify(const F&, JSON::internal::Prefer) [with F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>
>&)>::<lambda(JSON::ObjectWriter*)>; <template-parameter-1-2> =
void]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
std::basic_ostream<char, std::char_traits<char> > *) (__functor=Unhandled dwarf
expression opcode 0xf3
) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#38 0x00007fae8d57ea4c in operator() (this=0x7fae787e7ad0, value=...,
jsonp=...) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#39 operator<< (this=0x7fae787e7ad0, value=..., jsonp=...) at
../../../3rdparty/stout/include/stout/jsonify.hpp:172
#40 process::http::OK::OK (this=0x7fae787e7ad0, value=..., jsonp=...) at
../../../3rdparty/libprocess/src/http.cpp:690
#41 0x00007fae8c81dcda in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at ../../src/master/http.cpp:2967
#42 operator() (__functor=Unhandled dwarf expression opcode 0xf3
) at ../../3rdparty/libprocess/include/process/deferred.hpp:225
#43 std::_Function_handler<process::Future<process::http::Response>(),
process::_Deferred<G>::operator std::function<R(P0)>() const::<lambda(P0)>
[with R = process::Future<process::http::Response>; P0 = const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >&; F =
mesos::internal::master::Master::Http::state(const process::http::Request&,
const Option<process::http::authentication::Principal>&) const::<lambda(const
std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor>,
process::Owned<mesos::AuthorizationAcceptor> >&)>]::<lambda()>
>::_M_invoke(const std::_Any_data &) (__functor=Unhandled dwarf expression
opcode 0xf3
) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2025
#44 0x00007fae8c6797c2 in operator() (__functor=Unhandled dwarf expression
opcode 0xf3
) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
#45 operator() (__functor=Unhandled dwarf expression opcode 0xf3
) at ../../3rdparty/libprocess/include/process/dispatch.hpp:112
#46 std::_Function_handler<void(process::ProcessBase*),
process::internal::Dispatch<process::Future<T> >::operator()(const
process::UPID&, F&&) [with F =
std::function<process::Future<process::http::Response>()>&; R =
process::http::Response]::<lambda(process::ProcessBase*)> >::_M_invoke(const
std::_Any_data &, process::ProcessBase *) (__functor=Unhandled dwarf expression
opcode 0xf3
)
at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
#47 0x00007fae8d60cd1c in serve (this=0x7fae840a2a80, process=0x7fae84098058)
at ../../../3rdparty/libprocess/include/process/process.hpp:84
#48 process::ProcessManager::resume (this=0x7fae840a2a80,
process=0x7fae84098058) at ../../../3rdparty/libprocess/src/process.cpp:3242
#49 0x00007fae8d6147a6 in operator() (this=0x7fae8401cf20) at
../../../3rdparty/libprocess/src/process.cpp:2835
#50 _M_invoke<> (this=0x7fae8401cf20) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1700
#51 operator() (this=0x7fae8401cf20) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1688
#52
std::thread::_Impl<std::_Bind_simple<process::ProcessManager::init_threads()::<lambda()>()>
>::_M_run(void) (this=0x7fae8401cf20) at
/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/thread:115
#53 0x00007fae8b002470 in ?? () from /usr/lib64/libstdc++.so.6
#54 0x00007fae8b2599d1 in start_thread () from /lib64/libpthread.so.0
#55 0x00007fae8a806b5d in clone () from /lib64/libc.so.6
{noformat}
> Resource refinement is not applied to tasks in completed_frameworks.
> --------------------------------------------------------------------
>
> Key: MESOS-7831
> URL: https://issues.apache.org/jira/browse/MESOS-7831
> Project: Mesos
> Issue Type: Bug
> Affects Versions: 1.4.0
> Reporter: Yan Xu
> Priority: Blocker
>
> When an agent reregisters, the master [doesn't apply refinement to
> completed_frameworks|https://github.pie.apple.com/pie/mesos/blob/cd3380c4e9521b4b26f9030658816eee7a4b89a1/src/master/master.cpp#L8600]
> the agents sends. This then would violate CHECKs that could be triggered by
> querying the /state endpoint.
> A sample CHECK failure backtrace:
> {noformat:title=}
> Program terminated with signal SIGABRT, Aborted.
> #0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
> #0 0x00007fae8a750925 in raise () from /lib64/libc.so.6
> #1 0x00007fae8a752105 in abort () from /lib64/libc.so.6
> #2 0x00007fae8d6bccf4 in DumpStackTraceAndExit () at src/utilities.cc:147
> #3 0x00007fae8d6b5aaa in Fail () at src/logging.cc:1458
> #4 0x00007fae8d6b5a06 in SendToLog () at src/logging.cc:1412
> #5 0x00007fae8d6b53fc in Flush () at src/logging.cc:1281
> #6 0x00007fae8d6b81b6 in ~LogMessageFatal () at src/logging.cc:1984
> #7 0x00007fae8c6d3c1a in mesos::Resources::isEmpty (resource=...) at
> ../../src/common/resources.cpp:1052
> #8 0x00007fae8c6d3d16 in mesos::Resources::Resource_::isEmpty
> (this=this@entry=0x7fae787e6910) at ../../src/common/resources.cpp:1174
> #9 0x00007fae8c6d3d43 in mesos::Resources::add (this=0x7fae787e6a40,
> that=...) at ../../src/common/resources.cpp:1994
> #10 0x00007fae8c6d53f0 in mesos::Resources::operator+=
> (this=this@entry=0x7fae787e6a40, that=...) at
> ../../src/common/resources.cpp:2017
> #11 0x00007fae8c6d54a8 in mesos::Resources::operator+=
> (this=this@entry=0x7fae787e6a40, that=...) at
> ../../src/common/resources.cpp:2026
> #12 0x00007fae8c6d55fb in mesos::Resources::Resources (this=0x7fae787e6a40,
> _resources=...) at ../../src/common/resources.cpp:1278
> #13 0x00007fae8c6ade1e in mesos::json (writer=writer@entry=0x7fae787e6b30,
> task=...) at ../../src/common/http.cpp:675
> #14 0x00007fae8c85d62f in operator() (stream=<optimized out>,
> __closure=<optimized out>) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:771
> #15 std::_Function_handler<void (std::ostream*), std::function<void
> (std::ostream*)> JSON::internal::jsonify<mesos::Task>(mesos::Task const&,
> JSON::internal::LessPrefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
> const&, std::ostream*) (__functor=..., __args#0=<optimized out>) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #16 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
> this=0x7fae787e6bd0) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #17 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:172
> #18 0x00007fae8c85f3f9 in JSON::ArrayWriter::element<mesos::Task>
> (this=0x7fae787e6cf0, value=...) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:406
> #19 0x00007fae8c86a378 in
> mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
> const::{lambda(JSON::ArrayWriter*)#3}::operator()(JSON::ArrayWriter*) const (
> __closure=0x7fae787e6db0, writer=writer@entry=0x7fae787e6cf0) at
> ../../src/master/http.cpp:348
> #20 0x00007fae8c86a5ad in operator() (stream=<optimized out>,
> __closure=0x7fad9a48af68) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:759
> #21 std::_Function_handler<void (std::ostream*), std::function<void
> (std::ostream*)>
> JSON::internal::jsonify<mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
> const::{lambda(JSON::ArrayWriter*)#3},
> void>(mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
> const::{lambda(JSON::ArrayWriter*)#3} const&,
> JSON::internal::Prefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
> const&, std::ostream*) (__functor=..., __args#0=<optimized out>) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #22 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
> this=0x7fae787e6e00) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #23 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:172
> #24 0x00007fae8c8b9331 in
> field<mesos::internal::master::FullFrameworkWriter::operator()(JSON::ObjectWriter*)
> const::<lambda(JSON::ArrayWriter*)> > (value=..., key="completed_tasks",
> this=0x7fae787e6eb0)
> at ../../3rdparty/stout/include/stout/jsonify.hpp:440
> #25 mesos::internal::master::FullFrameworkWriter::operator()
> (this=0x7fae787e6f70, writer=writer@entry=0x7fae787e6eb0) at
> ../../src/master/http.cpp:350
> #26 0x00007fae8c8b9bfd in operator() (stream=<optimized out>,
> __closure=0x7fad9a49b080) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:759
> #27 std::_Function_handler<void (std::ostream*), std::function<void
> (std::ostream*)>
> JSON::internal::jsonify<mesos::internal::master::FullFrameworkWriter,
> void>(mesos::internal::master::FullFrameworkWriter const&,
> JSON::internal::Prefer)::{lambda(std::ostream*)#1}>::_M_invoke(std::_Any_data
> const&, std::ostream*) (__functor=..., __args#0=<optimized out>)
> at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #28 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
> this=0x7fae787e6f90) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #29 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:172
> #30 0x00007fae8c81792f in
> element<mesos::internal::master::FullFrameworkWriter> (value=...,
> this=0x7fae787e6fe0) at ../../3rdparty/stout/include/stout/jsonify.hpp:406
> #31 operator() (writer=0x7fae787e6fe0, __closure=0x7fae787e7310) at
> ../../src/master/http.cpp:2931
> #32 operator() (stream=<optimized out>, __closure=<optimized out>) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:759
> #33 std::_Function_handler<void(std::basic_ostream<char,
> std::char_traits<char> >*), JSON::internal::jsonify(const F&,
> JSON::internal::Prefer) [with F =
> mesos::internal::master::Master::Http::state(const process::http::Request&,
> const Option<process::http::authentication::Principal>&) const::<lambda(const
> std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>
> >&)>::<lambda(JSON::ObjectWriter*)>::<lambda(JSON::ArrayWriter*)>;
> <template-parameter-1-2> = void]::<lambda(std::ostream*)> >::_M_invoke(const
> std::_Any_data &, std::basic_ostream<char, std::char_traits<char> > *)
> (__functor=..., __args#0=<optimized out>)
> at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #34 0x00007fae8c69af73 in operator() (__args#0=0x7fae787e7820,
> this=0x7fae787e76a0) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #35 JSON::operator<<(std::ostream&, JSON::Proxy&&) (stream=..., that=...) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:172
> #36 0x00007fae8c84b24d in
> mesos::internal::master::Master::Http::state(process::http::Request const&,
> Option<process::http::authentication::Principal> const&)
> const::{lambda(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor> >
> const&)#1}::operator()(std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor> > const&)
> const::{lambda(JSON::ObjectWriter*)#1}::operator()(JSON::ObjectWriter) const
> ()
> at ../../3rdparty/stout/include/stout/jsonify.hpp:440
> #37 0x00007fae8c84c39d in operator() (stream=<optimized out>,
> __closure=0x7fadbaef9b08) at
> ../../3rdparty/stout/include/stout/jsonify.hpp:759
> #38 std::_Function_handler<void(std::basic_ostream<char,
> std::char_traits<char> >*), JSON::internal::jsonify(const F&,
> JSON::internal::Prefer) [with F =
> mesos::internal::master::Master::Http::state(const process::http::Request&,
> const Option<process::http::authentication::Principal>&) const::<lambda(const
> std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>
> >&)>::<lambda(JSON::ObjectWriter*)>; <template-parameter-1-2> =
> void]::<lambda(std::ostream*)> >::_M_invoke(const std::_Any_data &,
> std::basic_ostream<char, std::char_traits<char> > *) (__functor=...,
> __args#0=<optimized out>)
> at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #39 0x00007fae8d57ea4c in operator() (__args#0=0x7fae787e7820,
> this=0x7fae787e7a10) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #40 operator<< (that=..., stream=...) at
> ../../../3rdparty/stout/include/stout/jsonify.hpp:172
> #41 process::http::OK::OK(JSON::Proxy&&, Option<std::string> const&)
> (this=0x7fae787e7ad0, value=..., jsonp=...) at
> ../../../3rdparty/libprocess/src/http.cpp:690
> #42 0x00007fae8c81dcda in operator() (acceptors=..., __closure=<optimized
> out>) at ../../src/master/http.cpp:2967
> #43 operator() (__closure=<optimized out>) at
> ../../3rdparty/libprocess/include/process/deferred.hpp:225
> #44 std::_Function_handler<process::Future<process::http::Response>(),
> process::_Deferred<G>::operator std::function<R(P0)>() const::<lambda(P0)>
> [with R = process::Future<process::http::Response>; P0 = const
> std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor> >&; F =
> mesos::internal::master::Master::Http::state(const process::http::Request&,
> const Option<process::http::authentication::Principal>&) const::<lambda(const
> std::tuple<process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor>,
> process::Owned<mesos::AuthorizationAcceptor> >&)>]::<lambda()>
> >::_M_invoke(const std::_Any_data &) (__functor=...) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2025
> #45 0x00007fae8c6797c2 in operator() (this=0x7fadb2c971d0) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2439
> #46 operator() (__closure=0x7fadb2c971c0) at
> ../../3rdparty/libprocess/include/process/dispatch.hpp:112
> #47 std::_Function_handler<void (process::ProcessBase*),
> process::Future<process::http::Response>
> process::internal::Dispatch<process::Future<process::http::Response>
> >::operator()<std::function<process::Future<process::http::Response>
> ()>&>(process::UPID const&,
> std::function<process::Future<process::http::Response>
> ()>&)::{lambda(process::ProcessBase*)#1}>::_M_invoke(std::_Any_data const&,
> process::ProcessBase*) (__functor=..., __args#0=<optimized out>) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:2039
> #48 0x00007fae8d60cd1c in serve (event=..., this=0x7fae84098058) at
> ../../../3rdparty/libprocess/include/process/process.hpp:84
> #49 process::ProcessManager::resume (this=0x7fae840a2a80,
> process=0x7fae84098058) at ../../../3rdparty/libprocess/src/process.cpp:3242
> #50 0x00007fae8d6147a6 in operator() (__closure=0x7fae8401cf38) at
> ../../../3rdparty/libprocess/src/process.cpp:2835
> #51 _M_invoke<> (this=0x7fae8401cf38) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1700
> #52 operator() (this=0x7fae8401cf38) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1688
> #53
> std::thread::_Impl<std::_Bind_simple<process::ProcessManager::init_threads()::<lambda()>()>
> >::_M_run(void) (this=0x7fae8401cf20) at
> /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/thread:115
> #54 0x00007fae8b002470 in ?? () from /usr/lib64/libstdc++.so.6
> #55 0x00007fae8b2599d1 in start_thread () from /lib64/libpthread.so.0
> #56 0x00007fae8a806b5d in clone () from /lib64/libc.so.6
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
