[2/3] mesos git commit: Implemented parse methods for OCI image spec.

[qianzhang]

Implemented parse methods for OCI image spec.

Review: https://reviews.apache.org/r/55139/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3681d61a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3681d61a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3681d61a

Branch: refs/heads/master
Commit: 3681d61aa6469bdb0ea7955dd7c9680c0fb5496b
Parents: 6726cd1
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Tue Feb 7 16:54:35 2017 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Tue Feb 7 16:54:35 2017 +0800

--
 include/mesos/oci/spec.hpp |  24 ++-
 src/CMakeLists.txt |   7 +
 src/Makefile.am|   1 +
 src/oci/spec.cpp   | 373 
 4 files changed, 404 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/include/mesos/oci/spec.hpp
--
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
index ac0b063..ea4f29e 100644
--- a/include/mesos/oci/spec.hpp
+++ b/include/mesos/oci/spec.hpp
@@ -24,7 +24,29 @@ namespace spec {
 namespace image {
 namespace v1 {
 
-// TODO(qianzhang): Add methods to parse OCI image spec
+// Constant strings for OCI image media types:
+// https://github.com/opencontainers/image-spec/blob/master/media-types.md
+constexpr char MEDIA_TYPE_MANIFEST_LIST[] =
+"application/vnd.oci.image.manifest.list.v1+json";
+
+constexpr char MEDIA_TYPE_MANIFEST[] =
+"application/vnd.oci.image.manifest.v1+json";
+
+constexpr char MEDIA_TYPE_CONFIG[] =
+"application/vnd.oci.image.config.v1+json";
+
+constexpr char MEDIA_TYPE_LAYER[] =
+"application/vnd.oci.image.layer.v1.tar+gzip";
+
+constexpr char MEDIA_TYPE_NONDIST_LAYER[] =
+"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip";
+
+/**
+ * Returns the OCI v1 descriptor, image manifest list, image manifest
+ * and image configuration from the given string.
+ */
+template 
+Try parse(const std::string& s);
 
 } // namespace v1 {
 } // namespace image {

http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/CMakeLists.txt
--
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c09bcde..3a4ace9 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -37,6 +37,7 @@ PROTOC_TO_INCLUDE_DIR(MASTER   mesos/master/master)
 PROTOC_TO_INCLUDE_DIR(MESOSmesos/mesos)
 PROTOC_TO_INCLUDE_DIR(MODULE   mesos/module/module)
 PROTOC_TO_INCLUDE_DIR(OVERSUBSCRIPTION mesos/slave/oversubscription)
+PROTOC_TO_INCLUDE_DIR(OCI_SPEC mesos/oci/spec)
 PROTOC_TO_INCLUDE_DIR(QUOTAmesos/quota/quota)
 PROTOC_TO_INCLUDE_DIR(SCHEDULERmesos/scheduler/scheduler)
 PROTOC_TO_INCLUDE_DIR(STATEmesos/state/state)
@@ -77,6 +78,7 @@ set(PUBLIC_PROTOBUF_SRC
   ${MASTER_PROTO_CC}
   ${MESOS_PROTO_CC}
   ${MODULE_PROTO_CC}
+  ${OCI_SPEC_PROTO_CC}
   ${OVERSUBSCRIPTION_PROTO_CC}
   ${QUOTA_PROTO_CC}
   ${SCHEDULER_PROTO_CC}
@@ -365,6 +367,10 @@ set(MODULE_SRC
   module/manager.cpp
   )
 
+set(OCI_SRC
+  oci/spec.cpp
+  )
+
 set(POSIX_SRC
   posix/rlimits.cpp
   )
@@ -452,6 +458,7 @@ set(MESOS_SRC
   ${MASTER_SRC}
   ${MESSAGES_SRC}
   ${MODULE_SRC}
+  ${OCI_SRC}
   ${SCHEDULER_SRC}
   ${STATE_SRC}
   ${URI_SRC}

http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index f432571..cb8e604 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -893,6 +893,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   master/detector/zookeeper.cpp
\
   messages/messages.cpp
\
   module/manager.cpp   \
+  oci/spec.cpp \
   posix/rlimits.cpp\
   sched/sched.cpp  \
   scheduler/scheduler.cpp  \

http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/oci/spec.cpp
--
diff --git a/src/oci/spec.cpp b/src/oci/spec.cpp
new file mode 100644
index 000..f1713a2
--- /dev/null
+++ b/src/oci/spec.cpp
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for 

[1/3] mesos git commit: Add protobuf messages for OCI image spec.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 974581bdb -> 25f4feae4


Add protobuf messages for OCI image spec.

Review: https://reviews.apache.org/r/52349/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6726cd19
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6726cd19
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6726cd19

Branch: refs/heads/master
Commit: 6726cd19cad23480d68f92fe3b6019b9452ab1e7
Parents: 974581b
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Tue Feb 7 16:54:24 2017 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Tue Feb 7 16:54:24 2017 +0800

--
 include/mesos/oci/spec.hpp   |  34 +
 include/mesos/oci/spec.proto | 156 ++
 src/Makefile.am  |  13 
 3 files changed, 203 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/6726cd19/include/mesos/oci/spec.hpp
--
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
new file mode 100644
index 000..ac0b063
--- /dev/null
+++ b/include/mesos/oci/spec.hpp
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_OCI_SPEC_HPP__
+#define __MESOS_OCI_SPEC_HPP__
+
+#include 
+
+namespace oci {
+namespace spec {
+namespace image {
+namespace v1 {
+
+// TODO(qianzhang): Add methods to parse OCI image spec
+
+} // namespace v1 {
+} // namespace image {
+} // namespace spec {
+} // namespace oci {
+
+#endif // __MESOS_OCI_SPEC_HPP__

http://git-wip-us.apache.org/repos/asf/mesos/blob/6726cd19/include/mesos/oci/spec.proto
--
diff --git a/include/mesos/oci/spec.proto b/include/mesos/oci/spec.proto
new file mode 100644
index 000..f3083dd
--- /dev/null
+++ b/include/mesos/oci/spec.proto
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package oci.spec.image.v1;
+
+/**
+ * Protobuf for the OCI content descriptor JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/descriptor.md
+ */
+message Descriptor {
+  // Media types identify their referenced resources, see more details in:
+  // https://github.com/opencontainers/image-spec/blob/master/media-types.md
+  required string mediaType = 1;
+  required string digest = 2;
+  required int64 size = 3;
+  repeated string urls = 4;
+}
+
+
+message Platform {
+  required string architecture = 1;
+  required string os = 2;
+
+  // NOTE: The actual field names in OCI spec are 'os.version' and 
'os.features',
+  // but we can not use them in protobuf message since "." is not allowed in
+  // field name, so these two fields will be handled manually during parsing.
+  optional string os_version = 3;
+  repeated string os_features = 4;
+  optional string variant = 5;
+  repeated string features = 6;
+}
+
+
+message ManifestDescriptor {
+  required string mediaType = 1;
+  required string digest = 2;
+  required int64 size = 3;
+  required Platform platform = 4;
+  repeated string urls = 5;
+}
+
+
+/**
+ * Protobuf for the string-string map field entry.
+ */
+message Label {
+  required string key = 1;
+  requ

[3/3] mesos git commit: Added tests for parsing OCI image spec.

[qianzhang]

Added tests for parsing OCI image spec.

Review: https://reviews.apache.org/r/55140/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/25f4feae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/25f4feae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/25f4feae

Branch: refs/heads/master
Commit: 25f4feae487d53a701adb787fd8a2e5f6166b789
Parents: 3681d61
Author: Qian Zhang 
Authored: Tue Feb 7 16:54:56 2017 +0800
Committer: Qian Zhang 
Committed: Tue Feb 7 16:54:56 2017 +0800

--
 src/Makefile.am|   1 +
 src/tests/CMakeLists.txt   |   1 +
 src/tests/containerizer/oci_spec_tests.cpp | 401 
 3 files changed, 403 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index cb8e604..c21a073 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -2309,6 +2309,7 @@ mesos_tests_SOURCES = 
\
   tests/containerizer/memory_test_helper.cpp   \
   tests/containerizer/mesos_containerizer_tests.cpp\
   tests/containerizer/mesos_containerizer_paths_tests.cpp  \
+  tests/containerizer/oci_spec_tests.cpp   \
   tests/containerizer/posix_rlimits_isolator_tests.cpp \
   tests/containerizer/provisioner_appc_tests.cpp   \
   tests/containerizer/provisioner_backend_tests.cpp\

http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/tests/CMakeLists.txt
--
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index c12a9f7..7898833 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -190,6 +190,7 @@ if (NOT WIN32)
 containerizer/memory_isolator_tests.cpp
 containerizer/mesos_containerizer_paths_tests.cpp
 containerizer/mesos_containerizer_tests.cpp
+containerizer/oci_spec_tests.cpp
 containerizer/posix_rlimits_isolator_tests.cpp
 containerizer/provisioner_appc_tests.cpp
 containerizer/provisioner_backend_tests.cpp

http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/tests/containerizer/oci_spec_tests.cpp
--
diff --git a/src/tests/containerizer/oci_spec_tests.cpp 
b/src/tests/containerizer/oci_spec_tests.cpp
new file mode 100644
index 000..29a8f23
--- /dev/null
+++ b/src/tests/containerizer/oci_spec_tests.cpp
@@ -0,0 +1,401 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include "tests/mesos.hpp"
+
+namespace image = ::oci::spec::image;
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+class OCISpecTest : public ::testing::Test {};
+
+
+TEST_F(OCISpecTest, ParseDescriptor)
+{
+  const string json =
+  R"~(
+  {
+"mediaType": "application/vnd.oci.image.manifest.v1+json",
+"digest": 
"sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+"size": 7682,
+"urls": [
+  "https://example.com/example-manifest;
+]
+  })~";
+
+  Try descriptor =
+  image::v1::parse(json);
+
+  ASSERT_SOME(descriptor);
+
+  EXPECT_EQ(
+  "application/vnd.oci.image.manifest.v1+json",
+  descriptor->mediatype());
+
+  EXPECT_EQ(
+  
"sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+  descriptor->digest());
+
+  EXPECT_EQ(7682u, descriptor->size());
+
+  EXPECT_EQ(
+  "https://example.com/example-manifest;,
+  descriptor->urls(0));
+}
+
+
+TEST_F(OCISpecTest, ParseManifestList)
+{
+  const string json =
+  R"~(
+  {
+"schemaVersion": 2,
+"manifests": [
+  {
+"mediaType": "application/vnd.oci.image.manifest.v1+json",
+

[6/6] mesos git commit: Removed `ProvisionerProcess::__provision()`.

[qianzhang]

Removed `ProvisionerProcess::__provision()`.

Review: https://reviews.apache.org/r/53116


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3b6af1d8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3b6af1d8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3b6af1d8

Branch: refs/heads/master
Commit: 3b6af1d8d336edeefb14f5a86e028c3b099371f5
Parents: 0f4c15a
Author: Qian Zhang 
Authored: Sun Oct 23 09:03:14 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:30 2016 +0800

--
 .../mesos/provisioner/provisioner.cpp   | 118 +--
 .../mesos/provisioner/provisioner.hpp   |   5 -
 2 files changed, 2 insertions(+), 121 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3b6af1d8/src/slave/containerizer/mesos/provisioner/provisioner.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp 
b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
index de2c121..db2d267 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
@@ -311,124 +311,10 @@ Future ProvisionerProcess::_provision(
   imageInfo.layers,
   rootfs,
   backendDir)
-.then(defer(self(), ::__provision, rootfs, image, imageInfo));
-}
-
-
-// This function is currently docker image specific. Depending
-// on docker v1 spec, a docker image may include filesystem
-// changeset, which may need to delete directories or files.
-// The file/directory to be deleted will be labeled by creating
-// a 'whiteout' file, which is at the same location and with the
-// basename of the deleted file or directory prefixed with '.wh.'.
-// For the directory which has an opaque whiteout file '.wh..wh..opq'
-// under it, we need to delete all the files/directories under it.
-// Please see:
-// https://github.com/docker/docker/blob/master/image/spec/v1.md
-// https://github.com/docker/docker/blob/master/pkg/archive/whiteouts.go
-// And OCI image spec also has the concepts 'whiteout' and 'opaque whiteout':
-// https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
-Future ProvisionerProcess::__provision(
-const string& rootfs,
-const Image& image,
-const ImageInfo& imageInfo)
-{
-#ifdef __WINDOWS__
-  return ProvisionInfo{
-  rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
-#else
-  // Skip single-layered images since no 'whiteout' files needs
-  // to be handled, and this excludes any image using the bind
-  // backend.
-  if (imageInfo.layers.size() == 1 || image.type() != Image::DOCKER) {
+.then([=]() -> Future {
   return ProvisionInfo{
   rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
-  }
-
-  // TODO(hausdorff): The FTS API is not available on some platforms, such as
-  // Windows. We will need to either (1) prove that this is not necessary for
-  // Windows Containers, which use much of the Docker spec themselves, or (2)
-  // make this code compatible with Windows, as we did with other code that
-  // depended on FTS, such as `os::rmdir`. See MESOS-5610.
-  char* _rootfs[] = {const_cast(rootfs.c_str()), nullptr};
-
-  FTS* tree = ::fts_open(_rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
-  if (tree == nullptr) {
-return Failure("Failed to open '" + rootfs + "': " + os::strerror(errno));
-  }
-
-  vector whiteout;
-  vector whiteoutOpaque;
-
-  for (FTSENT *node = ::fts_read(tree);
-   node != nullptr; node = ::fts_read(tree)) {
-if (node->fts_info == FTS_F &&
-strings::startsWith(node->fts_name, string(spec::WHITEOUT_PREFIX))) {
-  Path path = Path(node->fts_path);
-  if (node->fts_name == string(spec::WHITEOUT_OPAQUE_PREFIX)) {
-whiteoutOpaque.push_back(path.dirname());
-  } else {
-whiteout.push_back(path::join(
-path.dirname(),
-path.basename().substr(strlen(spec::WHITEOUT_PREFIX;
-  }
-
-  Try rm = os::rm(path.string());
-  if (rm.isError()) {
-::fts_close(tree);
-return Failure(
-"Failed to remove whiteout file '" +
-path.string() + "': " + rm.error());
-  }
-}
-  }
-
-  if (errno != 0) {
-Error error = ErrnoError();
-::fts_close(tree);
-return Failure(error);
-  }
-
-  if (::fts_close(tree) != 0) {
-return Failure(
-"Failed to stop traversing file system: " + os::strerror(errno));
-  }
-
-  foreach (const string& path, whiteoutOpaque) {
-Try rmdir = os::rmdir(path, true, false);
-if (rmdir.isError()) {
-  return Failure(
-  "Failed to remove the entries under the directory labeled as"
-  " opaque 

[2/6] mesos git commit: Added `setxattr()` and `getxattr()` in stout.

[qianzhang]

Added `setxattr()` and `getxattr()` in stout.

Review: https://reviews.apache.org/r/53041


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e3c12c4c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e3c12c4c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e3c12c4c

Branch: refs/heads/master
Commit: e3c12c4c3fabc64c8a3de9e514340bfb52237d69
Parents: 61de698
Author: Qian Zhang 
Authored: Wed Oct 19 16:19:59 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:29 2016 +0800

--
 3rdparty/stout/include/Makefile.am  |  3 +
 3rdparty/stout/include/stout/os.hpp |  1 +
 3rdparty/stout/include/stout/os/posix/xattr.hpp | 99 
 .../stout/include/stout/os/windows/xattr.hpp| 38 
 3rdparty/stout/include/stout/os/xattr.hpp   | 26 +
 5 files changed, 167 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/Makefile.am
--
diff --git a/3rdparty/stout/include/Makefile.am 
b/3rdparty/stout/include/Makefile.am
index 1eb9c14..d5bc728 100644
--- a/3rdparty/stout/include/Makefile.am
+++ b/3rdparty/stout/include/Makefile.am
@@ -104,6 +104,7 @@ nobase_include_HEADERS =\
   stout/os/utime.hpp   \
   stout/os/wait.hpp\
   stout/os/write.hpp   \
+  stout/os/xattr.hpp   \
   stout/os/posix/bootid.hpp\
   stout/os/posix/chown.hpp \
   stout/os/posix/chroot.hpp\
@@ -128,6 +129,7 @@ nobase_include_HEADERS =\
   stout/os/posix/stat.hpp  \
   stout/os/posix/su.hpp\
   stout/os/posix/write.hpp \
+  stout/os/posix/xattr.hpp \
   stout/os/raw/argv.hpp\
   stout/os/raw/environment.hpp \
   stout/os/windows/bootid.hpp  \
@@ -153,6 +155,7 @@ nobase_include_HEADERS =\
   stout/os/windows/stat.hpp\
   stout/os/windows/su.hpp  \
   stout/os/windows/write.hpp   \
+  stout/os/windows/xattr.hpp   \
   stout/path.hpp   \
   stout/preprocessor.hpp   \
   stout/proc.hpp   \

http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os.hpp
--
diff --git a/3rdparty/stout/include/stout/os.hpp 
b/3rdparty/stout/include/stout/os.hpp
index 96e8621..af43bbf 100644
--- a/3rdparty/stout/include/stout/os.hpp
+++ b/3rdparty/stout/include/stout/os.hpp
@@ -64,6 +64,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 

http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os/posix/xattr.hpp
--
diff --git a/3rdparty/stout/include/stout/os/posix/xattr.hpp 
b/3rdparty/stout/include/stout/os/posix/xattr.hpp
new file mode 100644
index 000..518940f
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/posix/xattr.hpp
@@ -0,0 +1,99 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_POSIX_XATTR_HPP__
+#define __STOUT_OS_POSIX_XATTR_HPP__
+
+#include 
+
+namespace os {
+
+inline Try setxattr(
+const std::string& path,
+const std::string& name,
+const std::string& value,
+int flags)
+{
+#ifdef __APPLE__
+  if (::setxattr(
+  path.c_str(),
+  name.c_str(),
+  value.c_str(),
+  value.length(),
+  0,
+  flags) < 0) {
+#else
+  if (::setxattr(
+path.c_str(),
+name.c_str(),
+value.c_str(),
+value.length(),
+flags) < 0) {
+#endif
+return ErrnoError();
+  }
+
+  return Nothing();
+}
+
+
+inline Try getxattr(
+const std::string& path,
+const std::string& name)
+{
+  // Get the current size of the attribute.
+#ifdef __APPLE__
+  ssize_t size = ::getxattr(path.c_str(), name.c_str(), nullptr, 0, 0, 

[4/6] mesos git commit: Implemented the conversion from AUFS whiteouts to OverlayFS whiteouts.

[qianzhang]

Implemented the conversion from AUFS whiteouts to OverlayFS whiteouts.

Review: https://reviews.apache.org/r/53161


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3800cfd0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3800cfd0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3800cfd0

Branch: refs/heads/master
Commit: 3800cfd0d2d65e649b21d079e1c1a4ce743558dd
Parents: ac62ee2
Author: Qian Zhang 
Authored: Tue Oct 25 11:20:26 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:30 2016 +0800

--
 src/Makefile.am |   2 +
 .../mesos/provisioner/docker/store.cpp  |  24 +++-
 .../containerizer/mesos/provisioner/utils.cpp   | 111 +++
 .../containerizer/mesos/provisioner/utils.hpp   |  36 ++
 4 files changed, 169 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index f08b986..5a47c93 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -842,6 +842,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/provisioner/paths.cpp  \
   slave/containerizer/mesos/provisioner/provisioner.cpp
\
   slave/containerizer/mesos/provisioner/store.cpp  \
+  slave/containerizer/mesos/provisioner/utils.cpp  \
   slave/containerizer/mesos/provisioner/appc/cache.cpp \
   slave/containerizer/mesos/provisioner/appc/fetcher.cpp   \
   slave/containerizer/mesos/provisioner/appc/paths.cpp \
@@ -976,6 +977,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/provisioner/paths.hpp  \
   slave/containerizer/mesos/provisioner/provisioner.hpp
\
   slave/containerizer/mesos/provisioner/store.hpp  \
+  slave/containerizer/mesos/provisioner/utils.hpp  \
   slave/containerizer/mesos/provisioner/appc/cache.hpp \
   slave/containerizer/mesos/provisioner/appc/fetcher.hpp   \
   slave/containerizer/mesos/provisioner/appc/paths.hpp \

http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/docker/store.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp 
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index e192f86..9dccd06 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -30,6 +30,9 @@
 
 #include 
 
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
 #include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -331,6 +334,23 @@ Future StoreProcess::moveLayer(
   flags.docker_store_dir,
   layerId);
 
+  const string sourceRootfs = paths::getImageLayerRootfsPath(
+  source,
+  flags.image_provisioner_backend);
+
+#ifdef __linux__
+  // If the backend is "overlay", we need to convert
+  // AUFS whiteout files to OverlayFS whiteout files.
+  if (flags.image_provisioner_backend == OVERLAY_BACKEND) {
+Try convert = convertWhiteouts(sourceRootfs);
+if (convert.isError()) {
+  return Failure(
+  "Failed to convert the whiteout files under '" +
+  sourceRootfs + "': " + convert.error());
+}
+  }
+#endif
+
   if (!os::exists(target)) {
 // This is the case that we pull the layer for the first time.
 Try mkdir = os::mkdir(target);
@@ -349,10 +369,6 @@ Future StoreProcess::moveLayer(
   } else {
 // This is the case where the layer has already been pulled with a
 // different backend.
-const string sourceRootfs = paths::getImageLayerRootfsPath(
-source,
-flags.image_provisioner_backend);
-
 Try rename = os::rename(sourceRootfs, targetRootfs);
 if (rename.isError()) {
   return Failure(

http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/utils.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/utils.cpp 
b/src/slave/containerizer/mesos/provisioner/utils.cpp
new file mode 100644
index 000..340cf48
--- /dev/null
+++ 

[3/6] mesos git commit: Added `FsTest.Xattr` test.

[qianzhang]

Added `FsTest.Xattr` test.

Review: https://reviews.apache.org/r/53042


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/50f6c117
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/50f6c117
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/50f6c117

Branch: refs/heads/master
Commit: 50f6c117e3874efbefe605b51cd3d3efb2cf64ba
Parents: e3c12c4
Author: Qian Zhang 
Authored: Wed Oct 19 21:39:02 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:29 2016 +0800

--
 3rdparty/stout/tests/os/filesystem_tests.cpp | 36 +++
 1 file changed, 36 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/50f6c117/3rdparty/stout/tests/os/filesystem_tests.cpp
--
diff --git a/3rdparty/stout/tests/os/filesystem_tests.cpp 
b/3rdparty/stout/tests/os/filesystem_tests.cpp
index e3894ef..5c8f422 100644
--- a/3rdparty/stout/tests/os/filesystem_tests.cpp
+++ b/3rdparty/stout/tests/os/filesystem_tests.cpp
@@ -31,6 +31,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 
@@ -436,3 +437,38 @@ TEST_F(FsTest, Close)
   _CrtSetReportMode(_CRT_ASSERT, previous_report_mode);
 #endif // __WINDOWS__
 }
+
+
+TEST_F(FsTest, Xattr)
+{
+  const string file = path::join(os::getcwd(), UUID::random().toString());
+
+  // Create file.
+  ASSERT_SOME(os::touch(file));
+  ASSERT_TRUE(os::exists(file));
+
+  // Set an extended attribute.
+  Try setxattr = os::setxattr(
+  file,
+  "user.mesos.test",
+  "y",
+  0);
+
+  // Only run this test if extended attribute is supported.
+  if (setxattr.isError() && setxattr.error() == os::strerror(ENOTSUP)) {
+return;
+  }
+
+  ASSERT_SOME(setxattr);
+
+  // Get the extended attribute.
+  Try value = os::getxattr(file, "user.mesos.test");
+  ASSERT_SOME(value);
+  EXPECT_EQ(value.get(), "y");
+
+  // Remove the extended attribute.
+  ASSERT_SOME(os::removexattr(file, "user.mesos.test"));
+
+  // Get the extended attribute again which should not exist.
+  ASSERT_ERROR(os::getxattr(file, "user.mesos.test"));
+}

[5/6] mesos git commit: Implemented handling AUFS whiteouts for copy backend.

[qianzhang]

Implemented handling AUFS whiteouts for copy backend.

Review: https://reviews.apache.org/r/53115


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0f4c15ad
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0f4c15ad
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0f4c15ad

Branch: refs/heads/master
Commit: 0f4c15ad735e437a8e8b9bb6846025aea02c765b
Parents: 3800cfd
Author: Qian Zhang 
Authored: Sat Oct 22 21:45:44 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:30 2016 +0800

--
 .../mesos/provisioner/backends/copy.cpp | 103 ++-
 1 file changed, 98 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/0f4c15ad/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp 
b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
index 9c5354e..7e98110 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
@@ -16,6 +16,8 @@
 
 #include 
 
+#include 
+
 #include 
 #include 
 #include 
@@ -24,7 +26,6 @@
 #include 
 #include 
 
-
 #include 
 #include 
 
@@ -32,7 +33,6 @@
 
 #include "slave/containerizer/mesos/provisioner/backends/copy.hpp"
 
-
 using namespace process;
 
 using std::string;
@@ -128,9 +128,92 @@ Future CopyBackendProcess::provision(
 
 
 Future CopyBackendProcess::_provision(
-  string layer,
-  const string& rootfs)
+string layer,
+const string& rootfs)
 {
+  // Traverse the layer to check if there is any whiteout files, if
+  // yes, remove the corresponding files/directories from the rootfs.
+  // Note: We assume all image types use AUFS whiteout format.
+  char* source[] = {const_cast(layer.c_str()), nullptr};
+
+  FTS* tree = ::fts_open(source, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+  if (tree == nullptr) {
+return Failure("Failed to open '" + layer + "': " + os::strerror(errno));
+  }
+
+  vector whiteouts;
+  for (FTSENT *node = ::fts_read(tree);
+   node != nullptr; node = ::fts_read(tree)) {
+if (node->fts_info != FTS_F) {
+  continue;
+}
+
+if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+  continue;
+}
+
+string ftsPath = string(node->fts_path);
+Path whiteout = Path(ftsPath.substr(layer.length() + 1));
+
+// Keep the relative paths of the whiteout files, we will
+// remove them from rootfs after layer is copied to rootfs.
+whiteouts.push_back(whiteout.string());
+
+if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+  const string path = path::join(rootfs, Path(whiteout).dirname());
+
+  // Remove the entries under the directory labeled
+  // as opaque whiteout from rootfs.
+  Try rmdir = os::rmdir(path, true, false);
+  if (rmdir.isError()) {
+::fts_close(tree);
+return Failure(
+"Failed to remove the entries under the directory labeled as"
+" opaque whiteout '" + path + "': " + rmdir.error());
+  }
+} else {
+  const string path = path::join(
+  rootfs,
+  whiteout.dirname(),
+  whiteout.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+  // The file/directory labeled as whiteout may have already been
+  // removed with the code above due to its parent directory labeled
+  // as opaque whiteout, so here we need to check if it still exists
+  // before trying to remove it.
+  if (os::exists(path)) {
+if (os::stat::isdir(path)) {
+  Try rmdir = os::rmdir(path);
+  if (rmdir.isError()) {
+::fts_close(tree);
+return Failure(
+"Failed to remove the directory labeled as whiteout '" +
+path + "': " + rmdir.error());
+  }
+} else {
+  Try rm = os::rm(path);
+  if (rm.isError()) {
+::fts_close(tree);
+return Failure(
+"Failed to remove the file labeled as whiteout '" +
+path + "': " + rm.error());
+  }
+}
+  }
+}
+  }
+
+  if (errno != 0) {
+Error error = ErrnoError();
+::fts_close(tree);
+return Failure(error);
+  }
+
+  if (::fts_close(tree) != 0) {
+return Failure(
+"Failed to stop traversing file system: " + os::strerror(errno));
+  }
+
   VLOG(1) << "Copying layer path '" << layer << "' to rootfs '" << rootfs
   << "'";
 
@@ -160,7 +243,7 @@ Future CopyBackendProcess::_provision(
   Subprocess cp = s.get();
 
   return cp.status()
-.then([cp](const Option& status) -> 

[1/6] mesos git commit: Divided utils.hpp to utils.hpp and utils.cpp.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 61de69849 -> 3b6af1d8d


Divided utils.hpp to utils.hpp and utils.cpp.

Review: https://reviews.apache.org/r/53053


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ac62ee27
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ac62ee27
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ac62ee27

Branch: refs/heads/master
Commit: ac62ee27f3f2d226732da439a74ed33aed7c071f
Parents: 50f6c11
Author: Qian Zhang 
Authored: Thu Oct 20 17:34:43 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:18:29 2016 +0800

--
 src/Makefile.am |  1 +
 src/slave/containerizer/mesos/utils.cpp | 40 
 src/slave/containerizer/mesos/utils.hpp | 15 +--
 3 files changed, 42 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index a5569ec..f08b986 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -829,6 +829,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/launcher.cpp   \
   slave/containerizer/mesos/mount.cpp  \
   slave/containerizer/mesos/paths.cpp  \
+  slave/containerizer/mesos/utils.cpp  \
   slave/containerizer/mesos/isolators/docker/volume/driver.cpp \
   slave/containerizer/mesos/isolators/docker/volume/paths.cpp  \
   slave/containerizer/mesos/isolators/filesystem/posix.cpp \

http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/slave/containerizer/mesos/utils.cpp
--
diff --git a/src/slave/containerizer/mesos/utils.cpp 
b/src/slave/containerizer/mesos/utils.cpp
new file mode 100644
index 000..237aea4
--- /dev/null
+++ b/src/slave/containerizer/mesos/utils.cpp
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "slave/containerizer/mesos/utils.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+ContainerID getRootContainerId(const ContainerID& containerId)
+{
+  ContainerID rootContainerId = containerId;
+  while (rootContainerId.has_parent()) {
+// NOTE: Looks like protobuf does not handle copying well when
+// nesting message is involved, because the source and the target
+// point to the same object. Therefore, we create a temporary
+// variable and use an extra copy here.
+ContainerID id = rootContainerId.parent();
+rootContainerId = id;
+  }
+
+  return rootContainerId;
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {

http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/slave/containerizer/mesos/utils.hpp
--
diff --git a/src/slave/containerizer/mesos/utils.hpp 
b/src/slave/containerizer/mesos/utils.hpp
index 178ebf3..f24215b 100644
--- a/src/slave/containerizer/mesos/utils.hpp
+++ b/src/slave/containerizer/mesos/utils.hpp
@@ -23,20 +23,7 @@ namespace mesos {
 namespace internal {
 namespace slave {
 
-static ContainerID getRootContainerId(const ContainerID& containerId)
-{
-  ContainerID rootContainerId = containerId;
-  while (rootContainerId.has_parent()) {
-// NOTE: Looks like protobuf does not handle copying well when
-// nesting message is involved, because the source and the target
-// point to the same object. Therefore, we create a temporary
-// variable and use an extra copy here.
-ContainerID id = rootContainerId.parent();
-rootContainerId = id;
-  }
-
-  return rootContainerId;
-}
+ContainerID getRootContainerId(const ContainerID& containerId);
 
 } // namespace slave {
 } // namespace internal {

[3/3] mesos git commit: Fixed a typo in the header files of stout.

[qianzhang]

Fixed a typo in the header files of stout.

Review: https://reviews.apache.org/r/53044/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/69abbec9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/69abbec9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/69abbec9

Branch: refs/heads/master
Commit: 69abbec9b7f5f79f77613d1c80104702d1758a6d
Parents: dc7f3f6
Author: Qian Zhang 
Authored: Sat Nov 5 16:57:21 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:57:21 2016 +0800

--
 3rdparty/stout/include/stout/dynamiclibrary.hpp | 2 +-
 3rdparty/stout/include/stout/fs.hpp | 2 +-
 3rdparty/stout/include/stout/os.hpp | 2 +-
 3rdparty/stout/include/stout/os/bootid.hpp  | 2 +-
 3rdparty/stout/include/stout/os/exists.hpp  | 2 +-
 3rdparty/stout/include/stout/os/fcntl.hpp   | 2 +-
 3rdparty/stout/include/stout/os/fork.hpp| 2 +-
 3rdparty/stout/include/stout/os/fsync.hpp   | 2 +-
 3rdparty/stout/include/stout/os/ftruncate.hpp   | 2 +-
 3rdparty/stout/include/stout/os/kill.hpp| 2 +-
 3rdparty/stout/include/stout/os/killtree.hpp| 2 +-
 3rdparty/stout/include/stout/os/mkdtemp.hpp | 2 +-
 3rdparty/stout/include/stout/os/pagesize.hpp| 2 +-
 3rdparty/stout/include/stout/os/rm.hpp  | 2 +-
 3rdparty/stout/include/stout/os/rmdir.hpp   | 2 +-
 3rdparty/stout/include/stout/os/sendfile.hpp| 2 +-
 3rdparty/stout/include/stout/os/shell.hpp   | 2 +-
 3rdparty/stout/include/stout/os/signals.hpp | 2 +-
 3rdparty/stout/include/stout/os/stat.hpp| 2 +-
 19 files changed, 19 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/69abbec9/3rdparty/stout/include/stout/dynamiclibrary.hpp
--
diff --git a/3rdparty/stout/include/stout/dynamiclibrary.hpp 
b/3rdparty/stout/include/stout/dynamiclibrary.hpp
index e5b2d00..37520d0 100644
--- a/3rdparty/stout/include/stout/dynamiclibrary.hpp
+++ b/3rdparty/stout/include/stout/dynamiclibrary.hpp
@@ -14,7 +14,7 @@
 #define __STOUT_DYNAMICLIBRARY_HPP__
 
 // For readability, we minimize the number of #ifdef blocks in the code by
-// splitting platform specifc system calls into separate directories.
+// splitting platform specific system calls into separate directories.
 #ifdef __WINDOWS__
 #include 
 #else

http://git-wip-us.apache.org/repos/asf/mesos/blob/69abbec9/3rdparty/stout/include/stout/fs.hpp
--
diff --git a/3rdparty/stout/include/stout/fs.hpp 
b/3rdparty/stout/include/stout/fs.hpp
index 0e11fb8..f18b4fb 100644
--- a/3rdparty/stout/include/stout/fs.hpp
+++ b/3rdparty/stout/include/stout/fs.hpp
@@ -14,7 +14,7 @@
 #define __STOUT_FS_HPP__
 
 // For readability, we minimize the number of #ifdef blocks in the code by
-// splitting platform specifc system calls into separate directories.
+// splitting platform specific system calls into separate directories.
 #ifdef __WINDOWS__
 #include 
 #else

http://git-wip-us.apache.org/repos/asf/mesos/blob/69abbec9/3rdparty/stout/include/stout/os.hpp
--
diff --git a/3rdparty/stout/include/stout/os.hpp 
b/3rdparty/stout/include/stout/os.hpp
index af43bbf..bd085e4 100644
--- a/3rdparty/stout/include/stout/os.hpp
+++ b/3rdparty/stout/include/stout/os.hpp
@@ -70,7 +70,7 @@
 #include 
 
 // For readability, we minimize the number of #ifdef blocks in the code by
-// splitting platform specifc system calls into separate directories.
+// splitting platform specific system calls into separate directories.
 #ifdef __WINDOWS__
 #include 
 #else

http://git-wip-us.apache.org/repos/asf/mesos/blob/69abbec9/3rdparty/stout/include/stout/os/bootid.hpp
--
diff --git a/3rdparty/stout/include/stout/os/bootid.hpp 
b/3rdparty/stout/include/stout/os/bootid.hpp
index ec4b1ad..3204c7f 100644
--- a/3rdparty/stout/include/stout/os/bootid.hpp
+++ b/3rdparty/stout/include/stout/os/bootid.hpp
@@ -15,7 +15,7 @@
 
 
 // For readability, we minimize the number of #ifdef blocks in the code by
-// splitting platform specifc system calls into separate directories.
+// splitting platform specific system calls into separate directories.
 #ifdef __WINDOWS__
 #include 
 #else

http://git-wip-us.apache.org/repos/asf/mesos/blob/69abbec9/3rdparty/stout/include/stout/os/exists.hpp
--
diff --git a/3rdparty/stout/include/stout/os/exists.hpp 
b/3rdparty/stout/include/stout/os/exists.hpp
index 7b0fbaf..e09fcc5 100644
--- a/3rdparty/stout/include/stout/os/exists.hpp
+++ 

[2/3] mesos git commit: Fixed a typo in `executor.hpp`.

[qianzhang]

Fixed a typo in `executor.hpp`.

Review: https://reviews.apache.org/r/53045/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/dc7f3f6e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/dc7f3f6e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/dc7f3f6e

Branch: refs/heads/master
Commit: dc7f3f6eb84598887c91dc7027895a80b9130365
Parents: 8ccf7e8
Author: Qian Zhang 
Authored: Sat Nov 5 16:55:51 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:55:51 2016 +0800

--
 src/launcher/executor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/dc7f3f6e/src/launcher/executor.hpp
--
diff --git a/src/launcher/executor.hpp b/src/launcher/executor.hpp
index 217680d..c7c134a 100644
--- a/src/launcher/executor.hpp
+++ b/src/launcher/executor.hpp
@@ -19,7 +19,7 @@
 
 
 // For readability, we minimize the number of #ifdef blocks in the code by
-// splitting platform specifc system calls into separate directories.
+// splitting platform specific system calls into separate directories.
 #ifdef __WINDOWS__
 #include "launcher/windows/executor.hpp"
 #else

[1/3] mesos git commit: Fixed a comment in `stat.hpp`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 3b6af1d8d -> 69abbec9b


Fixed a comment in `stat.hpp`.

Review: https://reviews.apache.org/r/53018/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8ccf7e81
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8ccf7e81
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8ccf7e81

Branch: refs/heads/master
Commit: 8ccf7e81db932445269b329b5d811e0594d54d78
Parents: 3b6af1d
Author: Qian Zhang 
Authored: Sat Nov 5 16:54:43 2016 +0800
Committer: Qian Zhang 
Committed: Sat Nov 5 16:54:43 2016 +0800

--
 3rdparty/stout/include/stout/os/posix/stat.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/8ccf7e81/3rdparty/stout/include/stout/os/posix/stat.hpp
--
diff --git a/3rdparty/stout/include/stout/os/posix/stat.hpp 
b/3rdparty/stout/include/stout/os/posix/stat.hpp
index 32fa426..1ab20e7 100644
--- a/3rdparty/stout/include/stout/os/posix/stat.hpp
+++ b/3rdparty/stout/include/stout/os/posix/stat.hpp
@@ -172,4 +172,4 @@ inline Try inode(const std::string& path)
 
 } // namespace os {
 
-#endif // __STOUT_OS_STAT_HPP__
+#endif // __STOUT_OS_POSIX_STAT_HPP__

mesos git commit: Added the test `ProvisionerDockerWhiteoutTest`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master f56b5c013 -> 9cea997c8


Added the test `ProvisionerDockerWhiteoutTest`.

Review: https://reviews.apache.org/r/53127


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9cea997c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9cea997c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9cea997c

Branch: refs/heads/master
Commit: 9cea997c811b869d54b5c892710efbf074b08e3a
Parents: f56b5c0
Author: Qian Zhang 
Authored: Mon Oct 24 16:23:12 2016 +0800
Committer: Qian Zhang 
Committed: Sun Nov 6 09:42:09 2016 +0800

--
 .../containerizer/provisioner_docker_tests.cpp  | 95 ++--
 1 file changed, 70 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/9cea997c/src/tests/containerizer/provisioner_docker_tests.cpp
--
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp 
b/src/tests/containerizer/provisioner_docker_tests.cpp
index d8fb7ab..9682169 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -28,6 +28,12 @@
 
 #include 
 
+#ifdef __linux__
+#include "linux/fs.hpp"
+#endif
+
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+
 #include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -55,6 +61,10 @@ using process::Promise;
 
 using master::Master;
 
+using mesos::internal::slave::AUFS_BACKEND;
+using mesos::internal::slave::COPY_BACKEND;
+using mesos::internal::slave::OVERLAY_BACKEND;
+
 using mesos::master::detector::MasterDetector;
 
 using slave::ImageInfo;
@@ -64,6 +74,8 @@ using slave::docker::Puller;
 using slave::docker::RegistryPuller;
 using slave::docker::Store;
 
+using testing::WithParamInterface;
+
 namespace mesos {
 namespace internal {
 namespace tests {
@@ -552,9 +564,10 @@ TEST_F(ProvisionerDockerPullerTest, 
ROOT_INTERNET_CURL_Normalize)
 }
 
 
-// This test verifies that any docker image containing whiteout files
-// will be processed correctly.
-TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Whiteout)
+// This test verifies that the scratch based docker image (that
+// only contain a single binary and its dependencies) can be
+// launched correctly.
+TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
 {
   Try master = StartMaster();
   ASSERT_SOME(master);
@@ -585,17 +598,8 @@ TEST_F(ProvisionerDockerPullerTest, 
ROOT_INTERNET_CURL_Whiteout)
 
   const Offer& offer = offers.get()[0];
 
-  // We are using the docker image 'cirros' to verify that
-  // the symlink /etc/rc3.d/S40-network does not exist in
-  // container's rootfs because it is labeled by a '.wh.'
-  // whiteout file, and both should be removed.
   CommandInfo command;
   command.set_shell(false);
-  command.set_value("/usr/bin/test");
-  command.add_arguments("test");
-  command.add_arguments("!");
-  command.add_arguments("-f");
-  command.add_arguments("/etc/rc3.d/S40-network");
 
   TaskInfo task = createTask(
   offer.slave_id(),
@@ -604,7 +608,10 @@ TEST_F(ProvisionerDockerPullerTest, 
ROOT_INTERNET_CURL_Whiteout)
 
   Image image;
   image.set_type(Image::DOCKER);
-  image.mutable_docker()->set_name("cirros");
+
+  // 'hello-world' is a scratch image. It contains only one
+  // binary 'hello' in its rootfs.
+  image.mutable_docker()->set_name("hello-world");
 
   ContainerInfo* container = task.mutable_container();
   container->set_type(ContainerInfo::MESOS);
@@ -631,10 +638,40 @@ TEST_F(ProvisionerDockerPullerTest, 
ROOT_INTERNET_CURL_Whiteout)
 }
 
 
-// This test verifies that the scratch based docker image (that
-// only contain a single binary and its dependencies) can be
-// launched correctly.
-TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
+class ProvisionerDockerWhiteoutTest
+  : public MesosTest,
+public WithParamInterface
+{
+public:
+  // Returns the supported backends.
+  static vector parameters()
+  {
+vector backends = {COPY_BACKEND};
+
+Try aufsSupported = fs::supported("aufs");
+if (aufsSupported.isSome() && aufsSupported.get()) {
+  backends.push_back(AUFS_BACKEND);
+}
+
+Try overlayfsSupported = fs::supported("overlayfs");
+if (overlayfsSupported.isSome() && overlayfsSupported.get()) {
+  backends.push_back(OVERLAY_BACKEND);
+}
+
+return backends;
+  }
+};
+
+
+INSTANTIATE_TEST_CASE_P(
+BackendFlag,
+ProvisionerDockerWhiteoutTest,
+::testing::ValuesIn(ProvisionerDockerWhiteoutTest::parameters()));
+
+
+// 

mesos git commit: Fixed the broken cmake build.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 69abbec9b -> f56b5c013


Fixed the broken cmake build.

Review: https://reviews.apache.org/r/53513


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f56b5c01
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f56b5c01
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f56b5c01

Branch: refs/heads/master
Commit: f56b5c013a5cc76d66fd5cb3b3ac9c2dba647b5c
Parents: 69abbec
Author: Qian Zhang 
Authored: Sun Nov 6 08:52:41 2016 +0800
Committer: Qian Zhang 
Committed: Sun Nov 6 08:54:54 2016 +0800

--
 src/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f56b5c01/src/CMakeLists.txt
--
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 75c3a2b..aef9ae6 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -325,9 +325,11 @@ set(AGENT_SRC
   slave/containerizer/mesos/launcher.cpp
   slave/containerizer/mesos/mount.cpp
   slave/containerizer/mesos/paths.cpp
+  slave/containerizer/mesos/utils.cpp
   slave/containerizer/mesos/provisioner/paths.cpp
   slave/containerizer/mesos/provisioner/provisioner.cpp
   slave/containerizer/mesos/provisioner/store.cpp
+  slave/containerizer/mesos/provisioner/utils.cpp
   slave/containerizer/mesos/provisioner/appc/cache.cpp
   slave/containerizer/mesos/provisioner/appc/fetcher.cpp
   slave/containerizer/mesos/provisioner/appc/paths.cpp

[4/4] mesos git commit: Added backport of r52910 to 1.1.x.

[qianzhang]

Added backport of r52910 to 1.1.x.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b7f0bc19
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b7f0bc19
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b7f0bc19

Branch: refs/heads/1.1.x
Commit: b7f0bc1943bb5cbe5830df987a9888d90bbcd18e
Parents: de3a1b5
Author: Qian Zhang 
Authored: Wed Nov 23 17:49:17 2016 +0800
Committer: Qian Zhang 
Committed: Thu Nov 24 09:37:56 2016 +0800

--
 src/Makefile.am |  1 +
 .../containerizer/mesos/provisioner/backend.cpp |  9 ++---
 .../mesos/provisioner/constants.hpp | 34 ++
 .../mesos/provisioner/docker/paths.cpp  |  4 ++-
 src/slave/flags.cpp |  4 ++-
 .../containerizer/provisioner_appc_tests.cpp| 13 ---
 .../containerizer/provisioner_backend_tests.cpp | 37 
 7 files changed, 76 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index 3bcc0f2..73d337d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -967,6 +967,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/isolators/volume/sandbox_path.hpp  \
   slave/containerizer/mesos/isolators/windows.hpp  \
   slave/containerizer/mesos/provisioner/backend.hpp\
+  slave/containerizer/mesos/provisioner/constants.hpp  \
   slave/containerizer/mesos/provisioner/paths.hpp  \
   slave/containerizer/mesos/provisioner/provisioner.hpp
\
   slave/containerizer/mesos/provisioner/store.hpp  \

http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/containerizer/mesos/provisioner/backend.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/backend.cpp 
b/src/slave/containerizer/mesos/provisioner/backend.cpp
index 9c258ed..157967f 100644
--- a/src/slave/containerizer/mesos/provisioner/backend.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backend.cpp
@@ -23,6 +23,7 @@
 #endif
 
 #include "slave/containerizer/mesos/provisioner/backend.hpp"
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
 
 #ifdef __linux__
 #include "slave/containerizer/mesos/provisioner/backends/aufs.hpp"
@@ -46,14 +47,14 @@ hashmap Backend::create(const 
Flags& flags)
   hashmap(*)(const Flags&)> creators;
 
 #ifdef __linux__
-  creators.put("bind", ::create);
+  creators.put(BIND_BACKEND, ::create);
 
   Try aufsSupported = fs::supported("aufs");
   if (aufsSupported.isError()) {
 LOG(WARNING) << "Failed to check aufs availability: '"
  << aufsSupported.error();
   } else if (aufsSupported.get()) {
-creators.put("aufs", ::create);
+creators.put(AUFS_BACKEND, ::create);
   }
 
   Try overlayfsSupported = fs::supported("overlayfs");
@@ -61,11 +62,11 @@ hashmap Backend::create(const 
Flags& flags)
 LOG(WARNING) << "Failed to check overlayfs availability: '"
  << overlayfsSupported.error();
   } else if (overlayfsSupported.get()) {
-creators.put("overlay", ::create);
+creators.put(OVERLAY_BACKEND, ::create);
   }
 #endif // __linux__
 
-  creators.put("copy", ::create);
+  creators.put(COPY_BACKEND, ::create);
 
   hashmap backends;
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/containerizer/mesos/provisioner/constants.hpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/constants.hpp 
b/src/slave/containerizer/mesos/provisioner/constants.hpp
new file mode 100644
index 000..ab488eb
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/constants.hpp
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 

[1/4] mesos git commit: Added backport of MESOS-6360 (Mesos part) to 1.1.x.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.1.x de3a1b58a -> b810e22d3


Added backport of MESOS-6360 (Mesos part) to 1.1.x.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b810e22d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b810e22d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b810e22d

Branch: refs/heads/1.1.x
Commit: b810e22d37cffd4431599ea75ec509db56d10460
Parents: 2d65576
Author: Qian Zhang 
Authored: Wed Nov 23 17:39:33 2016 +0800
Committer: Qian Zhang 
Committed: Thu Nov 24 09:37:56 2016 +0800

--
 src/CMakeLists.txt  |   2 +
 src/Makefile.am |   3 +
 .../mesos/provisioner/backends/copy.cpp | 108 ++-
 .../mesos/provisioner/docker/store.cpp  |  24 +++-
 .../mesos/provisioner/provisioner.cpp   | 118 +---
 .../mesos/provisioner/provisioner.hpp   |   5 -
 .../containerizer/mesos/provisioner/utils.cpp   | 133 +++
 .../containerizer/mesos/provisioner/utils.hpp   |  37 ++
 src/slave/containerizer/mesos/utils.cpp |  40 ++
 src/slave/containerizer/mesos/utils.hpp |  15 +--
 .../containerizer/provisioner_docker_tests.cpp  |  95 +
 11 files changed, 411 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/CMakeLists.txt
--
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e60d34c..85acbbe 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -325,9 +325,11 @@ set(AGENT_SRC
   slave/containerizer/mesos/launcher.cpp
   slave/containerizer/mesos/mount.cpp
   slave/containerizer/mesos/paths.cpp
+  slave/containerizer/mesos/utils.cpp
   slave/containerizer/mesos/provisioner/paths.cpp
   slave/containerizer/mesos/provisioner/provisioner.cpp
   slave/containerizer/mesos/provisioner/store.cpp
+  slave/containerizer/mesos/provisioner/utils.cpp
   slave/containerizer/mesos/provisioner/appc/cache.cpp
   slave/containerizer/mesos/provisioner/appc/fetcher.cpp
   slave/containerizer/mesos/provisioner/appc/paths.cpp

http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index 73d337d..f82c985 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -828,6 +828,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/launcher.cpp   \
   slave/containerizer/mesos/mount.cpp  \
   slave/containerizer/mesos/paths.cpp  \
+  slave/containerizer/mesos/utils.cpp  \
   slave/containerizer/mesos/isolators/docker/volume/driver.cpp \
   slave/containerizer/mesos/isolators/docker/volume/paths.cpp  \
   slave/containerizer/mesos/isolators/filesystem/posix.cpp \
@@ -839,6 +840,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/provisioner/paths.cpp  \
   slave/containerizer/mesos/provisioner/provisioner.cpp
\
   slave/containerizer/mesos/provisioner/store.cpp  \
+  slave/containerizer/mesos/provisioner/utils.cpp  \
   slave/containerizer/mesos/provisioner/appc/cache.cpp \
   slave/containerizer/mesos/provisioner/appc/fetcher.cpp   \
   slave/containerizer/mesos/provisioner/appc/paths.cpp \
@@ -971,6 +973,7 @@ libmesos_no_3rdparty_la_SOURCES +=  
\
   slave/containerizer/mesos/provisioner/paths.hpp  \
   slave/containerizer/mesos/provisioner/provisioner.hpp
\
   slave/containerizer/mesos/provisioner/store.hpp  \
+  slave/containerizer/mesos/provisioner/utils.hpp  \
   slave/containerizer/mesos/provisioner/appc/cache.hpp \
   slave/containerizer/mesos/provisioner/appc/fetcher.hpp   \
   slave/containerizer/mesos/provisioner/appc/paths.hpp \

http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp 
b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
index 9c5354e..0ce3e1e 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
+++ 

[3/4] mesos git commit: Added backport of MESOS-6360 (stout part) to 1.1.x.

[qianzhang]

Added backport of MESOS-6360 (stout part) to 1.1.x.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2d655760
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2d655760
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2d655760

Branch: refs/heads/1.1.x
Commit: 2d6557601ead04d51b9d5ca461c2c20c3f5f65a4
Parents: 14355b7
Author: Qian Zhang 
Authored: Wed Nov 23 17:38:11 2016 +0800
Committer: Qian Zhang 
Committed: Thu Nov 24 09:37:56 2016 +0800

--
 3rdparty/stout/include/Makefile.am  |  3 +
 3rdparty/stout/include/stout/os.hpp |  1 +
 3rdparty/stout/include/stout/os/posix/xattr.hpp | 99 
 .../stout/include/stout/os/windows/xattr.hpp| 38 
 3rdparty/stout/include/stout/os/xattr.hpp   | 26 +
 3rdparty/stout/tests/os/filesystem_tests.cpp| 38 
 6 files changed, 205 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/Makefile.am
--
diff --git a/3rdparty/stout/include/Makefile.am 
b/3rdparty/stout/include/Makefile.am
index 1eb9c14..d5bc728 100644
--- a/3rdparty/stout/include/Makefile.am
+++ b/3rdparty/stout/include/Makefile.am
@@ -104,6 +104,7 @@ nobase_include_HEADERS =\
   stout/os/utime.hpp   \
   stout/os/wait.hpp\
   stout/os/write.hpp   \
+  stout/os/xattr.hpp   \
   stout/os/posix/bootid.hpp\
   stout/os/posix/chown.hpp \
   stout/os/posix/chroot.hpp\
@@ -128,6 +129,7 @@ nobase_include_HEADERS =\
   stout/os/posix/stat.hpp  \
   stout/os/posix/su.hpp\
   stout/os/posix/write.hpp \
+  stout/os/posix/xattr.hpp \
   stout/os/raw/argv.hpp\
   stout/os/raw/environment.hpp \
   stout/os/windows/bootid.hpp  \
@@ -153,6 +155,7 @@ nobase_include_HEADERS =\
   stout/os/windows/stat.hpp\
   stout/os/windows/su.hpp  \
   stout/os/windows/write.hpp   \
+  stout/os/windows/xattr.hpp   \
   stout/path.hpp   \
   stout/preprocessor.hpp   \
   stout/proc.hpp   \

http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os.hpp
--
diff --git a/3rdparty/stout/include/stout/os.hpp 
b/3rdparty/stout/include/stout/os.hpp
index 96e8621..af43bbf 100644
--- a/3rdparty/stout/include/stout/os.hpp
+++ b/3rdparty/stout/include/stout/os.hpp
@@ -64,6 +64,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 

http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os/posix/xattr.hpp
--
diff --git a/3rdparty/stout/include/stout/os/posix/xattr.hpp 
b/3rdparty/stout/include/stout/os/posix/xattr.hpp
new file mode 100644
index 000..518940f
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/posix/xattr.hpp
@@ -0,0 +1,99 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_POSIX_XATTR_HPP__
+#define __STOUT_OS_POSIX_XATTR_HPP__
+
+#include 
+
+namespace os {
+
+inline Try setxattr(
+const std::string& path,
+const std::string& name,
+const std::string& value,
+int flags)
+{
+#ifdef __APPLE__
+  if (::setxattr(
+  path.c_str(),
+  name.c_str(),
+  value.c_str(),
+  value.length(),
+  0,
+  flags) < 0) {
+#else
+  if (::setxattr(
+path.c_str(),
+name.c_str(),
+value.c_str(),
+value.length(),
+flags) < 0) {
+#endif
+return ErrnoError();
+  }
+
+  return Nothing();
+}
+
+
+inline Try getxattr(
+const std::string& path,
+const std::string& name)
+{
+  // Get the current size of the attribute.
+#ifdef __APPLE__
+  ssize_t size = ::getxattr(path.c_str(), 

[2/4] mesos git commit: Added backport of MESOS-6002 to 1.1.x.

[qianzhang]

Added backport of MESOS-6002 to 1.1.x.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/14355b75
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/14355b75
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/14355b75

Branch: refs/heads/1.1.x
Commit: 14355b757ac0dd39db1928c00cb9dad5fd2deb65
Parents: b7f0bc1
Author: Qian Zhang 
Authored: Wed Nov 23 20:41:10 2016 +0800
Committer: Qian Zhang 
Committed: Thu Nov 24 09:37:56 2016 +0800

--
 .../mesos/provisioner/backends/aufs.cpp | 16 ++--
 1 file changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/14355b75/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
--
diff --git a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp 
b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
index c69dc42..c92c6c7 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
@@ -145,12 +145,16 @@ Future AufsBackendProcess::provision(
 
   // See http://aufs.sourceforge.net/aufs2/man.html
   // for the mount syntax for aufs.
-  string options = "dirs=" + workdir + ":";
-
-  // For aufs, the specified lower directories will be stacked
-  // beginning from the rightmost one and going left. But we need the
-  // first layer in the vector to be the bottom most layer.
-  options += strings::join(":", adaptor::reverse(layers));
+  string options = "dirs=" + workdir + "=rw";
+
+  // For aufs, the specified lower directories will be stacked beginning
+  // from the rightmost one and going left. But we need the first layer
+  // in the vector to be the bottom most layer. And for each layer, we
+  // need to mount it with the option 'ro+wh' so that it will be read-only
+  // and its whiteout files (if any) will be well handled.
+  foreach (const string& layer, adaptor::reverse(layers)) {
+options += ":" + layer + "=ro+wh";
+  }
 
   VLOG(1) << "Provisioning image rootfs with aufs: '" << options << "'";
 

[2/2] mesos git commit: Added '--task' into mesos-execute.

[qianzhang]

Added '--task' into mesos-execute.

Review: https://reviews.apache.org/r/53645/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/77da99a9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/77da99a9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/77da99a9

Branch: refs/heads/master
Commit: 77da99a9ca2d7bd5ba61c276290c1cb60bb55445
Parents: bf52cc1
Author: Qian Zhang 
Authored: Tue Nov 22 09:32:21 2016 +0800
Committer: Qian Zhang 
Committed: Tue Nov 22 09:32:21 2016 +0800

--
 src/cli/execute.cpp | 78 +++-
 1 file changed, 64 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/77da99a9/src/cli/execute.cpp
--
diff --git a/src/cli/execute.cpp b/src/cli/execute.cpp
index dac895c..ddf7eca 100644
--- a/src/cli/execute.cpp
+++ b/src/cli/execute.cpp
@@ -97,6 +97,43 @@ public:
 "master",
 "Mesos master (e.g., IP:PORT).");
 
+add(::task,
+"task",
+"The value could be a JSON-formatted string of `TaskInfo` or a\n"
+"file path containing the JSON-formatted `TaskInfo`. Path must\n"
+"be of the form `file:///path/to/file` or `/path/to/file`."
+"\n"
+"See the `TaskInfo` message in `mesos.proto` for the expected\n"
+"format. NOTE: `agent_id` need not to be set.\n"
+"\n"
+"Example:\n"
+"{\n"
+"  \"name\": \"Name of the task\",\n"
+"  \"task_id\": {\"value\" : \"Id of the task\"},\n"
+"  \"agent_id\": {\"value\" : \"\"},\n"
+"  \"resources\": [\n"
+"{\n"
+"  \"name\": \"cpus\",\n"
+"  \"type\": \"SCALAR\",\n"
+"  \"scalar\": {\n"
+"\"value\": 0.1\n"
+"  },\n"
+"  \"role\": \"*\"\n"
+"},\n"
+"{\n"
+"  \"name\": \"mem\",\n"
+"  \"type\": \"SCALAR\",\n"
+"  \"scalar\": {\n"
+"\"value\": 32\n"
+"  },\n"
+"  \"role\": \"*\"\n"
+"}\n"
+"  ],\n"
+"  \"command\": {\n"
+"\"value\": \"sleep 1000\"\n"
+"  }\n"
+"}");
+
 add(::task_group,
 "task_group",
 "The value could be a JSON-formatted string of `TaskGroupInfo` or a\n"
@@ -305,6 +342,7 @@ public:
 
   string master;
   Option name;
+  Option task;
   Option task_group;
   bool shell;
   Option command;
@@ -828,21 +866,12 @@ int main(int argc, char** argv)
 LOG(WARNING) << warning.message;
   }
 
-  if (flags.task_group.isSome() &&
-  (flags.name.isSome() ||
-   flags.command.isSome() ||
-   flags.environment.isSome() ||
-   flags.appc_image.isSome()  ||
-   flags.docker_image.isSome() ||
-   flags.volumes.isSome())) {
+  if (flags.task.isSome() && flags.task_group.isSome()) {
 cerr << flags.usage(
   "Either task or task group should be set but not both. Provide"
-  " either '--name, --command, --env, --appc_image, 
--docker_image,"
-  " --volumes' OR '--task_group'") << endl;
+  " either '--task' OR '--task_group'") << endl;
 return EXIT_FAILURE;
-  }
-
-  if (flags.task_group.isNone()) {
+  } else if (flags.task.isNone() && flags.task_group.isNone()) {
 if (flags.name.isNone()) {
   cerr << flags.usage("Missing required option --name") << endl;
   return EXIT_FAILURE;
@@ -852,6 +881,27 @@ int main(int argc, char** argv)
   cerr << flags.usage("Missing required option --command") << endl;
   return EXIT_FAILURE;
 }
+  } else {
+// Either --task or --task_group is set.
+if (flags.name.isSome() ||
+flags.command.isSome() ||
+flags.environment.isSome() ||
+flags.appc_image.isSome()  ||
+flags.docker_image.isSome() ||
+flags.volumes.isSome()) {
+  cerr << flags.usage(
+"'--name, --command, --env, --appc_image, --docker_image,"
+" --volumes' can only be set when both '--task' and"
+" '--task_group' are not set") << endl;
+  return EXIT_FAILURE;
+}
+
+if (flags.task.isSome() && flags.networks.isSome()) {
+  cerr << flags.usage(
+"'--networks' can only be set when"
+" '--task' is not set") << endl;
+  return EXIT_FAILURE;
+}
   }
 
   if (flags.content_type == "json" ||
@@ -1022,9 +1072,9 @@ int main(int argc, char** argv)
 }
   }
 
-  Option taskInfo = None();
+  Option taskInfo = flags.task;
 
-  if (flags.task_group.isNone()) {
+  if (flags.task.isNone() && flags.task_group.isNone()) {
 TaskInfo task;
 

[1/2] mesos git commit: Added parse function for v1::TaskInfo protobuf.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 22dd6e321 -> 77da99a9c


Added parse function for v1::TaskInfo protobuf.

Review: https://reviews.apache.org/r/53644/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bf52cc13
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bf52cc13
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bf52cc13

Branch: refs/heads/master
Commit: bf52cc1342b89b7d6923d017ddda06b066d03082
Parents: 22dd6e3
Author: Qian Zhang 
Authored: Tue Nov 22 09:31:47 2016 +0800
Committer: Qian Zhang 
Committed: Tue Nov 22 09:31:47 2016 +0800

--
 src/v1/parse.hpp | 14 ++
 1 file changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/bf52cc13/src/v1/parse.hpp
--
diff --git a/src/v1/parse.hpp b/src/v1/parse.hpp
index 86bd7e8..318bb21 100644
--- a/src/v1/parse.hpp
+++ b/src/v1/parse.hpp
@@ -61,6 +61,20 @@ inline Try parse(const 
std::string& value)
   return protobuf::parse(json.get());
 }
 
+
+template <>
+inline Try parse(const std::string& value)
+{
+  // Convert from string or file to JSON.
+  Try json = parse(value);
+  if (json.isError()) {
+return Error(json.error());
+  }
+
+  // Convert from JSON to Protobuf.
+  return protobuf::parse(json.get());
+}
+
 } // namespace flags {
 
 #endif // __V1_PARSE_HPP__

mesos git commit: Added Qian Zhang to the list of committers.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master fdd9304c4 -> 816fa1616


Added Qian Zhang to the list of committers.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/816fa161
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/816fa161
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/816fa161

Branch: refs/heads/master
Commit: 816fa16162dc5ddb7b2f36892be647afbf5426d2
Parents: fdd9304
Author: Qian Zhang 
Authored: Wed Nov 2 11:04:56 2016 +0800
Committer: Qian Zhang 
Committed: Wed Nov 2 11:04:56 2016 +0800

--
 docs/committers.md | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/816fa161/docs/committers.md
--
diff --git a/docs/committers.md b/docs/committers.md
index 9c2cf6b..9aa124f 100644
--- a/docs/committers.md
+++ b/docs/committers.md
@@ -243,6 +243,13 @@ We'd like to thank the following committers to the Apache 
Mesos project who have
   
   ma...@apache.org
 
+
+  +8
+  Qian Zhang
+  IBM
+  
+  qianzh...@apache.org
+
   
 
 

mesos git commit: Added backport of MESOS-6571 to 1.1.x.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.1.x 2342bc335 -> f5b6cb8d6


Added backport of MESOS-6571 to 1.1.x.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f5b6cb8d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f5b6cb8d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f5b6cb8d

Branch: refs/heads/1.1.x
Commit: f5b6cb8d65f8d704afe72b8adc84a3bb2e95e51b
Parents: 2342bc3
Author: Qian Zhang 
Authored: Thu Dec 29 10:50:33 2016 +0800
Committer: Qian Zhang 
Committed: Thu Dec 29 10:50:33 2016 +0800

--
 src/Makefile.am  |  1 +
 src/cli/execute.cpp  | 81 +++
 src/common/parse.hpp | 16 --
 src/v1/parse.hpp | 62 
 4 files changed, 130 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f5b6cb8d/src/Makefile.am
--
diff --git a/src/Makefile.am b/src/Makefile.am
index 790e8b1..5316cbb 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1084,6 +1084,7 @@ libmesos_no_3rdparty_la_SOURCES +=
\
   uri/schemes/hdfs.hpp \
   uri/schemes/http.hpp \
   usage/usage.hpp  \
+  v1/parse.hpp \
   version/version.hpp  \
   watcher/whitelist_watcher.hpp
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/f5b6cb8d/src/cli/execute.cpp
--
diff --git a/src/cli/execute.cpp b/src/cli/execute.cpp
index 94ac463..c2dd87a 100644
--- a/src/cli/execute.cpp
+++ b/src/cli/execute.cpp
@@ -46,6 +46,8 @@
 
 #include "internal/devolve.hpp"
 
+#include "v1/parse.hpp"
+
 using std::cerr;
 using std::cout;
 using std::endl;
@@ -95,6 +97,43 @@ public:
 "master",
 "Mesos master (e.g., IP:PORT).");
 
+add(::task,
+"task",
+"The value could be a JSON-formatted string of `TaskInfo` or a\n"
+"file path containing the JSON-formatted `TaskInfo`. Path must\n"
+"be of the form `file:///path/to/file` or `/path/to/file`."
+"\n"
+"See the `TaskInfo` message in `mesos.proto` for the expected\n"
+"format. NOTE: `agent_id` need not to be set.\n"
+"\n"
+"Example:\n"
+"{\n"
+"  \"name\": \"Name of the task\",\n"
+"  \"task_id\": {\"value\" : \"Id of the task\"},\n"
+"  \"agent_id\": {\"value\" : \"\"},\n"
+"  \"resources\": [\n"
+"{\n"
+"  \"name\": \"cpus\",\n"
+"  \"type\": \"SCALAR\",\n"
+"  \"scalar\": {\n"
+"\"value\": 0.1\n"
+"  },\n"
+"  \"role\": \"*\"\n"
+"},\n"
+"{\n"
+"  \"name\": \"mem\",\n"
+"  \"type\": \"SCALAR\",\n"
+"  \"scalar\": {\n"
+"\"value\": 32\n"
+"  },\n"
+"  \"role\": \"*\"\n"
+"}\n"
+"  ],\n"
+"  \"command\": {\n"
+"\"value\": \"sleep 1000\"\n"
+"  }\n"
+"}");
+
 add(_group,
 "task_group",
 "The value could be a JSON-formatted string of `TaskGroupInfo` or a\n"
@@ -267,6 +306,7 @@ public:
 
   Option master;
   Option name;
+  Option task;
   Option task_group;
   bool shell;
   Option command;
@@ -779,21 +819,13 @@ int main(int argc, char** argv)
 return EXIT_FAILURE;
   }
 
-  if (flags.task_group.isSome() &&
-  (flags.name.isSome() ||
-   flags.command.isSome() ||
-   flags.environment.isSome() ||
-   flags.appc_image.isSome()  ||
-   flags.docker_image.isSome() ||
-   flags.volumes.isSome())) {
+  if (flags.task.isSome() && flags.task_group.isSome()) {
 cerr << flags.usage(
   "Either task or task group should be set but not both. Provide"
-  " either '--name, --command, --env, --appc_image, 
--docker_image,"
-  " --volumes' OR '--task_group'") << endl;
-return EXIT_FAILURE;
-  }
+  " either '--task' OR '--task_group'") << endl;
 
-  if (flags.task_group.isNone()) {
+return EXIT_FAILURE;
+  } else if (flags.task.isNone() && flags.task_group.isNone()) {
 if (flags.name.isNone()) {
   cerr << flags.usage("Missing required option --name") << endl;
   return EXIT_FAILURE;
@@ -803,6 +835,27 @@ int main(int argc, char** argv)
   cerr << flags.usage("Missing required option --command") << endl;
   return EXIT_FAILURE;
 }
+  } else {
+ 

mesos git commit: Added MESOS-6571 to CHANGELOG for 1.1.1.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 3cb30eb16 -> 3ba3565ae


Added MESOS-6571 to CHANGELOG for 1.1.1.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3ba3565a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3ba3565a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3ba3565a

Branch: refs/heads/master
Commit: 3ba3565ae357de34e9bc58e0d0f1760a5d7ff930
Parents: 3cb30eb
Author: Qian Zhang 
Authored: Thu Dec 29 11:08:49 2016 +0800
Committer: Qian Zhang 
Committed: Thu Dec 29 11:08:49 2016 +0800

--
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3ba3565a/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index e2f8cac..059f71a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -24,6 +24,7 @@ All Issues:
   * [MESOS-6360] - The handling of whiteout files in provisioner is not 
correct.
   * [MESOS-6411] - Add documentation for CNI port-mapper plugin.
   * [MESOS-6526] - `mesos-containerizer launch --environment` exposes executor 
env vars in `ps`.
+  * [MESOS-6571] - Add "--task" flag to mesos-execute.
   * [MESOS-6621] - SSL downgrade path will CHECK-fail when using both 
temporary and persistent sockets.
   * [MESOS-6676] - Always re-link with scheduler during re-registration.
 

[2/3] mesos git commit: Updated OCI spec parsing & validation code with latest OCI image spec.

[qianzhang]

Updated OCI spec parsing & validation code with latest OCI image spec.

Review: https://reviews.apache.org/r/56852/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0c1d50d2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0c1d50d2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0c1d50d2

Branch: refs/heads/master
Commit: 0c1d50d236dbc100b9cbab194daba409483eed7a
Parents: 87ca2fb
Author: Qian Zhang 
Authored: Tue Mar 28 16:50:11 2017 +0800
Committer: Qian Zhang 
Committed: Tue Mar 28 16:50:11 2017 +0800

--
 include/mesos/oci/spec.hpp |  15 +++-
 src/oci/spec.cpp   | 152 +++-
 2 files changed, 116 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/0c1d50d2/include/mesos/oci/spec.hpp
--
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
index ea4f29e..d8eef84 100644
--- a/include/mesos/oci/spec.hpp
+++ b/include/mesos/oci/spec.hpp
@@ -26,8 +26,8 @@ namespace v1 {
 
 // Constant strings for OCI image media types:
 // https://github.com/opencontainers/image-spec/blob/master/media-types.md
-constexpr char MEDIA_TYPE_MANIFEST_LIST[] =
-"application/vnd.oci.image.manifest.list.v1+json";
+constexpr char MEDIA_TYPE_INDEX[] =
+"application/vnd.oci.image.index.v1+json";
 
 constexpr char MEDIA_TYPE_MANIFEST[] =
 "application/vnd.oci.image.manifest.v1+json";
@@ -36,13 +36,22 @@ constexpr char MEDIA_TYPE_CONFIG[] =
 "application/vnd.oci.image.config.v1+json";
 
 constexpr char MEDIA_TYPE_LAYER[] =
+"application/vnd.oci.image.layer.v1.tar";
+
+constexpr char MEDIA_TYPE_LAYER_GZIP[] =
 "application/vnd.oci.image.layer.v1.tar+gzip";
 
 constexpr char MEDIA_TYPE_NONDIST_LAYER[] =
+"application/vnd.oci.image.layer.nondistributable.v1.tar";
+
+constexpr char MEDIA_TYPE_NONDIST_LAYER_GZIP[] =
 "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip";
 
+// Rootfs type of OCI image configuration.
+constexpr char ROOTFS_TYPE[] = "layers";
+
 /**
- * Returns the OCI v1 descriptor, image manifest list, image manifest
+ * Returns the OCI v1 descriptor, image index, image manifest
  * and image configuration from the given string.
  */
 template 

http://git-wip-us.apache.org/repos/asf/mesos/blob/0c1d50d2/src/oci/spec.cpp
--
diff --git a/src/oci/spec.cpp b/src/oci/spec.cpp
index 0a88edb..06fceb4 100644
--- a/src/oci/spec.cpp
+++ b/src/oci/spec.cpp
@@ -43,25 +43,20 @@ Option validateDigest(const string& digest)
 }
 
 
-Option validate(const ManifestList& manifestList)
+Option validate(const Index& index)
 {
-  if (manifestList.schemaversion() != 2) {
+  if (index.schemaversion() != 2) {
 return Error(
 "Incorrect 'schemaVersion': " +
-stringify(manifestList.schemaversion()));
+stringify(index.schemaversion()));
   }
 
-  foreach (const ManifestDescriptor& manifest, manifestList.manifests()) {
+  foreach (const ManifestDescriptor& manifest, index.manifests()) {
 Option error = validateDigest(manifest.digest());
 if (error.isSome()) {
   return Error(
   "Failed to validate 'digest' of the 'manifest': " + error->message);
 }
-
-if (manifest.mediatype() != MEDIA_TYPE_MANIFEST) {
-  return Error(
-  "Incorrect 'mediaType' of the 'manifest': " + manifest.mediatype());
-}
   }
 
   return None();
@@ -101,7 +96,9 @@ Option validate(const Manifest& manifest)
 }
 
 if (layer.mediatype() != MEDIA_TYPE_LAYER &&
-layer.mediatype() != MEDIA_TYPE_NONDIST_LAYER) {
+layer.mediatype() != MEDIA_TYPE_LAYER_GZIP &&
+layer.mediatype() != MEDIA_TYPE_NONDIST_LAYER &&
+layer.mediatype() != MEDIA_TYPE_NONDIST_LAYER_GZIP) {
   return Error(
   "Incorrect 'mediaType' of the 'layer': " + layer.mediatype());
 }
@@ -110,6 +107,16 @@ Option validate(const Manifest& manifest)
   return None();
 }
 
+
+Option validate(const Configuration& configuration)
+{
+  if (configuration.rootfs().type() != ROOTFS_TYPE) {
+return Error("Incorrect 'type': " + configuration.rootfs().type());
+  }
+
+  return None();
+}
+
 } // namespace internal {
 
 
@@ -126,6 +133,28 @@ Try parse(const string& s)
 return Error("Protobuf parse failed: " + descriptor.error());
   }
 
+  // Manually parse 'annotations' field.
+  Result annotations = json->find("annotations");
+  if (annotations.isError()) {
+return Error(
+"Failed to find 'annotations': " + annotations.error());
+  }
+
+  if (annotations.isSome() && !annotations->is()) {
+foreachpair (const string& key,
+ const JSON::Value& value,
+   

[3/3] mesos git commit: Update OCI tests with the latest OCI image spec.

[qianzhang]

Update OCI tests with the latest OCI image spec.

Review: https://reviews.apache.org/r/56853/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/18f6642e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/18f6642e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/18f6642e

Branch: refs/heads/master
Commit: 18f6642e28d0fee1031a88e62265a49e3bd601a6
Parents: 0c1d50d
Author: Qian Zhang 
Authored: Tue Mar 28 16:50:18 2017 +0800
Committer: Qian Zhang 
Committed: Tue Mar 28 16:50:18 2017 +0800

--
 src/tests/containerizer/oci_spec_tests.cpp | 32 -
 1 file changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/18f6642e/src/tests/containerizer/oci_spec_tests.cpp
--
diff --git a/src/tests/containerizer/oci_spec_tests.cpp 
b/src/tests/containerizer/oci_spec_tests.cpp
index 29a8f23..011e6d0 100644
--- a/src/tests/containerizer/oci_spec_tests.cpp
+++ b/src/tests/containerizer/oci_spec_tests.cpp
@@ -68,7 +68,7 @@ TEST_F(OCISpecTest, ParseDescriptor)
 }
 
 
-TEST_F(OCISpecTest, ParseManifestList)
+TEST_F(OCISpecTest, ParseIndex)
 {
   const string json =
   R"~(
@@ -104,54 +104,54 @@ TEST_F(OCISpecTest, ParseManifestList)
 }
   })~";
 
-  Try manifestList =
-  image::v1::parse(json);
+  Try index =
+  image::v1::parse(json);
 
-  ASSERT_SOME(manifestList);
+  ASSERT_SOME(index);
 
-  EXPECT_EQ(2u, manifestList->schemaversion());
+  EXPECT_EQ(2u, index->schemaversion());
 
   EXPECT_EQ(
   "application/vnd.oci.image.manifest.v1+json",
-  manifestList->manifests(0).mediatype());
+  index->manifests(0).mediatype());
 
-  EXPECT_EQ(7143u, manifestList->manifests(0).size());
+  EXPECT_EQ(7143u, index->manifests(0).size());
 
   EXPECT_EQ(
   
"sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
-  manifestList->manifests(0).digest());
+  index->manifests(0).digest());
 
   EXPECT_EQ(
   "ppc64le",
-  manifestList->manifests(0).platform().architecture());
+  index->manifests(0).platform().architecture());
 
   EXPECT_EQ(
   "linux",
-  manifestList->manifests(0).platform().os());
+  index->manifests(0).platform().os());
 
   EXPECT_EQ(
   "16.04",
-  manifestList->manifests(0).platform().os_version());
+  index->manifests(0).platform().os_version());
 
   EXPECT_EQ(
   "sse4",
-  manifestList->manifests(1).platform().os_features(0));
+  index->manifests(1).platform().os_features(0));
 
   EXPECT_EQ(
   "com.example.key1",
-  manifestList->annotations(0).key());
+  index->annotations(0).key());
 
   EXPECT_EQ(
   "value1",
-  manifestList->annotations(0).value());
+  index->annotations(0).value());
 
   EXPECT_EQ(
   "com.example.key2",
-  manifestList->annotations(1).key());
+  index->annotations(1).key());
 
   EXPECT_EQ(
   "value2",
-  manifestList->annotations(1).value());
+  index->annotations(1).value());
 }
 
 

[1/3] mesos git commit: Updated OCI protobuf messages with latest OCI image spec.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 52dfc2f00 -> 18f6642e2


Updated OCI protobuf messages with latest OCI image spec.

Review: https://reviews.apache.org/r/56851/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/87ca2fb0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/87ca2fb0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/87ca2fb0

Branch: refs/heads/master
Commit: 87ca2fb0c9d179616a6c1b70882edac783bca34b
Parents: 52dfc2f
Author: Qian Zhang 
Authored: Tue Mar 28 16:50:03 2017 +0800
Committer: Qian Zhang 
Committed: Tue Mar 28 16:50:03 2017 +0800

--
 include/mesos/oci/spec.proto | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/87ca2fb0/include/mesos/oci/spec.proto
--
diff --git a/include/mesos/oci/spec.proto b/include/mesos/oci/spec.proto
index 97f3cf3..f7f1979 100644
--- a/include/mesos/oci/spec.proto
+++ b/include/mesos/oci/spec.proto
@@ -29,6 +29,11 @@ message Descriptor {
   required string digest = 2;
   required int64 size = 3;
   repeated string urls = 4;
+
+  // NOTE: We cannot use 'annotations' here because otherwise,
+  // json->protobuf parsing will fail. 'Annotations' is manually
+  // set during parsing.
+  repeated Label Annotations = 5;
 }
 
 
@@ -50,8 +55,13 @@ message ManifestDescriptor {
   required string mediaType = 1;
   required string digest = 2;
   required int64 size = 3;
-  required Platform platform = 4;
+  optional Platform platform = 4;
   repeated string urls = 5;
+
+  // NOTE: We cannot use 'annotations' here because otherwise,
+  // json->protobuf parsing will fail. 'Annotations' is manually
+  // set during parsing.
+  repeated Label Annotations = 6;
 }
 
 
@@ -65,13 +75,13 @@ message Label {
 
 
 /**
- * Protobuf for the OCI image manifest list JSON schema:
- * https://github.com/opencontainers/image-spec/blob/master/manifest-list.md
+ * Protobuf for the OCI image index JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/image-index.md
  *
  * The OCI MIME type of this message is:
- *   application/vnd.oci.image.manifest.list.v1+json
+ *   application/vnd.oci.image.index.v1+json
  */
-message ManifestList {
+message Index {
   required int64 schemaVersion = 1;
   repeated ManifestDescriptor manifests = 2;
 

mesos git commit: Fixed the samples in multiple-disk.md.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master a1610 -> 253c89483


Fixed the samples in multiple-disk.md.

Review: https://reviews.apache.org/r/58473/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/253c8948
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/253c8948
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/253c8948

Branch: refs/heads/master
Commit: 253c89483cb6e31225a1f55449ff88c676030eec
Parents: a16
Author: Qian Zhang 
Authored: Wed Apr 19 20:59:06 2017 +0800
Committer: Qian Zhang 
Committed: Wed Apr 19 20:59:06 2017 +0800

--
 docs/multiple-disk.md | 64 +-
 1 file changed, 29 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/253c8948/docs/multiple-disk.md
--
diff --git a/docs/multiple-disk.md b/docs/multiple-disk.md
index 748d423..5501cc0 100644
--- a/docs/multiple-disk.md
+++ b/docs/multiple-disk.md
@@ -43,15 +43,13 @@ An example resources value for a root disk is shown below. 
Note that the
 operator could optionally specify a `role` for the disk, which would result in
 [statically reserving](reservation.md) the disk for a single [role](roles.md).
 
-{
-  "resources" : [
-{
-  "name" : "disk",
-  "type" : "SCALAR",
-  "scalar" : { "value" : 2048 }
-}
-  ]
-}
+[
+  {
+"name" : "disk",
+"type" : "SCALAR",
+"scalar" : { "value" : 2048 }
+  }
+]
 
 ### `Path` disks
 
@@ -77,21 +75,19 @@ An example resources value for a `Path` disk is shown 
below. Note that the
 operator could optionally specify a `role` for the disk, which would result in
 [statically reserving](reservation.md) the disk for a single [role](roles.md).
 
-{
-  "resources" : [
-{
-  "name" : "disk",
-  "type" : "SCALAR",
-  "scalar" : { "value" : 2048 },
-  "disk" : {
-"source" : {
-  "type" : "PATH",
-  "path" : { "root" : "/mnt/data" }
-}
+[
+  {
+"name" : "disk",
+"type" : "SCALAR",
+"scalar" : { "value" : 2048 },
+"disk" : {
+  "source" : {
+"type" : "PATH",
+"path" : { "root" : "/mnt/data" }
   }
 }
-  ]
-}
+  }
+]
 
 ### `Mount` disks
 
@@ -117,21 +113,19 @@ An example resources value for a `Mount` disk is shown 
below. Note that the
 operator could optionally specify a `role` for the disk, which would result in
 [statically reserving](reservation.md) the disk for a single [role](roles.md).
 
-{
-  "resources" : [
-{
-  "name" : "disk",
-  "type" : "SCALAR",
-  "scalar" : { "value" : 2048 },
-  "disk" : {
-"source" : {
-  "type" : "MOUNT",
-  "mount" : { "root" : "/mnt/data" }
-}
+[
+  {
+"name" : "disk",
+"type" : "SCALAR",
+"scalar" : { "value" : 2048 },
+"disk" : {
+  "source" : {
+"type" : "MOUNT",
+"mount" : { "root" : "/mnt/data" }
   }
 }
-  ]
-}
+  }
+]
 
  `Block` disks
 

[8/9] mesos git commit: Added a test `DefaultContainerDNSCniTest.ROOT_VerifyDefaultDNS`.

[qianzhang]

Added a test `DefaultContainerDNSCniTest.ROOT_VerifyDefaultDNS`.

Review: https://reviews.apache.org/r/60793


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/18ad7fb7
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/18ad7fb7
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/18ad7fb7

Branch: refs/heads/master
Commit: 18ad7fb7473849ba67b1ff007e2d131556eb42d8
Parents: d67595c
Author: Qian Zhang 
Authored: Wed Jul 12 14:48:08 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 179 
 1 file changed, 179 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/18ad7fb7/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index ae0980b..0a16294 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1732,6 +1732,185 @@ TEST_F(CniIsolatorPortMapperTest, 
ROOT_INTERNET_CURL_PortMapper)
   driver.join();
 }
 
+
+class DefaultContainerDNSCniTest
+  : public CniIsolatorTest,
+public WithParamInterface {};
+
+
+INSTANTIATE_TEST_CASE_P(
+DefaultContainerDNSInfo,
+DefaultContainerDNSCniTest,
+::testing::Values(
+// A DNS information for the `__MESOS_TEST__` CNI network.
+R"~(
+{
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "network_name": "__MESOS_TEST__",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+"domain": "mesos.apache.org",
+"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
+"options": [ "timeout:3", "attempts:2" ]
+  }
+}
+  ]
+})~",
+// A DNS information with `network_mode == CNI`, but without a network
+// name, acts as a wildcard match making it the default DNS for any CNI
+// network not specified in the `--default_container_dns` flag.
+R"~(
+{
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+"domain": "mesos.apache.org",
+"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
+"options": [ "timeout:3", "attempts:2" ]
+  }
+}
+  ]
+})~",
+// Two DNS information, one is specific for `__MESOS_TEST__` CNI
+// network, the other is the defaule DNS for any CNI network not
+// specified in the `--default_container_dns` flag.
+R"~(
+{
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "network_name": "__MESOS_TEST__",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+"domain": "mesos.apache.org",
+"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
+"options": [ "timeout:3", "attempts:2" ]
+  }
+},
+{
+  "network_mode": "CNI",
+  "dns": {
+"nameservers": [ "8.8.8.9", "8.8.4.5" ],
+"domain": "mesos1.apache.org",
+"search": [ "b.mesos.apache.org", "b.mesos.apache.org" ],
+"options": [ "timeout:9", "attempts:5" ]
+  }
+}
+  ]
+})~"));
+
+
+// This test verifies the DNS configuration of the container can be
+// successfully set with the agent flag `--default_container_dns`.
+TEST_P(DefaultContainerDNSCniTest, ROOT_VerifyDefaultDNS)
+{
+  Try hostNetwork = getNonLoopbackIP();
+  ASSERT_SOME(hostNetwork);
+
+  Try mockPlugin = strings::format(
+  R"~(
+  #!/bin/sh
+  echo '{'
+  echo '  "ip4": {'
+  echo '"ip": "%s/%d"'
+  echo '  }'
+  echo '}'
+  )~",
+  hostNetwork.get().address(),
+  hostNetwork.get().prefix());
+
+  ASSERT_SOME(mockPlugin);
+
+  ASSERT_SOME(setupMockPlugin(mockPlugin.get()));
+
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  slave::Flags flags = CreateSlaveFlags();
+  flags.isolation = "network/cni";
+
+  flags.network_cni_plugins_dir = cniPluginDir;
+  flags.network_cni_config_dir = cniConfigDir;
+
+  Try parse = flags::parse(GetParam());
+  ASSERT_SOME(parse);
+
+  flags.default_container_dns = parse.get();
+
+  Owned detector = master.get()->createDetector();
+
+  Try slave = StartSlave(detector.get(), flags);
+  ASSERT_SOME(slave);
+
+  MockScheduler sched;

[1/9] mesos git commit: Added a test `DefaultContainerDNSFlagTest.ValidateFlag`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 1793f8f2a -> 01e5bc386


Added a test `DefaultContainerDNSFlagTest.ValidateFlag`.

Review: https://reviews.apache.org/r/61245


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/01e5bc38
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/01e5bc38
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/01e5bc38

Branch: refs/heads/master
Commit: 01e5bc386c744aa5e9976935674639595557a760
Parents: 18ad7fb
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Sun Jul 30 23:29:10 2017 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/tests/slave_tests.cpp | 163 +
 1 file changed, 163 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/01e5bc38/src/tests/slave_tests.cpp
--
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index a8c3a77..c55a425 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -133,6 +133,7 @@ using testing::Eq;
 using testing::Invoke;
 using testing::Return;
 using testing::SaveArg;
+using testing::WithParamInterface;
 
 namespace mesos {
 namespace internal {
@@ -7861,6 +7862,168 @@ TEST_F(SlaveTest, ChangeDomain)
   }
 }
 
+
+class DefaultContainerDNSFlagTest
+  : public MesosTest,
+public WithParamInterface {};
+
+
+INSTANTIATE_TEST_CASE_P(
+ContainerizerType,
+DefaultContainerDNSFlagTest,
+::testing::Values("mesos", "docker"));
+
+
+// This test verifies the validation for the
+// agent flag `--default_container_dns`.
+TEST_P(DefaultContainerDNSFlagTest, ValidateFlag)
+{
+  const int argc = 4;
+  const char* argv[argc] = {
+"/path/to/program",
+"--master=127.0.0.1:5050",
+"--work_dir=/tmp"
+  };
+
+  string containerizer = GetParam();
+
+  // Verifies the unknown network mode is not supported.
+  //
+  // TODO(qianzhang): Change the value of the `network_mode`
+  // to an non-existent enum value once MESOS-7828 is resolved.
+  string defaultContainerDNSInfo =
+"--default_container_dns={"
+"  \"" + containerizer + "\": [\n"
+"{\n"
+"  \"network_mode\": \"UNKNOWN\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\" ]\n"
+"  }\n"
+"}\n"
+"  ]\n"
+"}";
+
+  argv[3] = defaultContainerDNSInfo.c_str();
+
+  {
+slave::Flags flags;
+Try load = flags.load(None(), argc, argv);
+EXPECT_ERROR(load);
+  }
+
+  // Verifies the host network mode is not supported.
+  defaultContainerDNSInfo =
+"--default_container_dns={"
+"  \"" + containerizer + "\": [\n"
+"{\n"
+"  \"network_mode\": \"HOST\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\" ]\n"
+"  }\n"
+"}\n"
+"  ]\n"
+"}";
+
+  argv[3] = defaultContainerDNSInfo.c_str();
+
+  {
+slave::Flags flags;
+Try load = flags.load(None(), argc, argv);
+EXPECT_ERROR(load);
+  }
+
+  string network_mode = (containerizer == "mesos" ? "CNI"  : "USER");
+
+  // Verifies multiple DNS configuration without network name for
+  // user-defined CNM network or CNI network is not supported.
+  defaultContainerDNSInfo =
+"--default_container_dns={"
+"  \"" + containerizer + "\": [\n"
+"{\n"
+"  \"network_mode\": \"" + network_mode + "\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\" ]\n"
+"  }\n"
+"},\n"
+"{\n"
+"  \"network_mode\": \"" + network_mode + "\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\" ]\n"
+"  }\n"
+"}\n"
+"  ]\n"
+"}";
+
+  argv[3] = defaultContainerDNSInfo.c_str();
+
+  {
+slave::Flags flags;
+Try load = flags.load(None(), argc, argv);
+EXPECT_ERROR(load);
+  }
+
+  // Verifies multiple DNS configuration with the same network name for CNI
+  // network or user-defined CNM network or CNI network is not su

[7/9] mesos git commit: Set container DNS with `--default_container_dns` in CNI isolator.

[qianzhang]

Set container DNS with `--default_container_dns` in CNI isolator.

Review: https://reviews.apache.org/r/60600


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/30b49016
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/30b49016
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/30b49016

Branch: refs/heads/master
Commit: 30b49016adc30a9598d426ee35af14ee73963f77
Parents: cf841cd
Author: Qian Zhang 
Authored: Mon Jul 3 21:50:17 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 .../mesos/isolators/network/cni/cni.cpp | 52 +---
 .../mesos/isolators/network/cni/cni.hpp | 12 -
 2 files changed, 55 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/30b49016/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
--
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp 
b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index 831bc7d..fc68f04 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -86,7 +86,8 @@ Try NetworkCniIsolatorProcess::create(const Flags& 
flags)
 return new MesosIsolator(Owned(
 new NetworkCniIsolatorProcess(
 flags,
-hashmap(;
+hashmap(),
+hashmap(;
   }
 
   // Check for root permission.
@@ -239,10 +240,32 @@ Try NetworkCniIsolatorProcess::create(const 
Flags& flags)
 }
   }
 
+  hashmap cniDNSMap;
+  Option defaultCniDNS;
+
+  if (flags.default_container_dns.isSome()) {
+foreach (const ContainerDNSInfo::MesosInfo& dnsInfo,
+ flags.default_container_dns->mesos()) {
+  if (dnsInfo.network_mode() == ContainerDNSInfo::MesosInfo::CNI) {
+if (!dnsInfo.has_network_name()) {
+  // The DNS info which has network node set as `CNI` and has no
+  // network name set is considered as the default DNS for all CNI
+  // networks, it applies to the container which joins a CNI network
+  // but that network can not be found in `--default_container_dns`.
+  defaultCniDNS = dnsInfo;
+} else {
+  cniDNSMap[dnsInfo.network_name()] = dnsInfo;
+}
+  }
+}
+  }
+
   return new MesosIsolator(Owned(
   new NetworkCniIsolatorProcess(
   flags,
   networkConfigs.get(),
+  cniDNSMap,
+  defaultCniDNS,
   rootDir.get(),
   flags.network_cni_plugins_dir.get(;
 }
@@ -989,18 +1012,31 @@ Future NetworkCniIsolatorProcess::_isolate(
 
   cni::spec::DNS dns;
 
-  // Collect all the DNS resolver specifications from the networks'
-  // IPAM plugins. Ordering is preserved and for single-value fields,
-  // the last network will win.
+  // For each CNI network that the container joins, collect the DNS resolver
+  // specification returned from the networks' IPAM plugin, if it is not
+  // returned from the plugin, use the DNS resolver specification in the
+  // default container DNS if any. Ordering is preserved and for single-value
+  // fields, the last network will win.
   foreachvalue (const ContainerNetwork& network, info->containerNetworks) {
-if (network.cniNetworkInfo.isSome() && network.cniNetworkInfo->has_dns()) {
+if (network.cniNetworkInfo.isSome() &&
+network.cniNetworkInfo->has_dns() &&
+network.cniNetworkInfo->dns().nameservers_size() > 0) {
+  // NOTE: Just checking `has_dns()` is not enough since some IPAM plugins
+  // (e.g., host-local) will return an empty `dns` JSON string ("dns": {})
+  // even though the CNI network configuration has not specified DNS
+  // information, that will make `has_dns()` true, so here we further check
+  // the size of the `nameservers`.
   dns.MergeFrom(network.cniNetworkInfo->dns());
+} else if (cniDNSMap.contains(network.networkName)) {
+  dns.MergeFrom(cniDNSMap.at(network.networkName).dns());
+} else if (defaultCniDNS.isSome()) {
+  dns.MergeFrom(defaultCniDNS->dns());
 }
   }
 
-  // If IPAM has not specified any DNS servers, then we set
-  // the container 'resolv.conf' to be the same as the host
-  // 'resolv.conf' ('/etc/resolv.conf').
+  // If IPAM plugin has not specified any DNS servers and there is no default
+  // container DNS specified, then we set the container 'resolv.conf' to be the
+  // same as the host 'resolv.conf' ('/etc/resolv.conf').
   if (dns.nameservers().empty()) {
 if 

[3/9] mesos git commit: Set container DNS with `--default_container_dns` in Docker executor.

[qianzhang]

Set container DNS with `--default_container_dns` in Docker executor.

Review: https://reviews.apache.org/r/60558


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3da83b33
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3da83b33
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3da83b33

Branch: refs/heads/master
Commit: 3da83b3318a612b3bbf1223fbafe506c1ed4bcc4
Parents: 48b5ef0
Author: Qian Zhang 
Authored: Fri Jun 30 09:53:41 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/docker/docker.cpp   | 105 +--
 src/docker/docker.hpp   |  14 +-
 src/docker/executor.cpp |   3 +-
 3 files changed, 116 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3da83b33/src/docker/docker.cpp
--
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 8081c02..daa340f 100755
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -59,6 +59,8 @@ using std::map;
 using std::string;
 using std::vector;
 
+using mesos::internal::ContainerDNSInfo;
+
 
 template 
 static Future failure(
@@ -505,7 +507,8 @@ Try Docker::RunOptions::create(
 const Option& resources,
 bool enableCfsQuota,
 const Option>& env,
-const Option& devices)
+const Option& devices,
+const Option& defaultContainerDNS)
 {
   if (!containerInfo.has_docker()) {
 return Error("No docker info found in container info");
@@ -727,9 +730,71 @@ Try Docker::RunOptions::create(
 
   options.name = name;
 
+  bool dnsSpecified = false;
   foreach (const Parameter& parameter, dockerInfo.parameters()) {
 options.additionalOptions.push_back(
 "--" + parameter.key() + "=" + parameter.value());
+
+// In Docker 1.13.0, `--dns-option` was added and `--dns-opt` was hidden
+// (but it can still be used), so here we need to check both of them.
+if (!dnsSpecified &&
+(parameter.key() == "dns" ||
+ parameter.key() == "dns-search" ||
+ parameter.key() == "dns-opt" ||
+ parameter.key() == "dns-option")) {
+  dnsSpecified = true;
+}
+  }
+
+  if (!dnsSpecified && defaultContainerDNS.isSome()) {
+Option bridgeDNS;
+Option defaultUserDNS;
+hashmap userDNSMap;
+
+foreach (const ContainerDNSInfo::DockerInfo& dnsInfo,
+ defaultContainerDNS->docker()) {
+  // Currently we only support setting DNS for containers which join
+  // Docker bridge network or user-defined network.
+  if (dnsInfo.network_mode() == ContainerDNSInfo::DockerInfo::BRIDGE) {
+bridgeDNS = dnsInfo;
+  } else if (dnsInfo.network_mode() == ContainerDNSInfo::DockerInfo::USER) 
{
+if (!dnsInfo.has_network_name()) {
+  // The DNS info which has network node set as `USER` and has no
+  // network name set is considered as the default DNS for all
+  // user-defined networks. It applies to the Docker container which
+  // joins a user-defined network but that network can not be found in
+  // `defaultContainerDNS`.
+  defaultUserDNS = dnsInfo;
+} else {
+  userDNSMap[dnsInfo.network_name()] = dnsInfo;
+}
+  }
+}
+
+auto setDNSInfo = [&](const ContainerDNSInfo::DockerInfo& dnsInfo) {
+  options.dns.assign(
+  dnsInfo.dns().nameservers().begin(),
+  dnsInfo.dns().nameservers().end());
+
+  options.dnsSearch.assign(
+  dnsInfo.dns().search().begin(),
+  dnsInfo.dns().search().end());
+
+  options.dnsOpt.assign(
+  dnsInfo.dns().options().begin(),
+  dnsInfo.dns().options().end());
+};
+
+if (dockerInfo.network() == ContainerInfo::DockerInfo::BRIDGE &&
+bridgeDNS.isSome()) {
+  setDNSInfo(bridgeDNS.get());
+} else if (dockerInfo.network() == ContainerInfo::DockerInfo::USER) {
+  if (userDNSMap.contains(options.network.get())) {
+setDNSInfo(userDNSMap.at(options.network.get()));
+  } else if (defaultUserDNS.isSome()) {
+setDNSInfo(defaultUserDNS.get());
+  }
+}
   }
 
   options.image = dockerInfo.image();
@@ -827,14 +892,48 @@ Future Docker::run(
 if (network != "host" &&
 network != "bridge" &&
 network != "none") {
-  // User defined networks require docker version >= 1.9.0.
+  // User defined networks require Docker version >= 1.9.0.
   Try validateVer = validateVersion(Version(1, 9, 0));
-
   if (validateVer.isError()) {
 return Failure("User defined networks require Docker "
"version 1.9.0 or 

[6/9] mesos git commit: Passed default container DNS info to Docker executor.

[qianzhang]

Passed default container DNS info to Docker executor.

Review: https://reviews.apache.org/r/60557


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/48b5ef03
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/48b5ef03
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/48b5ef03

Branch: refs/heads/master
Commit: 48b5ef036905ae5a112af26ab6985953f8179c8c
Parents: ebfccf4
Author: Qian Zhang 
Authored: Wed Jun 28 22:26:02 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/docker/executor.cpp| 21 +
 src/docker/executor.hpp|  5 +
 src/slave/containerizer/docker.cpp |  6 ++
 3 files changed, 32 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/48b5ef03/src/docker/executor.cpp
--
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index e03f244..a14ac7d 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -50,6 +50,7 @@
 #include "logging/flags.hpp"
 #include "logging/logging.hpp"
 
+#include "messages/flags.hpp"
 #include "messages/messages.hpp"
 
 #include "slave/constants.hpp"
@@ -87,6 +88,7 @@ public:
   const Duration& shutdownGracePeriod,
   const string& launcherDir,
   const map& taskEnvironment,
+  const Option& defaultContainerDNS,
   bool cgroupsEnableCfs)
 : ProcessBase(ID::generate("docker-executor")),
   killed(false),
@@ -99,6 +101,7 @@ public:
   mappedDirectory(mappedDirectory),
   shutdownGracePeriod(shutdownGracePeriod),
   taskEnvironment(taskEnvironment),
+  defaultContainerDNS(defaultContainerDNS),
   cgroupsEnableCfs(cgroupsEnableCfs),
   stop(Nothing()),
   inspect(Nothing()) {}
@@ -605,6 +608,7 @@ private:
   string mappedDirectory;
   Duration shutdownGracePeriod;
   map taskEnvironment;
+  Option defaultContainerDNS;
   bool cgroupsEnableCfs;
 
   Option killPolicy;
@@ -631,6 +635,7 @@ public:
   const Duration& shutdownGracePeriod,
   const string& launcherDir,
   const map& taskEnvironment,
+  const Option& defaultContainerDNS,
   bool cgroupsEnableCfs)
   {
 process = Owned(new DockerExecutorProcess(
@@ -641,6 +646,7 @@ public:
 shutdownGracePeriod,
 launcherDir,
 taskEnvironment,
+defaultContainerDNS,
 cgroupsEnableCfs));
 
 spawn(process.get());
@@ -793,6 +799,20 @@ int main(int argc, char** argv)
 }
   }
 
+  Option defaultContainerDNS;
+  if (flags.default_container_dns.isSome()) {
+Try parse =
+  flags::parse(
+  flags.default_container_dns.get());
+
+if (parse.isError()) {
+  EXIT(EXIT_FAILURE) << flags.usage(
+  "Failed to parse --default_container_dns: " + parse.error());
+}
+
+defaultContainerDNS = parse.get();
+  }
+
   // Get executor shutdown grace period from the environment.
   //
   // NOTE: We avoided introducing a docker executor flag for this
@@ -847,6 +867,7 @@ int main(int argc, char** argv)
   shutdownGracePeriod,
   flags.launcher_dir.get(),
   taskEnvironment,
+  defaultContainerDNS,
   flags.cgroups_enable_cfs));
 
   Owned driver(

http://git-wip-us.apache.org/repos/asf/mesos/blob/48b5ef03/src/docker/executor.hpp
--
diff --git a/src/docker/executor.hpp b/src/docker/executor.hpp
index a4a8ec9..f21e84c 100644
--- a/src/docker/executor.hpp
+++ b/src/docker/executor.hpp
@@ -76,6 +76,10 @@ struct Flags : public virtual mesos::internal::logging::Flags
 "A JSON map of environment variables and values that should\n"
 "be passed into the task launched by this executor.");
 
+add(::default_container_dns,
+"default_container_dns",
+"JSON-formatted default DNS information for container.");
+
 add(::cgroups_enable_cfs,
 "cgroups_enable_cfs",
 "Cgroups feature flag to enable hard limits on CPU resources\n"
@@ -90,6 +94,7 @@ struct Flags : public virtual mesos::internal::logging::Flags
   Option mapped_directory;
   Option launcher_dir;
   Option task_environment;
+  Option default_container_dns;
 
   bool cgroups_enable_cfs;
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/48b5ef03/src/slave/containerizer/docker.cpp
--
diff --git a/src/slave/containerizer/docker.cpp 
b/src/slave/containerizer/docker.cpp
index 2fe9227..fee1e10 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -39,6 +39,7 @@
 #include 
 #include 
 #include 
+#include 
 

[4/9] mesos git commit: Introduced `--default_container_dns` agent flag.

[qianzhang]

Introduced `--default_container_dns` agent flag.

Review: https://reviews.apache.org/r/60500


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ebfccf4e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ebfccf4e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ebfccf4e

Branch: refs/heads/master
Commit: ebfccf4ea12ccc4b02700b44e69ab17affd50019
Parents: 1793f8f
Author: Qian Zhang 
Authored: Wed Jun 28 14:10:01 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 docs/configuration.md|  46 +++
 src/messages/flags.hpp   |  21 +++
 src/messages/flags.proto |  74 +++-
 src/slave/flags.cpp  | 128 ++
 src/slave/flags.hpp  |   1 +
 5 files changed, 268 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/ebfccf4e/docs/configuration.md
--
diff --git a/docs/configuration.md b/docs/configuration.md
index 058e366..041c3df 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1329,6 +1329,52 @@ Example:
 
 
   
+--default_container_dns=VALUE
+  
+  
+JSON-formatted DNS information for CNI networks (Mesos containerizer)
+and CNM networks (Docker containerizer). For CNI networks, this flag
+can be used to configure `nameservers`, `domain`, `search` and
+`options`, and its priority is lower than the DNS information returned
+by a CNI plugin, but higher than the DNS information in agent host's
+/etc/resolv.conf. For CNM networks, this flag can be used to configure
+`nameservers`, `search` and `options`, it will only be used if there
+is no DNS information provided in the ContainerInfo.docker.parameters
+message.
+
+See the ContainerDNS message in `flags.proto` for the expected format.
+
+Example:
+{
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "network_name": "net1",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ]
+  }
+}
+  ],
+  "docker": [
+{
+  "network_mode": "BRIDGE",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ]
+  }
+},
+{
+  "network_mode": "USER",
+  "network_name": "net2",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ]
+  }
+}
+  ]
+}
+  
+
+
+  
 --default_container_info=VALUE
   
   

http://git-wip-us.apache.org/repos/asf/mesos/blob/ebfccf4e/src/messages/flags.hpp
--
diff --git a/src/messages/flags.hpp b/src/messages/flags.hpp
index 70ad58c..b267677 100644
--- a/src/messages/flags.hpp
+++ b/src/messages/flags.hpp
@@ -45,6 +45,19 @@ inline Try parse(const 
std::string& value)
   return protobuf::parse(json.get());
 }
 
+
+template <>
+inline Try parse(const std::string& value)
+{
+  // Convert from string or file to JSON.
+  Try json = parse(value);
+  if (json.isError()) {
+return Error(json.error());
+  }
+
+  return protobuf::parse(json.get());
+}
+
 } // namespace flags {
 
 namespace mesos {
@@ -57,6 +70,14 @@ inline std::ostream& operator<<(
   return stream << rules.DebugString();
 }
 
+
+inline std::ostream& operator<<(
+std::ostream& stream,
+const ContainerDNSInfo& dns)
+{
+  return stream << dns.DebugString();
+}
+
 } // namespace internal {
 } // namespace mesos {
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/ebfccf4e/src/messages/flags.proto
--
diff --git a/src/messages/flags.proto b/src/messages/flags.proto
index e87075f..077353c 100644
--- a/src/messages/flags.proto
+++ b/src/messages/flags.proto
@@ -16,10 +16,14 @@
 
 syntax = "proto2";
 
+import "slave/containerizer/mesos/isolators/network/cni/spec.proto";
+
 package mesos.internal;
 
-// Initializes firewall rules to allow access control of the
-// libprocess endpoints.
+/**
+ * Initializes firewall rules to allow access control of the
+ * libprocess endpoints.
+ */
 message Firewall {
   message DisabledEndpointsRule {
 repeated string paths = 1;
@@ -27,3 +31,69 @@ message Firewall {
 
   optional DisabledEndpointsRule disabled_endpoints = 1;
 }
+
+
+/**
+ * DNS information for CNI networks (Mesos containerizer) and CNM
+ * networks (Docker containerizer).
+ */
+message ContainerDNSInfo {
+  message MesosInfo {
+enum NetworkMode {
+  UNKNOWN = 0;
+  HOST = 1; // Currently not supported.
+  CNI = 2;
+}
+
+optional NetworkMode network_mode = 1;
+
+// This field is valid only when `network_mode` is set to CNI, it informs
+// the `network/cni` isolator about the CNI network to which the DNS
+// configuration applies. Also, if the mode is CNI and this field 

[2/9] mesos git commit: Added a test `DockerContainerizerTest.ROOT_DOCKER_DefaultDNS`.

[qianzhang]

Added a test `DockerContainerizerTest.ROOT_DOCKER_DefaultDNS`.

Review: https://reviews.apache.org/r/60761


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d67595cd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d67595cd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d67595cd

Branch: refs/heads/master
Commit: d67595cdeff45f54aa227e5ae33afe6b7ac1c53a
Parents: 28faca0
Author: Qian Zhang 
Authored: Tue Jul 11 16:58:43 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 .../docker_containerizer_tests.cpp  | 153 +++
 1 file changed, 153 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/d67595cd/src/tests/containerizer/docker_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp 
b/src/tests/containerizer/docker_containerizer_tests.cpp
index 1e85a79..c798459 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -4352,6 +4352,159 @@ TEST_F(DockerContainerizerTest, 
ROOT_DOCKER_Non_Root_Sandbox)
 }
 
 
+// This test verifies the DNS configuration of the Docker container
+// can be successfully set with the agent flag `--default_container_dns`.
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_DefaultDNS)
+{
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  MockDocker* mockDocker =
+new MockDocker(tests::flags.docker, tests::flags.docker_socket);
+
+  Shared docker(mockDocker);
+
+  slave::Flags flags = CreateSlaveFlags();
+  Try parse = flags::parse(
+  R"~(
+  {
+"docker": [
+  {
+"network_mode": "BRIDGE",
+"dns": {
+  "nameservers": [ "8.8.8.8", "8.8.4.4" ],
+  "search": [ "example1.com", "example2.com" ],
+  "options": [ "timeout:3", "attempts:2" ]
+}
+  }
+]
+  })~");
+
+  ASSERT_SOME(parse);
+
+  flags.default_container_dns = parse.get();
+
+  Fetcher fetcher(flags);
+
+  Try logger =
+ContainerLogger::create(flags.container_logger);
+
+  ASSERT_SOME(logger);
+
+  MockDockerContainerizer dockerContainerizer(
+  flags,
+  ,
+  Owned(logger.get()),
+  docker);
+
+  Owned detector = master.get()->createDetector();
+
+  Try slave =
+StartSlave(detector.get(), , flags);
+  ASSERT_SOME(slave);
+
+  MockScheduler sched;
+  MesosSchedulerDriver driver(
+  , DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+  Future frameworkId;
+  EXPECT_CALL(sched, registered(, _, _))
+.WillOnce(FutureArg<1>());
+
+  Future offers;
+  EXPECT_CALL(sched, resourceOffers(, _))
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  driver.start();
+
+  AWAIT_READY(frameworkId);
+
+  AWAIT_READY(offers);
+  ASSERT_NE(0u, offers->size());
+
+  const Offer& offer = offers.get()[0];
+
+  TaskInfo task;
+  task.set_name("");
+  task.mutable_task_id()->set_value("1");
+  task.mutable_slave_id()->CopyFrom(offer.slave_id());
+  task.mutable_resources()->CopyFrom(offer.resources());
+
+  CommandInfo command;
+  command.set_value("sleep 1000");
+
+  ContainerInfo containerInfo;
+  containerInfo.set_type(ContainerInfo::DOCKER);
+
+  // TODO(tnachen): Use local image to test if possible.
+  ContainerInfo::DockerInfo dockerInfo;
+  dockerInfo.set_image("alpine");
+  dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
+  containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+  task.mutable_command()->CopyFrom(command);
+  task.mutable_container()->CopyFrom(containerInfo);
+
+  Future containerId;
+  EXPECT_CALL(dockerContainerizer, launch(_, _, _, _))
+.WillOnce(DoAll(FutureArg<0>(),
+Invoke(,
+   ::_launch)));
+
+  Future statusRunning;
+  EXPECT_CALL(sched, statusUpdate(, _))
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(DoDefault());
+
+  driver.launchTasks(offers.get()[0].id(), {task});
+
+  AWAIT_READY_FOR(containerId, Seconds(60));
+  AWAIT_READY_FOR(statusRunning, Seconds(60));
+  EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+  ASSERT_TRUE(statusRunning->has_data());
+
+  // Find the DNS configuration of the container and verify
+  // if it is consistent with `flags.default_container_dns`.
+  string name = containerName(containerId.get());
+  Future inspect = docker->inspect(name);
+  AWAIT_READY(inspect);
+
+  vector defaultDNS;
+  std::copy(
+  flags.default_container_dns->docker(0).dns().nameservers().begin(),
+  flags.default_container_dns->docker(0).dns().nameservers().end(),
+  

[9/9] mesos git commit: Parsed DNS related info from the output of `docker inspect`.

[qianzhang]

Parsed DNS related info from the output of `docker inspect`.

Review: https://reviews.apache.org/r/60760


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/28faca0a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/28faca0a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/28faca0a

Branch: refs/heads/master
Commit: 28faca0ae2b3aeeddd078a978f4b7b2483d03c20
Parents: 30b4901
Author: Qian Zhang 
Authored: Tue Jul 11 14:41:17 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/docker/docker.cpp | 74 +-
 src/docker/docker.hpp | 19 ++--
 2 files changed, 90 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/28faca0a/src/docker/docker.cpp
--
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index daa340f..df1490f 100755
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -409,7 +409,79 @@ Try Docker::Container::create(const 
string& output)
 }
   }
 
-  return Container(output, id, name, optionalPid, started, ipAddress, devices);
+  vector dns;
+
+  Result dnsArray =
+json.find("HostConfig.Dns");
+
+  if (dnsArray.isError()) {
+return Error("Failed to parse HostConfig.Dns: " + dnsArray.error());
+  }
+
+  if (dnsArray.isSome()) {
+foreach (const JSON::Value& entry, dnsArray->values) {
+  if (!entry.is()) {
+return Error("Malformed HostConfig.Dns"
+ " entry '" + stringify(entry) + "'");
+  }
+
+  dns.push_back(entry.as().value);
+}
+  }
+
+  vector dnsOptions;
+
+  Result dnsOptionArray =
+json.find("HostConfig.DnsOptions");
+
+  if (dnsOptionArray.isError()) {
+return Error("Failed to parse HostConfig.DnsOptions: " +
+ dnsOptionArray.error());
+  }
+
+  if (dnsOptionArray.isSome()) {
+foreach (const JSON::Value& entry, dnsOptionArray->values) {
+  if (!entry.is()) {
+return Error("Malformed HostConfig.DnsOptions"
+ " entry '" + stringify(entry) + "'");
+  }
+
+  dnsOptions.push_back(entry.as().value);
+}
+  }
+
+  vector dnsSearch;
+
+  Result dnsSearchArray =
+json.find("HostConfig.DnsSearch");
+
+  if (dnsSearchArray.isError()) {
+return Error("Failed to parse HostConfig.DnsSearch: " +
+ dnsSearchArray.error());
+  }
+
+  if (dnsSearchArray.isSome()) {
+foreach (const JSON::Value& entry, dnsSearchArray->values) {
+  if (!entry.is()) {
+return Error("Malformed HostConfig.DnsSearch"
+ " entry '" + stringify(entry) + "'");
+  }
+
+  dnsSearch.push_back(entry.as().value);
+}
+  }
+
+  return Container(
+  output,
+  id,
+  name,
+  optionalPid,
+  started,
+  ipAddress,
+  devices,
+  dns,
+  dnsOptions,
+  dnsSearch);
 }
 
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/28faca0a/src/docker/docker.hpp
--
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 9581aa2..95e60a7 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -115,6 +115,15 @@ public:
 
 const std::vector devices;
 
+// Returns the DNS nameservers set by "--dns" option.
+const std::vector dns;
+
+// Returns the DNS options set by "--dns-option" option.
+const std::vector dnsOptions;
+
+// Returns the DNS search domains set by "--dns-search" option.
+const std::vector dnsSearch;
+
   private:
 Container(
 const std::string& output,
@@ -123,14 +132,20 @@ public:
 const Option& pid,
 bool started,
 const Option& ipAddress,
-const std::vector& devices)
+const std::vector& devices,
+const std::vector& dns,
+const std::vector& dnsOptions,
+const std::vector& dnsSearch)
   : output(output),
 id(id),
 name(name),
 pid(pid),
 started(started),
 ipAddress(ipAddress),
-devices(devices) {}
+devices(devices),
+dns(dns),
+dnsOptions(dnsOptions),
+dnsSearch(dnsSearch) {}
   };
 
   class Image

[5/9] mesos git commit: Set container DNS with `--default_container_dns` in DockerContainerizer.

[qianzhang]

Set container DNS with `--default_container_dns` in DockerContainerizer.

Review: https://reviews.apache.org/r/61075


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cf841cdd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cf841cdd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cf841cdd

Branch: refs/heads/master
Commit: cf841cdd482d78e2872f04539cf82d151dffd689
Parents: 3da83b3
Author: Qian Zhang 
Authored: Mon Jul 24 14:38:22 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 13:53:26 2017 +0800

--
 src/slave/containerizer/docker.cpp | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/cf841cdd/src/slave/containerizer/docker.cpp
--
diff --git a/src/slave/containerizer/docker.cpp 
b/src/slave/containerizer/docker.cpp
index fee1e10..b84c392 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -1330,6 +1330,13 @@ Future 
DockerContainerizerProcess::launchExecutorContainer(
 self(),
 [=](const ContainerIO& containerIO)
   -> Future {
+// We need to pass `flags.default_container_dns` only when the agent is not
+// running in a Docker container. This is to handle the case of launching a
+// custom executor in a Docker container. If the agent is running in a
+// Docker container (i.e., flags.docker_mesos_image.isSome() == true), that
+// is the case of launching `mesos-docker-executor` in a Docker container
+// with the Docker image `flags.docker_mesos_image`. In that case we 
already
+// set `flags.default_container_dns` in the method `dockerFlags()`.
 Try runOptions = Docker::RunOptions::create(
 container->container,
 container->command,
@@ -1343,7 +1350,8 @@ Future 
DockerContainerizerProcess::launchExecutorContainer(
 false,
 #endif
 container->environment,
-None() // No extra devices.
+None(), // No extra devices.
+flags.docker_mesos_image.isNone() ? flags.default_container_dns : 
None()
 );
 
 if (runOptions.isError()) {

mesos git commit: Replaced raw string with normal string for `DefaultContainerDNSCniTest`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 9b658ce17 -> 50b7a3e0a


Replaced raw string with normal string for `DefaultContainerDNSCniTest`.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/50b7a3e0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/50b7a3e0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/50b7a3e0

Branch: refs/heads/master
Commit: 50b7a3e0a39df671892546c1b09d9c3c98e15125
Parents: 9b658ce
Author: Qian Zhang 
Authored: Thu Aug 3 20:48:57 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 20:59:04 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 107 ++--
 1 file changed, 51 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/50b7a3e0/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index baaed8c..60c85ad 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1743,67 +1743,62 @@ INSTANTIATE_TEST_CASE_P(
 DefaultContainerDNSCniTest,
 ::testing::Values(
 // A DNS information for the `__MESOS_TEST__` CNI network.
-string(
-R"~(
-{
-  "mesos": [
-{
-  "network_mode": "CNI",
-  "network_name": "__MESOS_TEST__",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-}
-  ]
-})~"),
+"{\n"
+"  \"mesos\": [\n"
+"{\n"
+"  \"network_mode\": \"CNI\",\n"
+"  \"network_name\": \"__MESOS_TEST__\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\", \"8.8.4.4\" ],\n"
+"\"domain\": \"mesos.apache.org\",\n"
+"\"search\": [ \"a.mesos.apache.org\" ],\n"
+"\"options\": [ \"timeout:3\", \"attempts:2\" ]\n"
+"  }\n"
+"}\n"
+"  ]\n"
+"}",
 // A DNS information with `network_mode == CNI`, but without a network
 // name, acts as a wildcard match making it the default DNS for any CNI
 // network not specified in the `--default_container_dns` flag.
-string(
-R"~(
-{
-  "mesos": [
-{
-  "network_mode": "CNI",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-}
-  ]
-})~"),
+"{\n"
+"  \"mesos\": [\n"
+"{\n"
+"  \"network_mode\": \"CNI\",\n"
+"  \"dns\": {\n"
+"\"nameservers\": [ \"8.8.8.8\", \"8.8.4.4\" ],\n"
+"\"domain\": \"mesos.apache.org\",\n"
+"\"search\": [ \"a.mesos.apache.org\" ],\n"
+"\"options\": [ \"timeout:3\", \"attempts:2\" ]\n"
+"  }\n"
+"}\n"
+"  ]\n"
+"}",
 // Two DNS information, one is specific for `__MESOS_TEST__` CNI
 // network, the other is the defaule DNS for any CNI network not
 // specified in the `--default_container_dns` flag.
-string(
-R"~(
-{
-  "mesos": [
-{
-  "network_mode": "CNI",
-  "network_name": "__MESOS_TEST__",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-},
-{
-  "network_mode": "CNI",
-  "dns": {
-"nameservers": [ "8.8.8.9", "8.8.4.5" ],
-"domain": "mesos1.apache.org",
-"search": [ "b.mesos.apache.org", "b.mesos.apache.org" ],
-"options": [ "timeout:9", "attempts:5" ]
-  }
-}
-  ]})~")));
+"{\n"
+"  \"mesos\": [\n"
+"{\n"
+  

mesos git commit: Fixed a typo in CNI isolator test.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 4fe22dd33 -> c0c6d3efd


Fixed a typo in CNI isolator test.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c0c6d3ef
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c0c6d3ef
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c0c6d3ef

Branch: refs/heads/master
Commit: c0c6d3efd20f4f675e1004581ba9dbe65a34493a
Parents: 4fe22dd
Author: Qian Zhang 
Authored: Wed Jul 12 10:09:08 2017 +0800
Committer: Qian Zhang 
Committed: Wed Jul 12 10:09:44 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/c0c6d3ef/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index 96e3e9e..ae0980b 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -907,7 +907,7 @@ TEST_F(CniIsolatorTest, ROOT_DynamicAddDelofCniConfig)
 
 
 // This test verifies that the hostname of the container can be
-// overriden by setting hostname field in ContainerInfo.
+// overridden by setting hostname field in ContainerInfo.
 TEST_F(CniIsolatorTest, ROOT_OverrideHostname)
 {
   Try master = StartMaster();

mesos git commit: Fixed an incorrect field name in fetcher.md.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 4b6269c1f -> 400d3002d


Fixed an incorrect field name in fetcher.md.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/400d3002
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/400d3002
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/400d3002

Branch: refs/heads/master
Commit: 400d3002d4aa82cbae4b55bced608e95225176e4
Parents: 4b6269c
Author: Qian Zhang 
Authored: Sun Jul 16 16:19:25 2017 +0800
Committer: Qian Zhang 
Committed: Sun Jul 16 16:19:25 2017 +0800

--
 docs/fetcher.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/400d3002/docs/fetcher.md
--
diff --git a/docs/fetcher.md b/docs/fetcher.md
index e8cb138..55d2435 100644
--- a/docs/fetcher.md
+++ b/docs/fetcher.md
@@ -71,7 +71,7 @@ each URI are determined based on the following protobuf 
structure. (See
 optional bool executable = 2;
 optional bool extract = 3 [default = true];
 optional bool cache = 4;
-optional string filename = 5;
+optional string output_file = 5;
   }
   ...
   optional string user = 5;

[1/3] mesos git commit: Generated protobuf_tests.pb.* files during the build.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master fc27129a1 -> e6ec4e836


http://git-wip-us.apache.org/repos/asf/mesos/blob/e6ec4e83/3rdparty/stout/tests/protobuf_tests.pb.h
--
diff --git a/3rdparty/stout/tests/protobuf_tests.pb.h 
b/3rdparty/stout/tests/protobuf_tests.pb.h
deleted file mode 100644
index 2e4ffe1..000
--- a/3rdparty/stout/tests/protobuf_tests.pb.h
+++ /dev/null
@@ -1,2262 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: protobuf_tests.proto
-
-#ifndef PROTOBUF_protobuf_5ftests_2eproto__INCLUDED
-#define PROTOBUF_protobuf_5ftests_2eproto__INCLUDED
-
-#include 
-
-#include 
-
-#if GOOGLE_PROTOBUF_VERSION < 3003000
-#error This file was generated by a newer version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please update
-#error your headers.
-#endif
-#if 3003000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
-#error This file was generated by an older version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please
-#error regenerate this file with a newer version of protoc.
-#endif
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include   // IWYU pragma: export
-#include   // IWYU pragma: export
-#include 
-#include 
-// @@protoc_insertion_point(includes)
-namespace tests {
-class ArrayMessage;
-class ArrayMessageDefaultTypeInternal;
-extern ArrayMessageDefaultTypeInternal _ArrayMessage_default_instance_;
-class Message;
-class MessageDefaultTypeInternal;
-extern MessageDefaultTypeInternal _Message_default_instance_;
-class Nested;
-class NestedDefaultTypeInternal;
-extern NestedDefaultTypeInternal _Nested_default_instance_;
-class SimpleMessage;
-class SimpleMessageDefaultTypeInternal;
-extern SimpleMessageDefaultTypeInternal _SimpleMessage_default_instance_;
-}  // namespace tests
-
-namespace tests {
-
-namespace protobuf_protobuf_5ftests_2eproto {
-// Internal implementation detail -- do not call these.
-struct TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[];
-  static const ::google::protobuf::uint32 offsets[];
-  static void InitDefaultsImpl();
-  static void Shutdown();
-};
-void AddDescriptors();
-void InitDefaults();
-}  // namespace protobuf_protobuf_5ftests_2eproto
-
-enum Enum {
-  ONE = 1,
-  TWO = 2
-};
-bool Enum_IsValid(int value);
-const Enum Enum_MIN = ONE;
-const Enum Enum_MAX = TWO;
-const int Enum_ARRAYSIZE = Enum_MAX + 1;
-
-const ::google::protobuf::EnumDescriptor* Enum_descriptor();
-inline const ::std::string& Enum_Name(Enum value) {
-  return ::google::protobuf::internal::NameOfEnum(
-Enum_descriptor(), value);
-}
-inline bool Enum_Parse(
-const ::std::string& name, Enum* value) {
-  return ::google::protobuf::internal::ParseNamedEnum(
-Enum_descriptor(), name, value);
-}
-// ===
-
-class Nested : public ::google::protobuf::Message /* 
@@protoc_insertion_point(class_definition:tests.Nested) */ {
- public:
-  Nested();
-  virtual ~Nested();
-
-  Nested(const Nested& from);
-
-  inline Nested& operator=(const Nested& from) {
-CopyFrom(from);
-return *this;
-  }
-
-  inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const {
-return _internal_metadata_.unknown_fields();
-  }
-
-  inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() {
-return _internal_metadata_.mutable_unknown_fields();
-  }
-
-  static const ::google::protobuf::Descriptor* descriptor();
-  static const Nested& default_instance();
-
-  static inline const Nested* internal_default_instance() {
-return reinterpret_cast(
-   &_Nested_default_instance_);
-  }
-  static PROTOBUF_CONSTEXPR int const kIndexInFileMessages =
-0;
-
-  void Swap(Nested* other);
-
-  // implements Message --
-
-  inline Nested* New() const PROTOBUF_FINAL { return New(NULL); }
-
-  Nested* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
-  void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void CopyFrom(const Nested& from);
-  void MergeFrom(const Nested& from);
-  void Clear() PROTOBUF_FINAL;
-  bool IsInitialized() const PROTOBUF_FINAL;
-
-  size_t ByteSizeLong() const PROTOBUF_FINAL;
-  bool MergePartialFromCodedStream(
-  ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL;
-  void SerializeWithCachedSizes(
-  ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL;
-  ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
-  bool deterministic, ::google::protobuf::uint8* target) const 
PROTOBUF_FINAL;
-  int 

[3/3] mesos git commit: Generated protobuf_tests.pb.* files during the build.

[qianzhang]

Generated protobuf_tests.pb.* files during the build.

Review: https://reviews.apache.org/r/60109/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e6ec4e83
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e6ec4e83
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e6ec4e83

Branch: refs/heads/master
Commit: e6ec4e8362552cbaf74bf3e844e7ac4b15f9aafd
Parents: fc27129
Author: Qian Zhang 
Authored: Thu Jul 27 09:45:48 2017 +0800
Committer: Qian Zhang 
Committed: Thu Jul 27 09:45:48 2017 +0800

--
 3rdparty/stout/Makefile.am|   20 +-
 3rdparty/stout/configure.ac   |2 +
 3rdparty/stout/tests/CMakeLists.txt   |   11 +
 3rdparty/stout/tests/protobuf_tests.pb.cc | 3985 
 3rdparty/stout/tests/protobuf_tests.pb.h  | 2262 --
 5 files changed, 31 insertions(+), 6249 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/e6ec4e83/3rdparty/stout/Makefile.am
--
diff --git a/3rdparty/stout/Makefile.am b/3rdparty/stout/Makefile.am
index 7956d1a..20c3c1b 100644
--- a/3rdparty/stout/Makefile.am
+++ b/3rdparty/stout/Makefile.am
@@ -110,10 +110,18 @@ PROTOBUF_INCLUDE_FLAGS = -I$(PROTOBUF)/src
 LIB_PROTOBUF = $(PROTOBUF)/src/libprotobuf.la
 $(LIB_PROTOBUF): $(PROTOBUF)-build-stamp
 BUNDLED_DEPS += $(PROTOBUF)-build-stamp
+PROTOC = $(PROTOBUF)/src/protoc
 else
 LIB_PROTOBUF = -lprotobuf
+PROTOC = @PROTOCOMPILER@
 endif
 
+PROTOCFLAGS = -I$(srcdir)/tests
+
+# Targets for generating C++ protocol buffer code.
+protobuf_tests.pb.cc protobuf_tests.pb.h: tests/protobuf_tests.proto
+   $(PROTOC) $(PROTOCFLAGS) --cpp_out=$(builddir) $^
+
 # Tests for stout.
 check_PROGRAMS = stout-tests
 
@@ -154,8 +162,6 @@ stout_tests_SOURCES =   \
   tests/os/systems_tests.cpp   \
   tests/path_tests.cpp \
   tests/protobuf_tests.cpp \
-  tests/protobuf_tests.pb.cc   \
-  tests/protobuf_tests.pb.h\
   tests/protobuf_tests.proto   \
   tests/recordio_tests.cpp \
   tests/result_tests.cpp   \
@@ -204,6 +210,16 @@ endif
 
 BUILT_SOURCES = $(BUNDLED_DEPS)
 
+BUILT_SOURCES +=   \
+  protobuf_tests.pb.cc
+
+nodist_stout_tests_SOURCES =   \
+  protobuf_tests.pb.cc
+
+CLEANFILES =   \
+  protobuf_tests.pb.cc \
+  protobuf_tests.pb.h
+
 # We use a check-local target for now to avoid the parallel test
 # runner that ships with newer versions of autotools.
 # See the following discussion for the workaround:

http://git-wip-us.apache.org/repos/asf/mesos/blob/e6ec4e83/3rdparty/stout/configure.ac
--
diff --git a/3rdparty/stout/configure.ac b/3rdparty/stout/configure.ac
index abb6521..6db2287 100644
--- a/3rdparty/stout/configure.ac
+++ b/3rdparty/stout/configure.ac
@@ -520,6 +520,7 @@ if test "x$without_bundled_protobuf" = "xyes" || \
   AC_MSG_ERROR([protoc not found in PATH])
 fi
 
+PROTOCOMPILER="$PROTOBUFPREFIX/bin/protoc"
   else
 AC_MSG_ERROR([cannot find protobuf
 ---
@@ -539,6 +540,7 @@ fi
 AM_CONDITIONAL([WITH_BUNDLED_PROTOBUF],
[test "x$with_bundled_protobuf" = "xyes"])
 
+AC_SUBST([PROTOCOMPILER])
 
 # Check if libsvn-1 prefix path was provided, and if so, add it to
 # the CPPFLAGS and LDFLAGS with respective /include and /lib path

http://git-wip-us.apache.org/repos/asf/mesos/blob/e6ec4e83/3rdparty/stout/tests/CMakeLists.txt
--
diff --git a/3rdparty/stout/tests/CMakeLists.txt 
b/3rdparty/stout/tests/CMakeLists.txt
index 8d881ab..12882ee 100644
--- a/3rdparty/stout/tests/CMakeLists.txt
+++ b/3rdparty/stout/tests/CMakeLists.txt
@@ -99,12 +99,23 @@ set(STOUT_TESTS_SRC
 #
 include_directories(SYSTEM ${STOUT_TEST_3RDPARTY_INCLUDE_DIRS})
 include_directories(${STOUT_INCLUDE_DIRS})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 # LINKING LIBRARIES BY DIRECTORY (might generate, e.g., -L/path/to/thing on
 # Linux).
 ###
 link_directories(${STOUT_TEST_LIB_DIRS})
 
+# THE C++ PROTOCOL BUFFER CODE (protobuf_tests.pb.h and protobuf_tests.pb.cc).
+##
+add_custom_command(
+  OUTPUT protobuf_tests.pb.h protobuf_tests.pb.cc
+  COMMAND ${PROTOC}
+-I${CMAKE_SOURCE_DIR}/3rdparty/stout/tests
+--cpp_out=.
+

[2/3] mesos git commit: Generated protobuf_tests.pb.* files during the build.

[qianzhang]

http://git-wip-us.apache.org/repos/asf/mesos/blob/e6ec4e83/3rdparty/stout/tests/protobuf_tests.pb.cc
--
diff --git a/3rdparty/stout/tests/protobuf_tests.pb.cc 
b/3rdparty/stout/tests/protobuf_tests.pb.cc
deleted file mode 100644
index ad6eff7..000
--- a/3rdparty/stout/tests/protobuf_tests.pb.cc
+++ /dev/null
@@ -1,3985 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: protobuf_tests.proto
-
-#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION
-#include "protobuf_tests.pb.h"
-
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-// @@protoc_insertion_point(includes)
-
-namespace tests {
-class NestedDefaultTypeInternal : public 
::google::protobuf::internal::ExplicitlyConstructed {
-} _Nested_default_instance_;
-class SimpleMessageDefaultTypeInternal : public 
::google::protobuf::internal::ExplicitlyConstructed {
-} _SimpleMessage_default_instance_;
-class MessageDefaultTypeInternal : public 
::google::protobuf::internal::ExplicitlyConstructed {
-} _Message_default_instance_;
-class ArrayMessageDefaultTypeInternal : public 
::google::protobuf::internal::ExplicitlyConstructed {
-} _ArrayMessage_default_instance_;
-
-namespace protobuf_protobuf_5ftests_2eproto {
-
-
-namespace {
-
-::google::protobuf::Metadata file_level_metadata[4];
-const ::google::protobuf::EnumDescriptor* file_level_enum_descriptors[1];
-
-}  // namespace
-
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTableField
-const TableStruct::entries[] = {
-  {0, 0, 0, ::google::protobuf::internal::kInvalidMask, 0, 0},
-};
-
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::AuxillaryParseTableField
-const TableStruct::aux[] = {
-  ::google::protobuf::internal::AuxillaryParseTableField(),
-};
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTable const
-TableStruct::schema[] = {
-  { NULL, NULL, 0, -1, -1, false },
-  { NULL, NULL, 0, -1, -1, false },
-  { NULL, NULL, 0, -1, -1, false },
-  { NULL, NULL, 0, -1, -1, false },
-};
-
-const ::google::protobuf::uint32 TableStruct::offsets[] = {
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Nested, _has_bits_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Nested, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Nested, str_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Nested, optional_str_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Nested, repeated_str_),
-  0,
-  1,
-  ~0u,
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(SimpleMessage, _has_bits_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(SimpleMessage, 
_internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(SimpleMessage, id_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(SimpleMessage, numbers_),
-  0,
-  ~0u,
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, _has_bits_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, b_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, str_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, bytes_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, int32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, int64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, uint32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, uint64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, sint32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, sint64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, f_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, d_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, e_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, nested_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_bool_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_string_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_bytes_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_int32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_int64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_uint32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_uint64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_sint32_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_sint64_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Message, repeated_float_),
-  

mesos git commit: Surrounded raw string with `string()` for `DefaultContainerDNSCniTest`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 01e5bc386 -> 9b0c836cd


Surrounded raw string with `string()` for `DefaultContainerDNSCniTest`.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9b0c836c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9b0c836c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9b0c836c

Branch: refs/heads/master
Commit: 9b0c836cd7fbf239788e8959ae10620caa7fd8a2
Parents: 01e5bc3
Author: Qian Zhang 
Authored: Thu Aug 3 19:29:45 2017 +0800
Committer: Qian Zhang 
Committed: Thu Aug 3 19:29:45 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 102 ++--
 1 file changed, 52 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/9b0c836c/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index 0a16294..baaed8c 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1743,65 +1743,67 @@ INSTANTIATE_TEST_CASE_P(
 DefaultContainerDNSCniTest,
 ::testing::Values(
 // A DNS information for the `__MESOS_TEST__` CNI network.
-R"~(
-{
-  "mesos": [
+string(
+R"~(
 {
-  "network_mode": "CNI",
-  "network_name": "__MESOS_TEST__",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-}
-  ]
-})~",
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "network_name": "__MESOS_TEST__",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+"domain": "mesos.apache.org",
+"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
+"options": [ "timeout:3", "attempts:2" ]
+  }
+}
+  ]
+})~"),
 // A DNS information with `network_mode == CNI`, but without a network
 // name, acts as a wildcard match making it the default DNS for any CNI
 // network not specified in the `--default_container_dns` flag.
-R"~(
-{
-  "mesos": [
+string(
+R"~(
 {
-  "network_mode": "CNI",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-}
-  ]
-})~",
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+"domain": "mesos.apache.org",
+"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
+"options": [ "timeout:3", "attempts:2" ]
+  }
+}
+  ]
+})~"),
 // Two DNS information, one is specific for `__MESOS_TEST__` CNI
 // network, the other is the defaule DNS for any CNI network not
 // specified in the `--default_container_dns` flag.
-R"~(
-{
-  "mesos": [
-{
-  "network_mode": "CNI",
-  "network_name": "__MESOS_TEST__",
-  "dns": {
-"nameservers": [ "8.8.8.8", "8.8.4.4" ],
-"domain": "mesos.apache.org",
-"search": [ "a.mesos.apache.org", "a.mesos.apache.org" ],
-"options": [ "timeout:3", "attempts:2" ]
-  }
-},
+string(
+R"~(
 {
-  "network_mode": "CNI",
-  "dns": {
-"nameservers": [ "8.8.8.9", "8.8.4.5" ],
-"domain": "mesos1.apache.org",
-"search": [ "b.mesos.apache.org", "b.mesos.apache.org" ],
-"options": [ "timeout:9", "attempts:5" ]
-  }
-}
-  ]
-})~"));
+  "mesos": [
+{
+  "network_mode": "CNI",
+  "network_name": "__MESOS_TEST__",
+  "dns": {
+"nameservers": [ "8.8.8.8", "8.8.4.4" ],
+

mesos git commit: Fixed a few typos.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 5693e3554 -> 36c48d81e


Fixed a few typos.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/36c48d81
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/36c48d81
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/36c48d81

Branch: refs/heads/master
Commit: 36c48d81e2af302cb3740887f6a59e15d8f5a7fd
Parents: 5693e35
Author: Qian Zhang 
Authored: Sun Aug 6 21:02:57 2017 +0800
Committer: Qian Zhang 
Committed: Sun Aug 6 21:02:57 2017 +0800

--
 src/linux/routing/diagnosis/diagnosis.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/36c48d81/src/linux/routing/diagnosis/diagnosis.hpp
--
diff --git a/src/linux/routing/diagnosis/diagnosis.hpp 
b/src/linux/routing/diagnosis/diagnosis.hpp
index 7722fd2..55b4fc2 100644
--- a/src/linux/routing/diagnosis/diagnosis.hpp
+++ b/src/linux/routing/diagnosis/diagnosis.hpp
@@ -93,10 +93,10 @@ struct Info
 
 
 // Return a list of socket information that matches the given protocol
-// family and socket states. 'states' can accpet multiple states using
+// family and socket states. 'states' can accept multiple states using
 // bitwise OR.
-// NOTE: 'family' is actually igored here because the older kernel
-// idiag API libnl3 uses only supports TCP and ingores this value. We
+// NOTE: 'family' is actually ignored here because the older kernel
+// idiag API libnl3 uses only supports TCP and ignores this value. We
 // keep it here to follow libnl3-idiag's suit.
 Try infos(int familiy, int states);
 

[1/2] mesos git commit: Added a test `ProtobufTest.ParseJSONUnrecognizedEnum`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 772c8f554 -> 2f3ceb451


Added a test `ProtobufTest.ParseJSONUnrecognizedEnum`.

Review: https://reviews.apache.org/r/61174


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2f3ceb45
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2f3ceb45
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2f3ceb45

Branch: refs/heads/master
Commit: 2f3ceb45106e79586f2c32bfd26db0318d608075
Parents: b10a4ea
Author: Qian Zhang 
Authored: Thu Jul 27 16:15:44 2017 +0800
Committer: Qian Zhang 
Committed: Tue Oct 3 08:53:11 2017 +0800

--
 3rdparty/stout/tests/protobuf_tests.cpp   | 33 ++
 3rdparty/stout/tests/protobuf_tests.proto |  9 +++
 2 files changed, 42 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/2f3ceb45/3rdparty/stout/tests/protobuf_tests.cpp
--
diff --git a/3rdparty/stout/tests/protobuf_tests.cpp 
b/3rdparty/stout/tests/protobuf_tests.cpp
index 8877e89..543f96c 100644
--- a/3rdparty/stout/tests/protobuf_tests.cpp
+++ b/3rdparty/stout/tests/protobuf_tests.cpp
@@ -418,6 +418,39 @@ TEST(ProtobufTest, ParseJSONNestedError)
 }
 
 
+// Tests when parsing protobuf from JSON, for the optional enum field which
+// has an unrecognized enum value, after the parsing the field will be unset
+// and its getter will return the default enum value. For the repeated enum
+// field which contains an unrecognized enum value, after the parsing the
+// field will not contain that unrecognized value anymore.
+TEST(ProtobufTest, ParseJSONUnrecognizedEnum)
+{
+  string message =
+"{"
+"  \"e1\": \"XXX\","
+"  \"e2\": \"\","
+"  \"repeated_enum\": [\"ONE\", \"XXX\", \"\", \"TWO\"]"
+"}";
+
+  Try json = JSON::parse(message);
+  ASSERT_SOME(json);
+
+  Try parse =
+protobuf::parse(json.get());
+
+  ASSERT_SOME(parse);
+
+  EXPECT_FALSE(parse->has_e1());
+  EXPECT_EQ(tests::UNKNOWN, parse->e1());
+  EXPECT_FALSE(parse->has_e2());
+  EXPECT_EQ(tests::UNKNOWN, parse->e2());
+
+  EXPECT_EQ(2, parse->repeated_enum_size());
+  EXPECT_EQ(tests::ONE, parse->repeated_enum(0));
+  EXPECT_EQ(tests::TWO, parse->repeated_enum(1));
+}
+
+
 TEST(ProtobufTest, Jsonify)
 {
   tests::Message message;

http://git-wip-us.apache.org/repos/asf/mesos/blob/2f3ceb45/3rdparty/stout/tests/protobuf_tests.proto
--
diff --git a/3rdparty/stout/tests/protobuf_tests.proto 
b/3rdparty/stout/tests/protobuf_tests.proto
index d16726a..cf8aadc 100644
--- a/3rdparty/stout/tests/protobuf_tests.proto
+++ b/3rdparty/stout/tests/protobuf_tests.proto
@@ -24,6 +24,7 @@ package tests;
 // dynamic message at run-time.
 
 enum Enum {
+  UNKNOWN = 0;
   ONE = 1;
   TWO = 2;
 }
@@ -96,3 +97,11 @@ message Message {
 message ArrayMessage {
   repeated SimpleMessage values = 1;
 }
+
+
+// A message for testing optional enum field.
+message EnumMessage {
+  optional Enum e1 = 1;
+  optional Enum e2 = 2;
+  repeated Enum repeated_enum = 3;
+}

mesos git commit: Allows port mapper plugin to have optional args.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 6f98b8d6d -> 7b5b728da


Allows port mapper plugin to have optional args.

Mesos port mapper cni plugin is a wrapper around bridge plugin
to add port mapping functionality to bridge plugin. However, in
certain cases the network creator doesn't need port mapping
functionality and just want to access bridge plugin. In this case,
the creator may not supply `args` in cni config which will make
mesos port mapper plugin to fail. This patch makes `args` in cni
config optional for mesos port mapper plugin.

Review: https://reviews.apache.org/r/62017/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7b5b728d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7b5b728d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7b5b728d

Branch: refs/heads/master
Commit: 7b5b728da41e0ebecca50a43cba80e680bebb37e
Parents: 6f98b8d
Author: Deepak Goel 
Authored: Wed Sep 6 09:53:20 2017 +0800
Committer: Qian Zhang 
Committed: Wed Sep 6 10:03:40 2017 +0800

--
 .../cni/plugins/port_mapper/port_mapper.cpp| 17 -
 1 file changed, 12 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/7b5b728d/src/slave/containerizer/mesos/isolators/network/cni/plugins/port_mapper/port_mapper.cpp
--
diff --git 
a/src/slave/containerizer/mesos/isolators/network/cni/plugins/port_mapper/port_mapper.cpp
 
b/src/slave/containerizer/mesos/isolators/network/cni/plugins/port_mapper/port_mapper.cpp
index 43cf3e4..de64d65 100644
--- 
a/src/slave/containerizer/mesos/isolators/network/cni/plugins/port_mapper/port_mapper.cpp
+++ 
b/src/slave/containerizer/mesos/isolators/network/cni/plugins/port_mapper/port_mapper.cpp
@@ -140,13 +140,20 @@ Try 
PortMapper::create(const string& _cniConfig)
 
   // While the 'args' field is optional in the CNI spec it is critical
   // to the port-mapper plugin to learn of any port-mappings that the
-  // framework might have requested for this container.
+  // framework might have requested for this container when this plugin
+  // is called in the Mesos context. However, to make the port-mapper
+  // plugin more generic rather than Mesos specific, we will create a
+  // fake 'args` field if it is not filled by the caller.
   Result args = cniConfig->find("args");
-  if (!args.isSome()) {
+  if (args.isError()) {
 return PluginError(
-"Failed to get the required field 'args': " +
-(args.isError() ? args.error() : "Not found"),
-ERROR_BAD_ARGS);
+"Failed to get the field 'args': " + args.error(), ERROR_BAD_ARGS);
+  } else if (args.isNone()) {
+JSON::Object _args;
+JSON::Object mesos;
+mesos.values["network_info"] = JSON::Object();
+_args.values["org.apache.mesos"] = mesos;
+args = _args;
   }
 
   // NOTE: We can't directly use `find` to check for 'network_info'

mesos git commit: Fixed a memory leak in composing containerizer.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 5125b80ea -> 257ee1740


Fixed a memory leak in composing containerizer.

Review: https://reviews.apache.org/r/62237


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/257ee174
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/257ee174
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/257ee174

Branch: refs/heads/master
Commit: 257ee17400dcf66f69d526d768c8fdc1b1740004
Parents: 5125b80
Author: Qian Zhang 
Authored: Tue Sep 12 15:17:43 2017 +0800
Committer: Qian Zhang 
Committed: Tue Sep 12 15:25:25 2017 +0800

--
 src/slave/containerizer/composing.cpp | 15 ++-
 1 file changed, 10 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/257ee174/src/slave/containerizer/composing.cpp
--
diff --git a/src/slave/containerizer/composing.cpp 
b/src/slave/containerizer/composing.cpp
index c076c74..587f009 100644
--- a/src/slave/containerizer/composing.cpp
+++ b/src/slave/containerizer/composing.cpp
@@ -340,6 +340,11 @@ Future ComposingContainerizerProcess::_launch(
 // Note that we don't update the state if a destroy is in progress.
 if (container->state == LAUNCHING) {
   container->state = LAUNCHED;
+
+  // This is needed for eventually removing the given container from
+  // the list of active containers.
+  container->containerizer->wait(containerId)
+.onAny(defer(self(), ::destroy, containerId));
 }
 
 // Note that the return value is not impacted
@@ -483,6 +488,11 @@ Future ComposingContainerizerProcess::_launch(
 // Note that we don't update the state if a destroy is in progress.
 if (container->state == LAUNCHING) {
   container->state = LAUNCHED;
+
+  // This is needed for eventually removing the given container from
+  // the list of active containers.
+  container->containerizer->wait(containerId)
+.onAny(defer(self(), ::destroy, containerId));
 }
 
 // Note that the return value is not impacted
@@ -651,11 +661,6 @@ Future ComposingContainerizerProcess::kill(
 return false;
   }
 
-  // This is needed for eventually removing the given container from
-  // the list of active containers.
-  containers_.at(containerId)->containerizer->wait(containerId)
-.onAny(defer(self(), ::destroy, containerId));
-
   return containers_.at(containerId)->containerizer->kill(containerId, signal);
 }
 

[1/3] mesos git commit: Updated docker executor to return IPv6 address of a container.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 1d980ad06 -> 4468e7bba


Updated docker executor to return IPv6 address of a container.

A docker container can have a v4 and a v6 address. Currently the docker
executor was returning only the IPv4 address. This changes allows the
executor to return the IPv4 and IPv6 address of the container.

Review: https://reviews.apache.org/r/61237/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ee6412cf
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ee6412cf
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ee6412cf

Branch: refs/heads/master
Commit: ee6412cfd5399cfb27c6ce6f4b6351c0e2d1ffa7
Parents: 1d980ad
Author: Avinash sridharan 
Authored: Fri Aug 25 05:25:12 2017 -0700
Committer: Qian Zhang 
Committed: Fri Aug 25 05:25:12 2017 -0700

--
 src/docker/docker.cpp   | 24 +++--
 src/docker/docker.hpp   | 50 +---
 src/docker/executor.cpp | 30 +++---
 3 files changed, 72 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/ee6412cf/src/docker/docker.cpp
--
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 192e170..722a54a 100755
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -319,11 +319,14 @@ Try Docker::Container::create(const 
string& output)
   bool started = startedAtValue.get().value != "0001-01-01T00:00:00Z";
 
   Option ipAddress;
+  Option ip6Address;
   bool findDeprecatedIP = false;
+
   Result networkMode =
 json.find("HostConfig.NetworkMode");
+
   if (!networkMode.isSome()) {
-// We need to fail back to the old field as Docker added NetworkMode
+// We need to fallback to the old field as Docker added NetworkMode
 // since Docker remote API 1.15.
 VLOG(1) << "Unable to detect HostConfig.NetworkMode, "
 << "attempting deprecated IP field";
@@ -341,7 +344,7 @@ Try Docker::Container::create(const 
string& output)
   json.find(addressLocation);
 
 if (!ipAddressValue.isSome()) {
-  // We also need to failback to the old field as the IP Address
+  // We also need to fallback to the old field as the IP Address
   // field location also changed since Docker remote API 1.20.
   VLOG(1) << "Unable to detect IP Address at '" << addressLocation << "',"
   << " attempting deprecated field";
@@ -349,6 +352,22 @@ Try Docker::Container::create(const 
string& output)
 } else if (!ipAddressValue->value.empty()) {
   ipAddress = ipAddressValue->value;
 }
+
+// Check if the container has an IPv6 address.
+//
+// NOTE: For IPv6 we don't need to worry about the old method of
+// looking at the deprecated "NetworkSettings.IPAddress" since we
+// want to support IPv6 addresses for docker versions that support
+// USER mode networking, which is a relatively recent feature.
+string address6Location = "NetworkSettings.Networks." +
+  networkMode->value + ".GlobalIPv6Address";
+
+Result ip6AddressValue =
+  json.find(address6Location);
+
+if (ip6AddressValue.isSome() && !ip6AddressValue->value.empty()) {
+  ip6Address = ip6AddressValue->value;
+}
   }
 
   if (findDeprecatedIP) {
@@ -478,6 +497,7 @@ Try Docker::Container::create(const 
string& output)
   optionalPid,
   started,
   ipAddress,
+  ip6Address,
   devices,
   dns,
   dnsOptions,

http://git-wip-us.apache.org/repos/asf/mesos/blob/ee6412cf/src/docker/docker.hpp
--
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 95e60a7..d9e71f8 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -109,10 +109,14 @@ public:
 // needed since pid is empty when the container terminates.
 const bool started;
 
-// Returns the IPAddress of the container, or None if no IP has
-// been not been assigned.
+// Returns the IPv4 address of the container, or `None()` if no
+// IPv4 address has been assigned.
 const Option ipAddress;
 
+// Returns the IPv6 address of the container, or `None()` if no
+// IPv6 address has been assigned.
+const Option ip6Address;
+
 const std::vector devices;
 
 // Returns the DNS nameservers set by "--dns" option.
@@ -126,26 +130,28 @@ public:
 
   private:
 Container(
-const std::string& output,
-const std::string& id,
-const std::string& name,
-const Option& pid,
-bool started,
-const Option& ipAddress,
-const std::vector& devices,
-const std::vector& dns,
-

[2/3] mesos git commit: Added test filter for docker user network tests.

[qianzhang]

Added test filter for docker user network tests.

Review: https://reviews.apache.org/r/61873/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0a6509b8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0a6509b8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0a6509b8

Branch: refs/heads/master
Commit: 0a6509b8921decd37a935e8a50e3b5f1abd6e327
Parents: ee6412c
Author: Avinash sridharan 
Authored: Fri Aug 25 05:26:28 2017 -0700
Committer: Qian Zhang 
Committed: Fri Aug 25 05:26:28 2017 -0700

--
 src/tests/environment.cpp | 24 ++--
 1 file changed, 22 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/0a6509b8/src/tests/environment.cpp
--
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index a7262cd..607ac6c 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -270,7 +270,12 @@ public:
 flags.docker,
 flags.docker_socket);
 
-if (docker.isError()) {
+if (!docker.isError()) {
+  Try version = docker.get()->validateVersion(Version(1, 9, 0));
+  if (version.isError()) {
+dockerUserNetworkError = version.error();
+  }
+} else {
   dockerError = docker.error();
 }
 #else
@@ -285,15 +290,30 @@ public:
 << "-"
 << std::endl;
 }
+
+if (dockerUserNetworkError.isSome()) {
+  std::cerr
+<< "-\n"
+<< "We cannot run any Docker user network tests because:\n"
+<< dockerUserNetworkError->message << "\n"
+<< "-"
+<< std::endl;
+}
   }
 
   bool disable(const ::testing::TestInfo* test) const
   {
-return matches(test, "DOCKER_") && dockerError.isSome();
+if (dockerError.isSome()) {
+  return matches(test, "DOCKER_");
+}
+
+return matches(test, "DOCKER_USERNETWORK_") &&
+  dockerUserNetworkError.isSome();
   }
 
 private:
   Option dockerError;
+  Option dockerUserNetworkError;
 };
 
 

[3/3] mesos git commit: Added a test for IPv6 containers on docker user networks.

[qianzhang]

Added a test for IPv6 containers on docker user networks.

The test creates a IPv6 docker user network. It than launches an alpine
image on the docker user network. Finally it checks if the IP address
allocated to the container are reflected correctly in state.

Review: https://reviews.apache.org/r/61874/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4468e7bb
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4468e7bb
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4468e7bb

Branch: refs/heads/master
Commit: 4468e7bba090f9ac1b01af567aef6b0f9ac64604
Parents: 0a6509b
Author: Avinash sridharan 
Authored: Fri Aug 25 05:26:35 2017 -0700
Committer: Qian Zhang 
Committed: Fri Aug 25 05:48:10 2017 -0700

--
 .../docker_containerizer_tests.cpp  | 247 +++
 1 file changed, 247 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/4468e7bb/src/tests/containerizer/docker_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp 
b/src/tests/containerizer/docker_containerizer_tests.cpp
index 6856ca2..45f0d1d 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -20,6 +20,7 @@
 
 #include 
 
+#include 
 #include 
 #include 
 #include 
@@ -77,6 +78,8 @@ using testing::Eq;
 using testing::Invoke;
 using testing::Return;
 
+constexpr char DOCKER_IPv6_NETWORK[] = "mesos-docker-ip6-test";
+
 namespace process {
 
 // We need to reinitialize libprocess in order to test against
@@ -4710,6 +4713,250 @@ TEST_F(DockerContainerizerIPv6Test, 
ROOT_DOCKER_LaunchIPv6HostNetwork)
 exists(docker, containerId.get(), ContainerState::RUNNING));
 }
 
+
+// Fixture for testing IPv6 support for docker containers on docker
+// user network.
+class DockerContainerizerIPv6UserNetworkTest : public DockerContainerizerTest
+{
+protected:
+  virtual void SetUp()
+  {
+DockerContainerizerTest::SetUp();
+
+Try dockerCommand = strings::format(
+"docker network create --driver=bridge --ipv6 "
+"--subnet=fd01::/64 %s",
+DOCKER_IPv6_NETWORK);
+
+Try s = subprocess(
+dockerCommand.get(),
+Subprocess::PATH("/dev/null"),
+Subprocess::PATH("/dev/null"),
+Subprocess::PIPE());
+
+ASSERT_SOME(s) << "Unable to create the docker IPv6 network: "
+   << DOCKER_IPv6_NETWORK;
+
+Future err = io::read(s->err().get());
+
+// Wait for the network to be created.
+AWAIT_READY(s->status());
+AWAIT_READY(err);
+
+ASSERT_SOME(s->status().get());
+ASSERT_EQ(s->status().get().get(), 0)
+  << "Unable to create the docker IPv6 network "
+  << DOCKER_IPv6_NETWORK
+  << " : " << err.get();
+  }
+
+  virtual void TearDown()
+  {
+DockerContainerizerTest::TearDown();
+
+Try dockerCommand = strings::format(
+"docker network rm %s",
+DOCKER_IPv6_NETWORK);
+
+Try s = subprocess(
+dockerCommand.get(),
+Subprocess::PATH("/dev/null"),
+Subprocess::PATH("/dev/null"),
+Subprocess::PIPE());
+
+// This is best effort cleanup. In case of an error just a log an
+// error.
+ASSERT_SOME(s) << "Unable to delete the docker IPv6 network: "
+   << DOCKER_IPv6_NETWORK;
+
+Future err = io::read(s->err().get());
+
+// Wait for the network to be deleted.
+AWAIT_READY(s->status());
+AWAIT_READY(err);
+ASSERT_SOME(s->status().get());
+
+ASSERT_EQ(s->status().get().get(), 0)
+  << "Unable to delete the docker IPv6 network "
+  << DOCKER_IPv6_NETWORK
+  << " : " << err.get();
+  }
+};
+
+
+// Launches a docker container on the docker user network. The docker network
+// is assumed to have an IPv4 address and an IPv6 address. The test passes if
+// the Mesos state correctly exposes both the IPv4 and IPv6 address.
+TEST_F(
+DockerContainerizerIPv6UserNetworkTest,
+ROOT_DOCKER_USERNETWORK_LaunchIPv6Container)
+{
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  MockDocker* mockDocker =
+new MockDocker(tests::flags.docker, tests::flags.docker_socket);
+
+  Shared docker(mockDocker);
+
+  slave::Flags flags = CreateSlaveFlags();
+
+  Fetcher fetcher(flags);
+
+  Try logger =
+ContainerLogger::create(flags.container_logger);
+
+  ASSERT_SOME(logger);
+
+  MockDockerContainerizer dockerContainerizer(
+  flags,
+  ,
+  Owned(logger.get()),
+  docker);
+
+  Owned detector = master.get()->createDetector();
+
+  Try slave =
+StartSlave(detector.get(), , flags);
+
+  ASSERT_SOME(slave);
+
+  MockScheduler sched;

[4/4] mesos git commit: Added 3 tests for TCP/HTTP(S) health check support for Docker container.

[qianzhang]

, so we cannot require
-  // executors to authenticate with the agent operator API if Mesos was not
-  // built with SSL support.
-  flags.authenticate_http_readwrite = false;
-
-  // Set permissive ACLs in the agent so that the local authorizer will be
-  // loaded and implicit executor authorization will be tested.
-  ACLs acls;
-  acls.set_permissive(true);
-
-  flags.acls = acls;
-#endif // USE_SSL_SOCKET
+// Tests a healthy Docker task via HTTPS. To emulate a task
+// responsive to HTTPS health checks, starts an HTTPS server
+// in the Docker "zhq527725/https-server" image.
+TEST_P_TEMP_DISABLED_ON_WINDOWS(
+DockerContainerizerHealthCheckTest,
+ROOT_DOCKER_USERNETWORK_HealthyTaskViaHTTPS)
+{
+  Shared docker(new MockDocker(
+  tests::flags.docker, tests::flags.docker_socket));
 
-  Fetcher fetcher(flags);
+  Try<Owned> master = StartMaster();
+  ASSERT_SOME(master);
 
-  // We have to explicitly create a `Containerizer` in non-local mode,
-  // because `LaunchNestedContainerSession` (used by command health
-  // checks) tries to start a IO switchboard, which doesn't work in
-  // local mode yet.
-  Try<MesosContainerizer*> _containerizer =
-MesosContainerizer::create(flags, false, );
+  slave::Flags agentFlags = CreateSlaveFlags();
 
-  ASSERT_SOME(_containerizer);
+  Fetcher fetcher(agentFlags);
 
-  Owned containerizer(_containerizer.get());
-  Owned detector = master.get()->createDetector();
+  Try<ContainerLogger*> logger =
+ContainerLogger::create(agentFlags.container_logger);
+  ASSERT_SOME(logger);
+
+  MockDockerContainerizer containerizer(
+  agentFlags,
+  ,
+  Owned(logger.get()),
+  docker);
 
+  Owned detector = master.get()->createDetector();
   Try<Owned> agent =
-StartSlave(detector.get(), containerizer.get(), flags);
+StartSlave(detector.get(), , agentFlags);
   ASSERT_SOME(agent);
 
   MockScheduler sched;
   MesosSchedulerDriver driver(
-, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+  , DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
 
-  Future frameworkId;
-  EXPECT_CALL(sched, registered(, _, _))
-.WillOnce(FutureArg<1>());
+  EXPECT_CALL(sched, registered(, _, _));
 
   Future<vector> offers;
   EXPECT_CALL(sched, resourceOffers(, _))
@@ -2409,132 +2426,128 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(
 
   driver.start();
 
-  AWAIT_READY(frameworkId);
-
   AWAIT_READY(offers);
   ASSERT_FALSE(offers->empty());
 
-  Future statusStarting;
-  Future statusRunning;
-  Future statusHealthy;
+  TaskInfo task = createTask(
+  offers.get()[0],
+  "python https_server.py 443");
 
-  EXPECT_CALL(sched, statusUpdate(, _))
-.WillOnce(FutureArg<1>())
-.WillOnce(FutureArg<1>())
-.WillOnce(FutureArg<1>());
+  // Refer to https://github.com/qianzhangxa/https-server for
+  // how the Docker image `zhq527725/https-server` works.
+  //
+  // TODO(qianzhang): Use local image to test if possible.
+  ContainerInfo containerInfo;
+  containerInfo.set_type(ContainerInfo::DOCKER);
+  containerInfo.mutable_docker()->set_image("zhq527725/https-server");
+  containerInfo.mutable_docker()->set_network(ContainerInfo::DockerInfo::USER);
 
-  TaskInfo task = createTask(offers->front(), "sleep 120");
+  // Setup the Docker IPv6 network.
+  NetworkInfo networkInfo;
+  networkInfo.set_name(DOCKER_IPv6_NETWORK);
+  containerInfo.add_network_infos()->CopyFrom(networkInfo);
 
-  HealthCheck healthCheck;
+  task.mutable_container()->CopyFrom(containerInfo);
 
-  healthCheck.set_type(HealthCheck::COMMAND);
-  healthCheck.mutable_command()->set_value("exit $STATUS");
+  // Set `grace_period_seconds` here because it takes some time to
+  // launch the HTTPS server to serve requests.
+  HealthCheck healthCheck;
+  healthCheck.set_type(HealthCheck::HTTP);
+  healthCheck.mutable_http()->set_port(443);
+  healthCheck.mutable_http()->set_scheme("https");
+  healthCheck.mutable_http()->set_protocol(GetParam());
   healthCheck.set_delay_seconds(0);
   healthCheck.set_interval_seconds(0);
-  healthCheck.set_grace_period_seconds(0);
-
-  Environment::Variable* variable = healthCheck.mutable_command()->
-mutable_environment()->mutable_variables()->Add();
-  variable->set_name("STATUS");
-  variable->set_value("0");
+  healthCheck.set_grace_period_seconds(15);
 
   task.mutable_health_check()->CopyFrom(healthCheck);
 
-  Resources executorResources =
-allocatedResources(Resources::parse("cpus:0.1;mem:32;disk:32").get(), "*");
+  Future containerId;
+  EXPECT_CALL(containerizer, launch(_, _, _, _))
+.WillOnce(DoAll(FutureArg<0>(),
+Invoke(,
+   ::_launch)));
 
-  task.mutable_resources()->CopyFrom(task.resources() - executorResources);
+  Future statusStar

[2/4] mesos git commit: Made `mesos-tcp-connect` support IPv6.

[qianzhang]

Made `mesos-tcp-connect` support IPv6.

Review: https://reviews.apache.org/r/63795


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f9cb1891
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f9cb1891
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f9cb1891

Branch: refs/heads/master
Commit: f9cb1891a95466b20f9cb88b2a5887b4b99adaae
Parents: cd60edf
Author: Qian Zhang 
Authored: Tue Nov 14 16:28:09 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 28 18:28:58 2017 +0800

--
 src/checks/tcp_connect.cpp | 25 +
 1 file changed, 13 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f9cb1891/src/checks/tcp_connect.cpp
--
diff --git a/src/checks/tcp_connect.cpp b/src/checks/tcp_connect.cpp
index 2514f4a..17d7abf 100644
--- a/src/checks/tcp_connect.cpp
+++ b/src/checks/tcp_connect.cpp
@@ -30,6 +30,9 @@
 #include 
 #include 
 
+#include 
+#include 
+
 #include 
 #include 
 #include 
@@ -49,8 +52,6 @@ using std::string;
 //
 // TODO(alexr): Add support for Windows, see MESOS-6117.
 //
-// TODO(alexr): Support IPv6, see MESOS-6120.
-//
 // NOTE: Consider using stout network abstractions instead of raw system
 // sockets. Once stout supports IPv6 migrating to it will buy us implicit
 // Windows and IPv6 compatibilities.
@@ -63,7 +64,7 @@ public:
   {
 add(::ip,
 "ip",
-"IP of the target host. Only IPv4 is supported.");
+"IP of the target host.");
 
 add(::port,
 "port",
@@ -79,18 +80,14 @@ public:
 // If the TCP handshake is successful, returns `EXIT_SUCCESS`.
 int testTCPConnect(const string& ip, int port)
 {
-  // Set up destination address.
-  struct sockaddr_in to;
-  memset(, 0, sizeof(to));
-  to.sin_family = AF_INET;
-  to.sin_port = htons(port);
-  if (inet_pton(AF_INET, ip.c_str(), _addr) != 1) {
+  Try parse = net::IP::parse(ip);
+  if (parse.isError()){
 cerr << "Cannot convert '" << ip << "' into a network address" << endl;
 return EXIT_FAILURE;
   }
 
   // Create a TCP socket.
-  int socket = ::socket(AF_INET, SOCK_STREAM, 0);
+  int socket = ::socket(parse->family(), SOCK_STREAM, 0);
   if (socket < 0) {
 cerr << "Failed to create socket: " << strerror(errno) << endl;
 return EXIT_FAILURE;
@@ -99,8 +96,12 @@ int testTCPConnect(const string& ip, int port)
   // Try to connect to socket. If the connection succeeds,
   // zero is returned, indicating the remote port is open.
   cout << "Connecting to " << ip << ":" << port << endl;
-  if (connect(socket, reinterpret_cast(), sizeof(to)) < 0) {
-cerr << "Connection failed: " << strerror(errno) << endl;
+  Try connect = process::network::connect(
+  socket,
+  process::network::inet::Address(parse.get(), port));
+
+  if (connect.isError()) {
+cerr << connect.error().message << endl;
 close(socket);
 return EXIT_FAILURE;
   }

[3/4] mesos git commit: Added a new member field `ipv6` to the `CheckerProcess` class.

[qianzhang]

Added a new member field `ipv6` to the `CheckerProcess` class.

Review: https://reviews.apache.org/r/63794


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cd60edf0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cd60edf0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cd60edf0

Branch: refs/heads/master
Commit: cd60edf0875bff951b95fbcb5cc5dae5e58bd894
Parents: 237b30d
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Mon Nov 13 15:16:32 2017 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Tue Nov 28 18:28:58 2017 +0800

--
 src/checks/checker_process.cpp |  6 --
 src/checks/checker_process.hpp | 13 +++--
 src/checks/health_checker.cpp  | 11 ++-
 3 files changed, 25 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/cd60edf0/src/checks/checker_process.cpp
--
diff --git a/src/checks/checker_process.cpp b/src/checks/checker_process.cpp
index 7985e8d..02ce7e2 100644
--- a/src/checks/checker_process.cpp
+++ b/src/checks/checker_process.cpp
@@ -195,7 +195,8 @@ CheckerProcess::CheckerProcess(
 const Option& _authorizationHeader,
 const Option& _scheme,
 const std::string& _name,
-bool _commandCheckViaAgent)
+bool _commandCheckViaAgent,
+bool _ipv6)
   : ProcessBase(process::ID::generate("checker")),
 check(_check),
 launcherDir(_launcherDir),
@@ -207,8 +208,9 @@ CheckerProcess::CheckerProcess(
 agentURL(_agentURL),
 authorizationHeader(_authorizationHeader),
 scheme(_scheme),
-commandCheckViaAgent(_commandCheckViaAgent),
 name(_name),
+commandCheckViaAgent(_commandCheckViaAgent),
+ipv6(_ipv6),
 paused(false)
 {
   Try create = Duration::create(check.delay_seconds());

http://git-wip-us.apache.org/repos/asf/mesos/blob/cd60edf0/src/checks/checker_process.hpp
--
diff --git a/src/checks/checker_process.hpp b/src/checks/checker_process.hpp
index 5d0c36d..510f3b2 100644
--- a/src/checks/checker_process.hpp
+++ b/src/checks/checker_process.hpp
@@ -55,7 +55,8 @@ public:
   const Option& _authorizationHeader,
   const Option& _scheme,
   const std::string& _name,
-  bool _commandCheckViaAgent);
+  bool _commandCheckViaAgent,
+  bool _ipv6 = false);
 
   void pause();
   void resume();
@@ -136,8 +137,16 @@ private:
   const Option agentURL;
   const Option authorizationHeader;
   const Option scheme;
-  const bool commandCheckViaAgent;
   const std::string name;
+  const bool commandCheckViaAgent;
+
+  // If set to true, the TCP/HTTP(S) check will be performed over IPv6,
+  // otherwise, it will be performed over IPv4.
+  //
+  // TODO(qianzhang): Once we figure out how the IPv4/IPv6 should be supported
+  // in the health check API (i.e., the `CheckInfo` protobuf message), we may
+  // consider to remove this field which is a temporary solution for now.
+  const bool ipv6;
 
   Option<lambda::function<pid_t(const lambda::function<int()>&)>> clone;
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/cd60edf0/src/checks/health_checker.cpp
--
diff --git a/src/checks/health_checker.cpp b/src/checks/health_checker.cpp
index d4bda6e..eaf9a18 100644
--- a/src/checks/health_checker.cpp
+++ b/src/checks/health_checker.cpp
@@ -262,6 +262,14 @@ HealthChecker::HealthChecker(
 scheme = healthCheck.http().scheme();
   }
 
+  bool ipv6 = false;
+  if ((healthCheck.type() == HealthCheck::HTTP &&
+   healthCheck.http().protocol() == NetworkInfo::IPv6) ||
+  (healthCheck.type() == HealthCheck::TCP &&
+   healthCheck.tcp().protocol() == NetworkInfo::IPv6)) {
+ipv6 = true;
+  }
+
   process.reset(
   new CheckerProcess(
   toCheckInfo(_healthCheck),
@@ -275,7 +283,8 @@ HealthChecker::HealthChecker(
   authorizationHeader,
   scheme,
   name,
-  commandCheckViaAgent));
+  commandCheckViaAgent,
+  ipv6));
 
   spawn(process.get());
 }

[1/4] mesos git commit: Made `CheckerProcess` support IPv6 for HTTP/TCP check.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 237b30dc9 -> 00dfbc80b


Made `CheckerProcess` support IPv6 for HTTP/TCP check.

Review: https://reviews.apache.org/r/63796


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/83805df1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/83805df1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/83805df1

Branch: refs/heads/master
Commit: 83805df12adc19b6d71fec0c41a779cadcfab0e0
Parents: f9cb189
Author: Qian Zhang 
Authored: Tue Nov 14 16:32:01 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 28 18:28:58 2017 +0800

--
 src/checks/checker_process.cpp | 21 -
 1 file changed, 16 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/83805df1/src/checks/checker_process.cpp
--
diff --git a/src/checks/checker_process.cpp b/src/checks/checker_process.cpp
index 02ce7e2..496bd41 100644
--- a/src/checks/checker_process.cpp
+++ b/src/checks/checker_process.cpp
@@ -94,9 +94,10 @@ constexpr char TCP_CHECK_COMMAND[] = "mesos-tcp-connect.exe";
 
 static const string DEFAULT_HTTP_SCHEME = "http";
 
-// Use '127.0.0.1' instead of 'localhost', because the host
-// file in some container images may not contain 'localhost'.
-constexpr char DEFAULT_DOMAIN[] = "127.0.0.1";
+// Use '127.0.0.1' and '::1' instead of 'localhost', because the
+// host file in some container images may not contain 'localhost'.
+constexpr char DEFAULT_IPV4_DOMAIN[] = "127.0.0.1";
+constexpr char DEFAULT_IPV6_DOMAIN[] = "::1";
 
 
 #ifdef __linux__
@@ -827,7 +828,15 @@ Future CheckerProcess::httpCheck()
 
   const string _scheme = scheme.isSome() ? scheme.get() : DEFAULT_HTTP_SCHEME;
   const string path = http.has_path() ? http.path() : "";
-  const string url = _scheme + "://" + DEFAULT_DOMAIN + ":" +
+
+  // As per "curl --manual", the square brackets are required to tell curl that
+  // it's an IPv6 address, and we need to set "-g" option below to stop curl
+  // from interpreting the square brackets as special globbing characters.
+  const string domain = ipv6 ?
+"[" + string(DEFAULT_IPV6_DOMAIN) + "]" :
+DEFAULT_IPV4_DOMAIN;
+
+  const string url = _scheme + "://" + domain + ":" +
  stringify(http.port()) + path;
 
   VLOG(1) << "Launching " << name << " '" << url << "'"
@@ -841,6 +850,7 @@ Future CheckerProcess::httpCheck()
 "-k", // Ignores SSL validation when scheme is https.
 "-w", "%{http_code}", // Displays HTTP response code on stdout.
 "-o", os::DEV_NULL,   // Ignores output.
+"-g", // Switches off the "URL globbing parser".
 url
   };
 
@@ -994,10 +1004,11 @@ Future CheckerProcess::tcpCheck()
   << " at port " << tcp.port();
 
   const string command = path::join(launcherDir, TCP_CHECK_COMMAND);
+  const string domain = ipv6 ? DEFAULT_IPV6_DOMAIN : DEFAULT_IPV4_DOMAIN;
 
   const vector argv = {
 command,
-"--ip=" + stringify(DEFAULT_DOMAIN),
+"--ip=" + domain,
 "--port=" + stringify(tcp.port())
   };
 

[3/4] mesos git commit: Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

[qianzhang]

Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

Review: https://reviews.apache.org/r/62327


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e9408127
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e9408127
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e9408127

Branch: refs/heads/1.4.x
Commit: e940812741d80b1ed3b5b3645f34e93e950ba119
Parents: 371959c
Author: Qian Zhang 
Authored: Thu Sep 14 17:33:05 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:31:27 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/e9408127/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index 60c85ad..0ff0867 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1707,13 +1707,7 @@ TEST_F(CniIsolatorPortMapperTest, 
ROOT_INTERNET_CURL_PortMapper)
 
   AWAIT_READY(statusKilled);
 
-  // The executor would issue a SIGTERM to the container, followed by
-  // a SIGKILL (in case the container ignores the SIGTERM). The
-  // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on
-  // receiving a SIGTERM making the executor send a `TASK_FINISHED`
-  // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED`
-  // instead of `TASK_KILLED`.
-  EXPECT_EQ(TASK_FINISHED, statusKilled.get().state());
+  EXPECT_EQ(TASK_KILLED, statusKilled.get().state());
 
   AWAIT_READY(gcSchedule);
 

[4/4] mesos git commit: Added release notes for 1.4.2 and added MESOS-7975 to it.

[qianzhang]

Added release notes for 1.4.2 and added MESOS-7975 to it.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b2bf5542
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b2bf5542
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b2bf5542

Branch: refs/heads/1.4.x
Commit: b2bf5542da52e65ea8c766139099605b54fbf408
Parents: e940812
Author: Qian Zhang 
Authored: Tue Nov 14 17:35:48 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:35:48 2017 +0800

--
 CHANGELOG | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/b2bf5542/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index d485e07..ab8844b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,11 @@
+Release Notes - Mesos - Version 1.4.2 (WIP)
+-
+* This is a bug fix release.
+
+** Bug
+ * [MESOS-7975] - The command/default/docker executor can incorrectly send a 
TASK_FINISHED update even when the task is killed.
+
+
 Release Notes - Mesos - Version 1.4.1
 -
 * This is a bug fix release.

[4/4] mesos git commit: Added MESOS-7975 to the 1.3.2 CHANGELOG.

[qianzhang]

Added MESOS-7975 to the 1.3.2 CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/022e5240
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/022e5240
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/022e5240

Branch: refs/heads/1.3.x
Commit: 022e5240523986f4a3ae938516b0c73c14576a40
Parents: 9417078
Author: Qian Zhang 
Authored: Tue Nov 14 17:29:49 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:29:49 2017 +0800

--
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/022e5240/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index 6ef0a0e..a2c913d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -15,6 +15,7 @@ All Issues:
   * [MESOS-7912] - Master WebUI not working in Chrome.
   * [MESOS-7926] - Abnormal termination of default executor can cause 
MesosContainerizer::destroy to fail.
   * [MESOS-7934] - OOM due to LibeventSSLSocket send incorrectly returning 0 
after shutdown.
+  * [MESOS-7975] - The command/default/docker executor can incorrectly send a 
TASK_FINISHED update even when the task is killed.
   * [MESOS-8051] - Killing TASK_GROUP fail to kill some tasks.
   * [MESOS-8080] - The default executor does not propagate missing task exit 
status correctly.
   * [MESOS-8135] - Masters can lose track of tasks' executor IDs.

[2/4] mesos git commit: Always send TASK_KILLED when the task is killed by a framework.

[qianzhang]

Always send TASK_KILLED when the task is killed by a framework.

This change is done for command, docker and default executors.

Review: https://reviews.apache.org/r/62326


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b320e2fc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b320e2fc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b320e2fc

Branch: refs/heads/1.3.x
Commit: b320e2fc73498f97931aa48785a22ae90a33b4de
Parents: 5da5ace
Author: Qian Zhang 
Authored: Thu Sep 14 17:29:39 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:26:23 2017 +0800

--
 src/docker/executor.cpp   | 6 +++---
 src/launcher/default_executor.cpp | 6 +++---
 src/launcher/executor.cpp | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/b320e2fc/src/docker/executor.cpp
--
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index c6f50fa..79cd4e5 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -495,9 +495,7 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-state = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown(). Note that in general there is a
 // race between signaling the container and it terminating
@@ -507,6 +505,8 @@ private:
 // determine whether the container was terminated via
 // our signal or terminated on its own.
 state = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+state = TASK_FINISHED;
   } else {
 state = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/b320e2fc/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index 81aa6e4..3c6f220 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -773,12 +773,12 @@ protected:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (container->killing) {
+  if (container->killing) {
 // Send TASK_KILLED if the task was killed as a result of
 // `killTask()` or `shutdown()`.
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/b320e2fc/src/launcher/executor.cpp
--
diff --git a/src/launcher/executor.cpp b/src/launcher/executor.cpp
index b05f73e..61c1512 100644
--- a/src/launcher/executor.cpp
+++ b/src/launcher/executor.cpp
@@ -891,12 +891,12 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown().
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

[1/4] mesos git commit: Updated the comments of TASK_FINISHED to make it more clear.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.3.x fa1d61003 -> 022e52405


Updated the comments of TASK_FINISHED to make it more clear.

Review: https://reviews.apache.org/r/62685


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5da5ace2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5da5ace2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5da5ace2

Branch: refs/heads/1.3.x
Commit: 5da5ace2f7abdf8ecc158d9c1a8b875f43be9591
Parents: fa1d610
Author: Qian Zhang 
Authored: Fri Sep 29 15:28:58 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:26:22 2017 +0800

--
 include/mesos/mesos.proto| 4 +++-
 include/mesos/v1/mesos.proto | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/5da5ace2/include/mesos/mesos.proto
--
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index fa23dc9..23207eb 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1700,7 +1700,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

http://git-wip-us.apache.org/repos/asf/mesos/blob/5da5ace2/include/mesos/v1/mesos.proto
--
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 564b5f0..31e841b 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1694,7 +1694,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

[3/4] mesos git commit: Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

[qianzhang]

Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

Review: https://reviews.apache.org/r/62327


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/94170788
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/94170788
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/94170788

Branch: refs/heads/1.3.x
Commit: 941707881795dc5ea63e581ca6a646bed8fae93d
Parents: b320e2f
Author: Qian Zhang 
Authored: Thu Sep 14 17:33:05 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:26:23 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/94170788/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index 565e58a..1f410ec 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1420,13 +1420,7 @@ TEST_F(CniIsolatorPortMapperTest, 
ROOT_INTERNET_CURL_PortMapper)
 
   AWAIT_READY(statusKilled);
 
-  // The executor would issue a SIGTERM to the container, followed by
-  // a SIGKILL (in case the container ignores the SIGTERM). The
-  // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on
-  // receiving a SIGTERM making the executor send a `TASK_FINISHED`
-  // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED`
-  // instead of `TASK_KILLED`.
-  EXPECT_EQ(TASK_FINISHED, statusKilled.get().state());
+  EXPECT_EQ(TASK_KILLED, statusKilled.get().state());
 
   AWAIT_READY(gcSchedule);
 

[2/4] mesos git commit: Updated the comments of TASK_FINISHED to make it more clear.

[qianzhang]

Updated the comments of TASK_FINISHED to make it more clear.

Review: https://reviews.apache.org/r/62685


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/065a9cae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/065a9cae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/065a9cae

Branch: refs/heads/1.4.x
Commit: 065a9cae060795fee082d5adf1ca1130b5827762
Parents: c844db9
Author: Qian Zhang 
Authored: Fri Sep 29 15:28:58 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:31:26 2017 +0800

--
 include/mesos/mesos.proto| 4 +++-
 include/mesos/v1/mesos.proto | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/065a9cae/include/mesos/mesos.proto
--
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 3449c2d..c602501 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -2028,7 +2028,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

http://git-wip-us.apache.org/repos/asf/mesos/blob/065a9cae/include/mesos/v1/mesos.proto
--
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 4d905d3..4612b40 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -2011,7 +2011,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

[1/4] mesos git commit: Always send TASK_KILLED when the task is killed by a framework.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.4.x c844db9ac -> b2bf5542d


Always send TASK_KILLED when the task is killed by a framework.

This change is done for command, docker and default executors.

Review: https://reviews.apache.org/r/62326


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/371959c5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/371959c5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/371959c5

Branch: refs/heads/1.4.x
Commit: 371959c5709946a8ecb4ecb49eced9612ca4a8fb
Parents: 065a9ca
Author: Qian Zhang 
Authored: Thu Sep 14 17:29:39 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:31:26 2017 +0800

--
 src/docker/executor.cpp   | 6 +++---
 src/launcher/default_executor.cpp | 6 +++---
 src/launcher/executor.cpp | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/371959c5/src/docker/executor.cpp
--
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 99a6224..5c430dc 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -500,9 +500,7 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-state = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown(). Note that in general there is a
 // race between signaling the container and it terminating
@@ -512,6 +510,8 @@ private:
 // determine whether the container was terminated via
 // our signal or terminated on its own.
 state = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+state = TASK_FINISHED;
   } else {
 state = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/371959c5/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index 4d30c94..eff7e45 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -809,12 +809,12 @@ protected:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (container->killing) {
+  if (container->killing) {
 // Send TASK_KILLED if the task was killed as a result of
 // `killTask()` or `shutdown()`.
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/371959c5/src/launcher/executor.cpp
--
diff --git a/src/launcher/executor.cpp b/src/launcher/executor.cpp
index 65b9a2c..12b0326 100644
--- a/src/launcher/executor.cpp
+++ b/src/launcher/executor.cpp
@@ -909,12 +909,12 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown().
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

[1/3] mesos git commit: Updated the comments of TASK_FINISHED to make it more clear.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.2.x fdadfa416 -> f8706e580


Updated the comments of TASK_FINISHED to make it more clear.

Review: https://reviews.apache.org/r/62685


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/00d79da8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/00d79da8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/00d79da8

Branch: refs/heads/1.2.x
Commit: 00d79da80956f1c9a75f1c6802e483d8c58127de
Parents: fdadfa4
Author: Qian Zhang 
Authored: Fri Sep 29 15:28:58 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 16:59:18 2017 +0800

--
 include/mesos/mesos.proto| 4 +++-
 include/mesos/v1/mesos.proto | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/00d79da8/include/mesos/mesos.proto
--
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index ae2de6b..ee767a1 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1678,7 +1678,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

http://git-wip-us.apache.org/repos/asf/mesos/blob/00d79da8/include/mesos/v1/mesos.proto
--
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index c0ad505..609d836 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1677,7 +1677,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

[3/3] mesos git commit: Added MESOS-7975 to the 1.2.3 CHANGELOG.

[qianzhang]

Added MESOS-7975 to the 1.2.3 CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f8706e58
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f8706e58
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f8706e58

Branch: refs/heads/1.2.x
Commit: f8706e580251cc7ad7cb6971577f761c6525952b
Parents: bfadf67
Author: Qian Zhang 
Authored: Tue Nov 14 17:19:05 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 17:19:05 2017 +0800

--
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f8706e58/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index 6553e68..6f9a72d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -16,6 +16,7 @@ All Issues:
   * [MESOS-7909] - Ordering dependency between 'linux/capabilities' and 
'docker/runtime' isolator.
   * [MESOS-7926] - Abnormal termination of default executor can cause 
MesosContainerizer::destroy to fail.
   * [MESOS-7934] - OOM due to LibeventSSLSocket send incorrectly returning 0 
after shutdown.
+  * [MESOS-7975] - The command/default/docker executor can incorrectly send a 
TASK_FINISHED update even when the task is killed.
   * [MESOS-8051] - Killing TASK_GROUP fail to kill some tasks.
   * [MESOS-8080] - The default executor does not propagate missing task exit 
status correctly.
   * [MESOS-8135] - Masters can lose track of tasks' executor IDs.

[2/3] mesos git commit: Always send TASK_KILLED when the task is killed by a framework.

[qianzhang]

Always send TASK_KILLED when the task is killed by a framework.

This change is done for command, docker and default executors.

Review: https://reviews.apache.org/r/62326


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bfadf67a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bfadf67a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bfadf67a

Branch: refs/heads/1.2.x
Commit: bfadf67a38eedbc1cdd3edfa0c6ea9b15b6d4eb3
Parents: 00d79da
Author: Qian Zhang 
Authored: Thu Sep 14 17:29:39 2017 +0800
Committer: Qian Zhang 
Committed: Tue Nov 14 16:59:19 2017 +0800

--
 src/docker/executor.cpp   | 6 +++---
 src/launcher/default_executor.cpp | 6 +++---
 src/launcher/executor.cpp | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/bfadf67a/src/docker/executor.cpp
--
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index d7f2134..ac826e6 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -463,9 +463,7 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-state = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown(). Note that in general there is a
 // race between signaling the container and it terminating
@@ -475,6 +473,8 @@ private:
 // determine whether the container was terminated via
 // our signal or terminated on its own.
 state = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+state = TASK_FINISHED;
   } else {
 state = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/bfadf67a/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index c03df7c..e04043a 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -650,12 +650,12 @@ protected:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (container->killing) {
+  if (container->killing) {
 // Send TASK_KILLED if the task was killed as a result of
 // `killTask()` or `shutdown()`.
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/bfadf67a/src/launcher/executor.cpp
--
diff --git a/src/launcher/executor.cpp b/src/launcher/executor.cpp
index 2605d82..5b278dd 100644
--- a/src/launcher/executor.cpp
+++ b/src/launcher/executor.cpp
@@ -700,12 +700,12 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown().
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

[1/2] mesos git commit: Ignored the tasks already being killed when killing the task group.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.3.x b6f6e7a13 -> fa1d61003


Ignored the tasks already being killed when killing the task group.

When the scheduler tries to kill multiple tasks in the task group
simultaneously, the default executor will kill the tasks one by
one. When the first task is killed, the default executor will kill
all the other tasks in the task group, however, we need to ignore
the tasks which are already being killed, otherwise, the check
`CHECK(!container->killing);` in `DefaultExecutor::kill()` will fail.

Review: https://reviews.apache.org/r/62836


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/25325711
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/25325711
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/25325711

Branch: refs/heads/1.3.x
Commit: 253257111d79dff5b1587c9047a55802ea3eca0d
Parents: b6f6e7a
Author: Qian Zhang 
Authored: Mon Oct 9 09:01:15 2017 +0800
Committer: Qian Zhang 
Committed: Sat Nov 4 11:08:39 2017 +0800

--
 src/launcher/default_executor.cpp | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/25325711/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index da496e3..81aa6e4 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -855,6 +855,14 @@ protected:
 Owned container_ = containers.at(taskId);
 container_->killingTaskGroup = true;
 
+// Ignore if the task is already being killed. This can happen
+// when the scheduler tries to kill multiple tasks in the task
+// group simultaneously and then one of the tasks is killed
+// while the other tasks are still being killed, see MESOS-8051.
+if (container_->killing) {
+  continue;
+}
+
 kill(container_);
   }
 }

[1/2] mesos git commit: Ignored the tasks already being killed when killing the task group.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.2.x a7fb1fbce -> fdadfa416


Ignored the tasks already being killed when killing the task group.

When the scheduler tries to kill multiple tasks in the task group
simultaneously, the default executor will kill the tasks one by
one. When the first task is killed, the default executor will kill
all the other tasks in the task group, however, we need to ignore
the tasks which are already being killed, otherwise, the check
`CHECK(!container->killing);` in `DefaultExecutor::kill()` will fail.

Review: https://reviews.apache.org/r/62836


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d381f730
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d381f730
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d381f730

Branch: refs/heads/1.2.x
Commit: d381f730aaed386c3013d234b6591d0ae62dcf10
Parents: a7fb1fb
Author: Qian Zhang 
Authored: Mon Oct 9 09:01:15 2017 +0800
Committer: Qian Zhang 
Committed: Sat Nov 4 10:58:14 2017 +0800

--
 src/launcher/default_executor.cpp | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/d381f730/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index f30947e..c03df7c 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -724,6 +724,14 @@ protected:
 Owned container_ = containers.at(taskId);
 container_->killingTaskGroup = true;
 
+// Ignore if the task is already being killed. This can happen
+// when the scheduler tries to kill multiple tasks in the task
+// group simultaneously and then one of the tasks is killed
+// while the other tasks are still being killed, see MESOS-8051.
+if (container_->killing) {
+  continue;
+}
+
 kill(container_);
   }
 }

[2/2] mesos git commit: Added MESOS-8051 to the 1.3.2 CHANGELOG.

[qianzhang]

Added MESOS-8051 to the 1.3.2 CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fa1d6100
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fa1d6100
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fa1d6100

Branch: refs/heads/1.3.x
Commit: fa1d610039c85518fa8ca868864f43aadfdf1bb5
Parents: 2532571
Author: Qian Zhang 
Authored: Sat Nov 4 20:28:51 2017 +0800
Committer: Qian Zhang 
Committed: Sat Nov 4 20:28:51 2017 +0800

--
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/fa1d6100/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index 410feae..6ef0a0e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -15,6 +15,7 @@ All Issues:
   * [MESOS-7912] - Master WebUI not working in Chrome.
   * [MESOS-7926] - Abnormal termination of default executor can cause 
MesosContainerizer::destroy to fail.
   * [MESOS-7934] - OOM due to LibeventSSLSocket send incorrectly returning 0 
after shutdown.
+  * [MESOS-8051] - Killing TASK_GROUP fail to kill some tasks.
   * [MESOS-8080] - The default executor does not propagate missing task exit 
status correctly.
   * [MESOS-8135] - Masters can lose track of tasks' executor IDs.
 

[2/2] mesos git commit: Added MESOS-8051 to the 1.4.1 CHANGELOG.

[qianzhang]

Added MESOS-8051 to the 1.4.1 CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a2b346b9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a2b346b9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a2b346b9

Branch: refs/heads/1.4.x
Commit: a2b346b987ace2bf41e451d223dc0ce4fe643ab7
Parents: ca03e92
Author: Qian Zhang 
Authored: Sat Nov 4 20:30:32 2017 +0800
Committer: Qian Zhang 
Committed: Sat Nov 4 20:30:32 2017 +0800

--
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a2b346b9/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index 4a24ede..5f79455 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,7 @@ All Resolved Issues:
  * [MESOS-7968] - Handle `/proc/self/ns/pid_for_children` when parsing 
available namespace.
  * [MESOS-7969] - Handle cgroups v2 hierarchy when parsing /proc/self/cgroups.
  * [MESOS-7980] - Stout fails to compile with libc >= 2.26.
+ * [MESOS-8051] - Killing TASK_GROUP fail to kill some tasks.
  * [MESOS-8080] - The default executor does not propagate missing task exit 
status correctly.
  * [MESOS-8135] - Masters can lose track of tasks' executor IDs.
 

[1/2] mesos git commit: Ignored the tasks already being killed when killing the task group.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/1.4.x 049bfecc0 -> a2b346b98


Ignored the tasks already being killed when killing the task group.

When the scheduler tries to kill multiple tasks in the task group
simultaneously, the default executor will kill the tasks one by
one. When the first task is killed, the default executor will kill
all the other tasks in the task group, however, we need to ignore
the tasks which are already being killed, otherwise, the check
`CHECK(!container->killing);` in `DefaultExecutor::kill()` will fail.

Review: https://reviews.apache.org/r/62836


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ca03e929
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ca03e929
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ca03e929

Branch: refs/heads/1.4.x
Commit: ca03e92976b30742a11c8a140e1d984b4fad44cc
Parents: 049bfec
Author: Qian Zhang 
Authored: Mon Oct 9 09:01:15 2017 +0800
Committer: Qian Zhang 
Committed: Sat Nov 4 11:11:10 2017 +0800

--
 src/launcher/default_executor.cpp | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/ca03e929/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index bc0ce7c..4d30c94 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -893,6 +893,14 @@ protected:
 Owned container_ = containers.at(taskId);
 container_->killingTaskGroup = true;
 
+// Ignore if the task is already being killed. This can happen
+// when the scheduler tries to kill multiple tasks in the task
+// group simultaneously and then one of the tasks is killed
+// while the other tasks are still being killed, see MESOS-8051.
+if (container_->killing) {
+  continue;
+}
+
 kill(container_);
   }
 }

[3/5] mesos git commit: Updated the comments of TASK_FINISHED to make it more clear.

[qianzhang]

Updated the comments of TASK_FINISHED to make it more clear.

Review: https://reviews.apache.org/r/62685


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ce834764
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ce834764
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ce834764

Branch: refs/heads/master
Commit: ce83476449a0c2cfa6d192b4d017d302d92fea7b
Parents: 8165188
Author: Qian Zhang 
Authored: Fri Sep 29 15:28:58 2017 +0800
Committer: Qian Zhang 
Committed: Wed Nov 1 15:34:10 2017 +0800

--
 include/mesos/mesos.proto| 4 +++-
 include/mesos/v1/mesos.proto | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/ce834764/include/mesos/mesos.proto
--
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 4966cd2..68a5538 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -2110,7 +2110,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

http://git-wip-us.apache.org/repos/asf/mesos/blob/ce834764/include/mesos/v1/mesos.proto
--
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index e18ac00..c46cec7 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -2091,7 +2091,9 @@ enum TaskState {
   // the TASK_KILLING_STATE capability.
   TASK_KILLING = 8;  // The task is being killed by the executor.
 
-  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
+  // The task finished successfully on its own without external interference.
+  TASK_FINISHED = 2; // TERMINAL.
+
   TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
   TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
   TASK_ERROR = 7;// TERMINAL: The task description contains an error.

[1/5] mesos git commit: Added a test `ROOT_DOCKER_NoTransitionFromKillingToFinished`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 816518867 -> 815263ead


Added a test `ROOT_DOCKER_NoTransitionFromKillingToFinished`.

Review: https://reviews.apache.org/r/62774


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f569aeee
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f569aeee
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f569aeee

Branch: refs/heads/master
Commit: f569aeee31ec7a9b9352fad729c6bdd7ef592f48
Parents: 8e3cb6a
Author: Qian Zhang 
Authored: Wed Oct 4 15:44:41 2017 +0800
Committer: Qian Zhang 
Committed: Wed Nov 1 15:34:10 2017 +0800

--
 .../docker_containerizer_tests.cpp  | 123 +++
 1 file changed, 123 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f569aeee/src/tests/containerizer/docker_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp 
b/src/tests/containerizer/docker_containerizer_tests.cpp
index 419be5d..ad05da6 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -4137,6 +4137,129 @@ TEST_F(DockerContainerizerTest, 
ROOT_DOCKER_NoTransitionFromKillingToRunning)
 }
 
 
+// This test ensures that a task will transition from `TASK_KILLING`
+// to `TASK_KILLED` rather than `TASK_FINISHED` when it is killed,
+// even if it returns an "EXIT_STATUS" of 0 on receiving a SIGTERM.
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_NoTransitionFromKillingToFinished)
+{
+  Shared docker(new MockDocker(
+  tests::flags.docker, tests::flags.docker_socket));
+
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  slave::Flags agentFlags = CreateSlaveFlags();
+
+  Fetcher fetcher(agentFlags);
+
+  Try logger =
+ContainerLogger::create(agentFlags.container_logger);
+
+  ASSERT_SOME(logger);
+
+  MockDockerContainerizer containerizer(
+  agentFlags,
+  ,
+  Owned(logger.get()),
+  docker);
+
+  Owned detector = master.get()->createDetector();
+
+  Try agent =
+StartSlave(detector.get(), , agentFlags);
+
+  ASSERT_SOME(agent);
+
+  // Start the framework with the task killing capability.
+  FrameworkInfo::Capability capability;
+  capability.set_type(FrameworkInfo::Capability::TASK_KILLING_STATE);
+
+  FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+  frameworkInfo.add_capabilities()->CopyFrom(capability);
+
+  MockScheduler sched;
+  MesosSchedulerDriver driver(
+  , frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+  EXPECT_CALL(sched, registered(, _, _));
+
+  Future offers;
+  EXPECT_CALL(sched, resourceOffers(, _))
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  driver.start();
+
+  AWAIT_READY(offers);
+  EXPECT_EQ(1u, offers->size());
+
+  const Offer& offer = offers.get()[0];
+
+  TaskInfo task;
+  task.set_name("");
+  task.mutable_task_id()->set_value("1");
+  task.mutable_slave_id()->CopyFrom(offer.slave_id());
+  task.mutable_resources()->CopyFrom(offer.resources());
+
+  CommandInfo command;
+  command.set_shell(false);
+
+  ContainerInfo containerInfo;
+  containerInfo.set_type(ContainerInfo::DOCKER);
+
+  // The "nginx:alpine" container returns an "EXIT_STATUS" of 0 on
+  // receiving a SIGTERM.
+  //
+  // TODO(tnachen): Use local image to test if possible.
+  ContainerInfo::DockerInfo dockerInfo;
+  dockerInfo.set_image("nginx:alpine");
+  containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+  task.mutable_command()->CopyFrom(command);
+  task.mutable_container()->CopyFrom(containerInfo);
+
+  Future containerId;
+  EXPECT_CALL(containerizer, launch(_, _, _, _))
+.WillOnce(DoAll(FutureArg<0>(),
+Invoke(,
+   ::_launch)));
+
+  Future statusRunning;
+  Future statusKilling;
+  Future statusKilled;
+
+  EXPECT_CALL(sched, statusUpdate(, _))
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>());
+
+  driver.launchTasks(offers->front().id(), {task});
+
+  AWAIT_READY_FOR(containerId, Seconds(60));
+  AWAIT_READY_FOR(statusRunning, Seconds(60));
+  EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+  // Docker executor will call "docker stop ..." to send SIGTERM
+  // to kill the task.
+  driver.killTask(task.task_id());
+
+  AWAIT_READY(statusKilling);
+  EXPECT_EQ(TASK_KILLING, statusKilling->state());
+
+  AWAIT_READY(statusKilled);
+  EXPECT_EQ(TASK_KILLED, statusKilled->state());
+
+  Future termination =
+containerizer.wait(containerId.get());
+
+  driver.stop();
+  driver.join();
+
+  AWAIT_READY(termination);
+  

[4/5] mesos git commit: Always send TASK_KILLED when the task is killed by a framework.

[qianzhang]

Always send TASK_KILLED when the task is killed by a framework.

This change is done for command, docker and default executors.

Review: https://reviews.apache.org/r/62326


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0449a084
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0449a084
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0449a084

Branch: refs/heads/master
Commit: 0449a0840d154fc169fb0b1d866d8c6b87ca5eae
Parents: ce83476
Author: Qian Zhang 
Authored: Thu Sep 14 17:29:39 2017 +0800
Committer: Qian Zhang 
Committed: Wed Nov 1 15:34:10 2017 +0800

--
 src/docker/executor.cpp   | 6 +++---
 src/launcher/default_executor.cpp | 6 +++---
 src/launcher/executor.cpp | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/0449a084/src/docker/executor.cpp
--
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 5f6a0d0..3974f20 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -524,9 +524,7 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-state = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown(). Note that in general there is a
 // race between signaling the container and it terminating
@@ -536,6 +534,8 @@ private:
 // determine whether the container was terminated via
 // our signal or terminated on its own.
 state = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+state = TASK_FINISHED;
   } else {
 state = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/0449a084/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index 3064052..248c46d 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -827,12 +827,12 @@ protected:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (container->killing) {
+  if (container->killing) {
 // Send TASK_KILLED if the task was killed as a result of
 // `killTask()` or `shutdown()`.
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/0449a084/src/launcher/executor.cpp
--
diff --git a/src/launcher/executor.cpp b/src/launcher/executor.cpp
index 34f6f7a..c688c04 100644
--- a/src/launcher/executor.cpp
+++ b/src/launcher/executor.cpp
@@ -913,12 +913,12 @@ private:
   CHECK(WIFEXITED(status) || WIFSIGNALED(status))
 << "Unexpected wait status " << status;
 
-  if (WSUCCEEDED(status)) {
-taskState = TASK_FINISHED;
-  } else if (killed) {
+  if (killed) {
 // Send TASK_KILLED if the task was killed as a result of
 // kill() or shutdown().
 taskState = TASK_KILLED;
+  } else if (WSUCCEEDED(status)) {
+taskState = TASK_FINISHED;
   } else {
 taskState = TASK_FAILED;
   }

[2/5] mesos git commit: Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

[qianzhang]

Checked TASK_KILLED in the test `ROOT_INTERNET_CURL_PortMapper`.

Review: https://reviews.apache.org/r/62327


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8e3cb6ae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8e3cb6ae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8e3cb6ae

Branch: refs/heads/master
Commit: 8e3cb6ae6ca7d25fe768467ecc0a65f7509fce43
Parents: 0449a08
Author: Qian Zhang 
Authored: Thu Sep 14 17:33:05 2017 +0800
Committer: Qian Zhang 
Committed: Wed Nov 1 15:34:10 2017 +0800

--
 src/tests/containerizer/cni_isolator_tests.cpp | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/8e3cb6ae/src/tests/containerizer/cni_isolator_tests.cpp
--
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp 
b/src/tests/containerizer/cni_isolator_tests.cpp
index 5a2af59..0d68dd6 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -1802,13 +1802,7 @@ TEST_F(CniIsolatorPortMapperTest, 
ROOT_INTERNET_CURL_PortMapper)
 
   AWAIT_READY(statusKilled);
 
-  // The executor would issue a SIGTERM to the container, followed by
-  // a SIGKILL (in case the container ignores the SIGTERM). The
-  // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on
-  // receiving a SIGTERM making the executor send a `TASK_FINISHED`
-  // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED`
-  // instead of `TASK_KILLED`.
-  EXPECT_EQ(TASK_FINISHED, statusKilled.get().state());
+  EXPECT_EQ(TASK_KILLED, statusKilled.get().state());
 
   AWAIT_READY(gcSchedule);
 

[5/5] mesos git commit: Added a test `ROOT_NoTransitionFromKillingToFinished`.

[qianzhang]

Added a test `ROOT_NoTransitionFromKillingToFinished`.

Review: https://reviews.apache.org/r/62775


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/815263ea
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/815263ea
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/815263ea

Branch: refs/heads/master
Commit: 815263ead42f64fd0ff320614cfada30b2e8f899
Parents: f569aee
Author: Qian Zhang 
Authored: Wed Oct 4 23:38:11 2017 +0800
Committer: Qian Zhang 
Committed: Wed Nov 1 17:21:45 2017 +0800

--
 src/tests/default_executor_tests.cpp  | 181 +
 src/tests/kill_policy_test_helper.cpp |   9 ++
 2 files changed, 190 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/815263ea/src/tests/default_executor_tests.cpp
--
diff --git a/src/tests/default_executor_tests.cpp 
b/src/tests/default_executor_tests.cpp
index f485b61..21950f5 100644
--- a/src/tests/default_executor_tests.cpp
+++ b/src/tests/default_executor_tests.cpp
@@ -48,6 +48,7 @@
 
 #include "tests/cluster.hpp"
 #include "tests/containerizer.hpp"
+#include "tests/kill_policy_test_helper.hpp"
 #include "tests/mesos.hpp"
 
 #include "slave/containerizer/mesos/containerizer.hpp"
@@ -1409,6 +1410,186 @@ TEST_P(DefaultExecutorTest, SigkillExecutor)
 }
 
 
+// This test verifies that a task will transition from `TASK_KILLING`
+// to `TASK_KILLED` rather than `TASK_FINISHED` when it is killed,
+// even if it returns an "EXIT_STATUS" of 0 on receiving a SIGTERM.
+TEST_P(DefaultExecutorTest, ROOT_NoTransitionFromKillingToFinished)
+{
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  slave::Flags flags = CreateSlaveFlags();
+  flags.containerizers = GetParam();
+
+  Fetcher fetcher(flags);
+
+  Try create =
+MesosContainerizer::create(flags, true, );
+
+  ASSERT_SOME(create);
+
+  Owned containerizer(create.get());
+
+  Owned detector = master.get()->createDetector();
+
+  Try slave = StartSlave(
+detector.get(),
+containerizer.get(),
+flags);
+
+  ASSERT_SOME(slave);
+
+  auto scheduler = std::make_shared();
+
+  // Start the framework with the task killing capability.
+  v1::FrameworkInfo::Capability capability;
+  capability.set_type(v1::FrameworkInfo::Capability::TASK_KILLING_STATE);
+
+  v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
+  frameworkInfo.add_capabilities()->CopyFrom(capability);
+
+  EXPECT_CALL(*scheduler, connected(_))
+.WillOnce(v1::scheduler::SendSubscribe(frameworkInfo));
+
+  Future subscribed;
+  EXPECT_CALL(*scheduler, subscribed(_, _))
+.WillOnce(FutureArg<1>());
+
+  Future offers;
+  EXPECT_CALL(*scheduler, offers(_, _))
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  EXPECT_CALL(*scheduler, heartbeat(_))
+.WillRepeatedly(Return()); // Ignore heartbeats.
+
+  v1::scheduler::TestMesos mesos(
+  master.get()->pid,
+  ContentType::PROTOBUF,
+  scheduler);
+
+  AWAIT_READY(subscribed);
+  v1::FrameworkID frameworkId(subscribed->framework_id());
+
+  v1::ExecutorInfo executorInfo = v1::createExecutorInfo(
+  v1::DEFAULT_EXECUTOR_ID,
+  None(),
+  "cpus:0.1;mem:32;disk:32",
+  v1::ExecutorInfo::DEFAULT,
+  frameworkId);
+
+  AWAIT_READY(offers);
+  ASSERT_FALSE(offers->offers().empty());
+
+  const v1::Offer& offer = offers->offers(0);
+  const v1::AgentID& agentId = offer.agent_id();
+
+  v1::CommandInfo commandInfo;
+  commandInfo.set_shell(false);
+  commandInfo.set_value(getTestHelperPath("test-helper"));
+  commandInfo.add_arguments("test-helper");
+  commandInfo.add_arguments(KillPolicyTestHelper::NAME);
+  commandInfo.add_arguments("--sleep_duration=0");
+
+  v1::TaskInfo taskInfo = v1::createTask(
+  agentId,
+  v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(),
+  commandInfo);
+
+  Future startingUpdate;
+  Future runningUpdate;
+  EXPECT_CALL(*scheduler, update(_, _))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)));
+
+  mesos.send(
+  v1::createCallAccept(
+  frameworkId,
+  offer,
+  {v1::LAUNCH_GROUP(
+  executorInfo, v1::createTaskGroupInfo({taskInfo}))}));
+
+  AWAIT_READY(startingUpdate);
+  ASSERT_EQ(TASK_STARTING, startingUpdate->status().state());
+
+  AWAIT_READY_FOR(runningUpdate, Seconds(60));
+  ASSERT_EQ(TASK_RUNNING, runningUpdate->status().state());
+  ASSERT_EQ(taskInfo.task_id(), 

mesos git commit: Fixed the unit test that broke Windows build.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 027bd9905 -> de19f7381


Fixed the unit test that broke Windows build.

Also updated an out-of-date comment in command_executor_tests.cpp.

Review: https://reviews.apache.org/r/63499/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/de19f738
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/de19f738
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/de19f738

Branch: refs/heads/master
Commit: de19f7381108d491fb41fe3d8ae13e9893213bbc
Parents: 027bd99
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Fri Nov 3 08:06:34 2017 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Fri Nov 3 08:06:34 2017 +0800

--
 src/tests/command_executor_tests.cpp | 4 ++--
 src/tests/default_executor_tests.cpp | 4 
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/de19f738/src/tests/command_executor_tests.cpp
--
diff --git a/src/tests/command_executor_tests.cpp 
b/src/tests/command_executor_tests.cpp
index 6d6f916..8d36454 100644
--- a/src/tests/command_executor_tests.cpp
+++ b/src/tests/command_executor_tests.cpp
@@ -230,8 +230,8 @@ TEST_P(CommandExecutorTest, TaskKillingCapability)
 }
 
 
-// TODO(hausdorff): Kill policy helpers are not yet enabled on Windows. See
-// MESOS-6698.
+// TODO(qianzhang): Kill policy helpers are not yet enabled on Windows. See
+// MESOS-8168.
 #ifndef __WINDOWS__
 // This test ensures that a task will transition straight from `TASK_KILLING` 
to
 // `TASK_KILLED`, even if the health check begins to fail during the kill 
policy

http://git-wip-us.apache.org/repos/asf/mesos/blob/de19f738/src/tests/default_executor_tests.cpp
--
diff --git a/src/tests/default_executor_tests.cpp 
b/src/tests/default_executor_tests.cpp
index 21950f5..6511398 100644
--- a/src/tests/default_executor_tests.cpp
+++ b/src/tests/default_executor_tests.cpp
@@ -1410,6 +1410,9 @@ TEST_P(DefaultExecutorTest, SigkillExecutor)
 }
 
 
+// TODO(qianzhang): Kill policy helpers are not yet enabled on Windows. See
+// MESOS-8168.
+#ifndef __WINDOWS__
 // This test verifies that a task will transition from `TASK_KILLING`
 // to `TASK_KILLED` rather than `TASK_FINISHED` when it is killed,
 // even if it returns an "EXIT_STATUS" of 0 on receiving a SIGTERM.
@@ -1588,6 +1591,7 @@ TEST_P(DefaultExecutorTest, 
ROOT_NoTransitionFromKillingToFinished)
   ASSERT_TRUE(wait.get()->has_status());
   ASSERT_EQ(0, wait.get()->status());
 }
+#endif // __WINDOWS__
 
 
 #ifdef __linux__

[1/2] mesos git commit: Ignored the tasks already being killed when killing the task group.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master f201bb558 -> 28831de34


Ignored the tasks already being killed when killing the task group.

When the scheduler tries to kill multiple tasks in the task group
simultaneously, the default executor will kill the tasks one by
one. When the first task is killed, the default executor will kill
all the other tasks in the task group, however, we need to ignore
the tasks which are already being killed, otherwise, the check
`CHECK(!container->killing);` in `DefaultExecutor::kill()` will fail.

Review: https://reviews.apache.org/r/62836


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/05c7dd88
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/05c7dd88
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/05c7dd88

Branch: refs/heads/master
Commit: 05c7dd88f269692b7248c1087a3f57759eba6853
Parents: f201bb5
Author: Qian Zhang 
Authored: Mon Oct 9 09:01:15 2017 +0800
Committer: Qian Zhang 
Committed: Tue Oct 31 11:21:15 2017 +0800

--
 src/launcher/default_executor.cpp | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/05c7dd88/src/launcher/default_executor.cpp
--
diff --git a/src/launcher/default_executor.cpp 
b/src/launcher/default_executor.cpp
index cdb3c3e..3064052 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -938,6 +938,14 @@ protected:
 Owned container_ = containers.at(taskId);
 container_->killingTaskGroup = true;
 
+// Ignore if the task is already being killed. This can happen
+// when the scheduler tries to kill multiple tasks in the task
+// group simultaneously and then one of the tasks is killed
+// while the other tasks are still being killed, see MESOS-8051.
+if (container_->killing) {
+  continue;
+}
+
 kill(container_);
   }
 }

[2/2] mesos git commit: Added a test `DefaultExecutorTest.KillMultipleTasks`.

[qianzhang]

Added a test `DefaultExecutorTest.KillMultipleTasks`.

Review: https://reviews.apache.org/r/62837


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/28831de3
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/28831de3
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/28831de3

Branch: refs/heads/master
Commit: 28831de34d098c894042246dd6fef402eb3b960d
Parents: 05c7dd8
Author: Qian Zhang 
Authored: Mon Oct 9 14:25:31 2017 +0800
Committer: Qian Zhang 
Committed: Tue Oct 31 12:47:17 2017 +0800

--
 src/tests/default_executor_tests.cpp | 141 ++
 1 file changed, 141 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/28831de3/src/tests/default_executor_tests.cpp
--
diff --git a/src/tests/default_executor_tests.cpp 
b/src/tests/default_executor_tests.cpp
index 5078bd4..f485b61 100644
--- a/src/tests/default_executor_tests.cpp
+++ b/src/tests/default_executor_tests.cpp
@@ -441,6 +441,147 @@ TEST_P(DefaultExecutorTest, KillTask)
 }
 
 
+// This is a regression test for MESOS-8051. It verifies that if the
+// default executor is asked to kill all tasks from a task group
+// simultaneously, all the tasks can be successfully killed and the
+// default executor can send TASK_KILLED updates for all of them.
+TEST_P(DefaultExecutorTest, KillMultipleTasks)
+{
+  Try master = StartMaster();
+  ASSERT_SOME(master);
+
+  slave::Flags flags = CreateSlaveFlags();
+  flags.containerizers = GetParam();
+
+  Owned detector = master.get()->createDetector();
+  Try slave = StartSlave(detector.get(), flags);
+  ASSERT_SOME(slave);
+
+  auto scheduler = std::make_shared();
+
+  EXPECT_CALL(*scheduler, connected(_))
+.WillOnce(v1::scheduler::SendSubscribe(v1::DEFAULT_FRAMEWORK_INFO));
+
+  Future subscribed;
+  EXPECT_CALL(*scheduler, subscribed(_, _))
+.WillOnce(FutureArg<1>());
+
+  Future offers;
+  EXPECT_CALL(*scheduler, offers(_, _))
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(Return());
+
+  EXPECT_CALL(*scheduler, heartbeat(_))
+.WillRepeatedly(Return()); // Ignore heartbeats.
+
+  v1::scheduler::TestMesos mesos(
+  master.get()->pid,
+  ContentType::PROTOBUF,
+  scheduler);
+
+  AWAIT_READY(subscribed);
+  v1::FrameworkID frameworkId(subscribed->framework_id());
+
+  v1::Resources resources =
+v1::Resources::parse("cpus:0.1;mem:32;disk:32").get();
+
+  v1::ExecutorInfo executorInfo = v1::createExecutorInfo(
+  v1::DEFAULT_EXECUTOR_ID,
+  None(),
+  resources,
+  v1::ExecutorInfo::DEFAULT,
+  frameworkId);
+
+  AWAIT_READY(offers);
+  ASSERT_FALSE(offers->offers().empty());
+
+  const v1::Offer& offer = offers->offers(0);
+  const v1::AgentID& agentId = offer.agent_id();
+
+  v1::TaskInfo taskInfo1 =
+v1::createTask(agentId, resources, SLEEP_COMMAND(1000));
+
+  v1::TaskInfo taskInfo2 =
+v1::createTask(agentId, resources, SLEEP_COMMAND(1000));
+
+  const hashset tasks{taskInfo1.task_id(), taskInfo2.task_id()};
+
+  Future startingUpdate1;
+  Future startingUpdate2;
+  Future runningUpdate1;
+  Future runningUpdate2;
+  EXPECT_CALL(*scheduler, update(_, _))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)))
+.WillOnce(
+DoAll(
+FutureArg<1>(),
+v1::scheduler::SendAcknowledge(frameworkId, agentId)));
+
+  mesos.send(
+  v1::createCallAccept(
+  frameworkId,
+  offer,
+  {v1::LAUNCH_GROUP(
+  executorInfo, v1::createTaskGroupInfo({taskInfo1, 
taskInfo2}))}));
+
+  AWAIT_READY(startingUpdate1);
+  ASSERT_EQ(TASK_STARTING, startingUpdate1->status().state());
+
+  AWAIT_READY(startingUpdate2);
+  ASSERT_EQ(TASK_STARTING, startingUpdate2->status().state());
+
+  AWAIT_READY(runningUpdate1);
+  ASSERT_EQ(TASK_RUNNING, runningUpdate1->status().state());
+
+  AWAIT_READY(runningUpdate2);
+  ASSERT_EQ(TASK_RUNNING, runningUpdate2->status().state());
+
+  // When running a task, TASK_RUNNING updates for the tasks in a
+  // task group can be received in any order.
+  const hashset tasksRunning{
+runningUpdate1->status().task_id(),
+runningUpdate2->status().task_id()};
+
+  ASSERT_EQ(tasks, tasksRunning);
+
+  Future killedUpdate1;
+  Future killedUpdate2;
+  EXPECT_CALL(*scheduler, update(_, _))
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>());
+
+  // Now kill all tasks 

mesos git commit: Added `libacl` into a few Dockerfiles.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master aa6594728 -> 617d55e24


Added `libacl` into a few Dockerfiles.

This commit adds `libacl` into Dockerfiles for the images:
  1. mesos/mesos-build
  2. mesos/mesos-tidy
  2. mesos/mesos-mini

Review: https://reviews.apache.org/r/66840


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/617d55e2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/617d55e2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/617d55e2

Branch: refs/heads/master
Commit: 617d55e24a3bef7305b75c8fc6cbd1d1f14d7f6a
Parents: aa65947
Author: Qian Zhang 
Authored: Fri Apr 27 09:30:48 2018 +0800
Committer: Qian Zhang 
Committed: Fri Apr 27 21:40:38 2018 +0800

--
 support/mesos-build/centos-7.dockerfile | 1 +
 support/mesos-build/ubuntu-16.04-arm.dockerfile | 1 +
 support/mesos-build/ubuntu-16.04.dockerfile | 1 +
 support/mesos-tidy/Dockerfile   | 1 +
 support/packaging/centos/mesos.spec | 1 +
 5 files changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/617d55e2/support/mesos-build/centos-7.dockerfile
--
diff --git a/support/mesos-build/centos-7.dockerfile 
b/support/mesos-build/centos-7.dockerfile
index acdb44d..068f946 100644
--- a/support/mesos-build/centos-7.dockerfile
+++ b/support/mesos-build/centos-7.dockerfile
@@ -34,6 +34,7 @@ RUN yum groupinstall -y 'Development Tools' && \
   devtoolset-4-gcc-c++ \
   git \
   java-1.8.0-openjdk-devel \
+  libacl-devel \
   libcurl-devel \
   libevent-devel \
   libev-devel \

http://git-wip-us.apache.org/repos/asf/mesos/blob/617d55e2/support/mesos-build/ubuntu-16.04-arm.dockerfile
--
diff --git a/support/mesos-build/ubuntu-16.04-arm.dockerfile 
b/support/mesos-build/ubuntu-16.04-arm.dockerfile
index 6d76090..352156f 100644
--- a/support/mesos-build/ubuntu-16.04-arm.dockerfile
+++ b/support/mesos-build/ubuntu-16.04-arm.dockerfile
@@ -25,6 +25,7 @@ RUN apt-get update && \
   curl \
   git \
   iputils-ping \
+  libacl1-dev \
   libapr1-dev \
   libcurl4-nss-dev \
   libev-dev \

http://git-wip-us.apache.org/repos/asf/mesos/blob/617d55e2/support/mesos-build/ubuntu-16.04.dockerfile
--
diff --git a/support/mesos-build/ubuntu-16.04.dockerfile 
b/support/mesos-build/ubuntu-16.04.dockerfile
index 0bbaa03..503b2e3 100644
--- a/support/mesos-build/ubuntu-16.04.dockerfile
+++ b/support/mesos-build/ubuntu-16.04.dockerfile
@@ -25,6 +25,7 @@ RUN apt-get update && \
   curl \
   git \
   iputils-ping \
+  libacl1-dev \
   libapr1-dev \
   libcurl4-nss-dev \
   libev-dev \

http://git-wip-us.apache.org/repos/asf/mesos/blob/617d55e2/support/mesos-tidy/Dockerfile
--
diff --git a/support/mesos-tidy/Dockerfile b/support/mesos-tidy/Dockerfile
index f046a14..51dfd37 100644
--- a/support/mesos-tidy/Dockerfile
+++ b/support/mesos-tidy/Dockerfile
@@ -56,6 +56,7 @@ ENV PATH /opt/bin:$PATH
 # TODO(mpark): Remove `libssl-dev` from this list once `MESOS-6942` is 
resolved.
 RUN apt-get install -qy \
   autoconf \
+  libacl1-dev \
   libapr1-dev \
   libcurl4-nss-dev \
   libsasl2-dev \

http://git-wip-us.apache.org/repos/asf/mesos/blob/617d55e2/support/packaging/centos/mesos.spec
--
diff --git a/support/packaging/centos/mesos.spec 
b/support/packaging/centos/mesos.spec
index 752c2aa..0532985 100644
--- a/support/packaging/centos/mesos.spec
+++ b/support/packaging/centos/mesos.spec
@@ -35,6 +35,7 @@ BuildRequires: python-devel
 BuildRequires: java-1.8.0-openjdk-devel
 BuildRequires: libnl3-devel
 BuildRequires: zlib-devel
+BuildRequires: libacl-devel
 BuildRequires: libcurl-devel
 BuildRequires: openssl-devel
 BuildRequires: cyrus-sasl-devel

mesos git commit: Fixed flaky `TerminatedNestedStatus` test.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 415451ee8 -> fcc8ca168


Fixed flaky `TerminatedNestedStatus` test.

This patch adds call of `destroy()` for a parent container.

Review: https://reviews.apache.org/r/67317/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fcc8ca16
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fcc8ca16
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fcc8ca16

Branch: refs/heads/master
Commit: fcc8ca168523c4bd5b3c1b748a60c8926fe66ef5
Parents: 415451e
Author: Andrei Budnik 
Authored: Fri May 25 21:33:10 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 21:33:10 2018 +0800

--
 .../containerizer/nested_mesos_containerizer_tests.cpp  | 9 +
 1 file changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/fcc8ca16/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp 
b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
index dac30d7..6050e6e 100644
--- a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
+++ b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
@@ -2761,6 +2761,15 @@ TEST_F(NestedMesosContainerizerTest, 
ROOT_CGROUPS_TerminatedNestedStatus)
   ASSERT_SOME(nestedTermination.get());
   ASSERT_TRUE(nestedTermination.get()->has_status());
   EXPECT_WEXITSTATUS_EQ(42, nestedTermination.get()->status());
+
+  // Destroy the top-level container.
+  Future termination =
+containerizer->destroy(containerId);
+
+  AWAIT_READY(termination);
+  ASSERT_SOME(termination.get());
+  ASSERT_TRUE(termination.get()->has_status());
+  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
 }
 
 

[4/8] mesos git commit: Enabled composing containerizer as a default containerizer in tests.

[qianzhang]

Enabled composing containerizer as a default containerizer in tests.

This patch enforces all tests that start an agent to use composing
containerizer. This is needed to make sure that composing containerizer
is fairly covered by tests.

Review: https://reviews.apache.org/r/66817/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3d156db1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3d156db1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3d156db1

Branch: refs/heads/master
Commit: 3d156db1b02fdd607246a899d399fe5010a1c560
Parents: bf901da
Author: Andrei Budnik 
Authored: Fri May 25 09:08:13 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:13 2018 +0800

--
 src/tests/cluster.cpp | 21 -
 1 file changed, 20 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/3d156db1/src/tests/cluster.cpp
--
diff --git a/src/tests/cluster.cpp b/src/tests/cluster.cpp
index b56212f..a65c4fa 100644
--- a/src/tests/cluster.cpp
+++ b/src/tests/cluster.cpp
@@ -92,6 +92,7 @@
 #include "slave/slave.hpp"
 #include "slave/task_status_update_manager.hpp"
 
+#include "slave/containerizer/composing.hpp"
 #include "slave/containerizer/containerizer.hpp"
 #include "slave/containerizer/fetcher.hpp"
 
@@ -434,10 +435,28 @@ Try Slave::create(
   return Error("Failed to create containerizer: " + 
_containerizer.error());
 }
 
-slave->ownedContainerizer.reset(_containerizer.get());
 slave->containerizer = _containerizer.get();
   }
 
+  // As composing containerizer doesn't affect behaviour of underlying
+  // containerizers, we can always use composing containerizer turned on
+  // by default in tests.
+  if (!dynamic_cast(slave->containerizer)) {
+Try composing =
+  slave::ComposingContainerizer::create({slave->containerizer});
+
+if (composing.isError()) {
+  return Error(
+  "Failed to create composing containerizer: " + composing.error());
+}
+
+slave->containerizer = composing.get();
+  }
+
+  if (containerizer.isNone()) {
+slave->ownedContainerizer.reset(slave->containerizer);
+  }
+
   Option authorizer = providedAuthorizer;
 
   // If the authorizer is not provided, create a default one.

[5/8] mesos git commit: Removed extra `containerizer->wait()` calls in tests.

[qianzhang]

Removed extra `containerizer->wait()` calls in tests.

Previously, `wait()` and `destroy()` containerizer methods returned
different types, so it was necessarry to call `wait()` before calling
`destroy()` to get the process's exit status. Now, as both methods
return `ContainerTermination`, we can get rid of redundant `wait()`
calls.

Review: https://reviews.apache.org/r/67128/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a8eac5bb
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a8eac5bb
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a8eac5bb

Branch: refs/heads/master
Commit: a8eac5bb33fd23e4fc114e8941cde0a666a17e84
Parents: 3d156db
Author: Andrei Budnik 
Authored: Fri May 25 09:08:18 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:18 2018 +0800

--
 src/tests/cluster.cpp   |  14 +-
 .../docker_containerizer_tests.cpp  |   4 +-
 .../containerizer/io_switchboard_tests.cpp  |  26 +-
 .../linux_filesystem_isolator_tests.cpp |   4 +-
 .../containerizer/mesos_containerizer_tests.cpp |  58 ++--
 .../nested_mesos_containerizer_tests.cpp| 273 +--
 .../volume_image_isolator_tests.cpp |  26 +-
 .../volume_sandbox_path_isolator_tests.cpp  |  26 +-
 .../volume_secret_isolator_tests.cpp|  12 +-
 src/tests/slave_recovery_tests.cpp  |  45 ++-
 10 files changed, 211 insertions(+), 277 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a8eac5bb/src/tests/cluster.cpp
--
diff --git a/src/tests/cluster.cpp b/src/tests/cluster.cpp
index a65c4fa..01eb095 100644
--- a/src/tests/cluster.cpp
+++ b/src/tests/cluster.cpp
@@ -673,18 +673,16 @@ Slave::~Slave()
 }
 
 foreach (const ContainerID& containerId, containers.get()) {
-  process::Future wait =
-containerizer->wait(containerId);
-
-  process::Future destroy =
+  process::Future termination =
 containerizer->destroy(containerId);
 
-  AWAIT(destroy);
-  AWAIT(wait);
+  AWAIT(termination);
 
-  if (!wait.isReady()) {
+  if (!termination.isReady()) {
 LOG(ERROR) << "Failed to destroy container " << containerId << ": "
-   << (wait.isFailed() ? wait.failure() : "discarded");
+   << (termination.isFailed() ?
+   termination.failure() :
+   "discarded");
   }
 }
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/a8eac5bb/src/tests/containerizer/docker_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp 
b/src/tests/containerizer/docker_containerizer_tests.cpp
index f474c71..ab00d6d 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -1862,9 +1862,7 @@ TEST_F(DockerContainerizerTest, 
ROOT_DOCKER_RecoverPersistentVolumes)
   AWAIT_READY(_recover);
 
   Future termination =
-dockerContainerizer->wait(containerId.get());
-
-  dockerContainerizer->destroy(containerId.get());
+dockerContainerizer->destroy(containerId.get());
 
   AWAIT_READY(termination);
   EXPECT_SOME(termination.get());

http://git-wip-us.apache.org/repos/asf/mesos/blob/a8eac5bb/src/tests/containerizer/io_switchboard_tests.cpp
--
diff --git a/src/tests/containerizer/io_switchboard_tests.cpp 
b/src/tests/containerizer/io_switchboard_tests.cpp
index 784386a..c00f6a9 100644
--- a/src/tests/containerizer/io_switchboard_tests.cpp
+++ b/src/tests/containerizer/io_switchboard_tests.cpp
@@ -754,16 +754,13 @@ TEST_F(IOSwitchboardTest, ContainerAttach)
   Future connection = containerizer->attach(containerId);
   AWAIT_READY(connection);
 
-  Future wait = containerizer->wait(containerId);
-
-  Future destroy =
+  Future termination =
 containerizer->destroy(containerId);
-  AWAIT_READY(destroy);
 
-  AWAIT_READY(wait);
-  ASSERT_SOME(wait.get());
-  ASSERT_TRUE(wait.get()->has_status());
-  EXPECT_WTERMSIG_EQ(SIGKILL, wait.get()->status());
+  AWAIT_READY(termination);
+  ASSERT_SOME(termination.get());
+  ASSERT_TRUE(termination.get()->has_status());
+  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
 }
 
 
@@ -906,15 +903,14 @@ TEST_F(IOSwitchboardTest, 
KillSwitchboardContainerDestroyed)
   ASSERT_EQ(TaskStatus::REASON_IO_SWITCHBOARD_EXITED,
 wait.get()->reason());
 
-  wait = containerizer->wait(containerId);
-
-  containerizer->destroy(containerId);
+  

[8/8] mesos git commit: Added test to verify presence of nested container termination status.

[qianzhang]

Added test to verify presence of nested container termination status.

This test verifies that both mesos and composing containerizers
maintain the contract described in the Containerizer API regarding
availability of a termination status for terminated nested containers.

Review: https://reviews.apache.org/r/67135/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d2ab700c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d2ab700c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d2ab700c

Branch: refs/heads/master
Commit: d2ab700cdc76056155d60b77fe0f4a210b59723d
Parents: a1ce9ad
Author: Andrei Budnik 
Authored: Fri May 25 09:08:43 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:43 2018 +0800

--
 .../nested_mesos_containerizer_tests.cpp| 98 
 1 file changed, 98 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/d2ab700c/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp 
b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
index 3481726..dac30d7 100644
--- a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
+++ b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
@@ -40,6 +40,8 @@
 #include "linux/cgroups.hpp"
 #include "linux/ns.hpp"
 
+#include "slave/containerizer/composing.hpp"
+
 #include "slave/containerizer/mesos/launch.hpp"
 #include "slave/containerizer/mesos/linux_launcher.hpp"
 #include "slave/containerizer/mesos/paths.hpp"
@@ -2666,6 +2668,102 @@ TEST_F(NestedMesosContainerizerTest, 
ROOT_CGROUPS_WaitAfterDestroy)
 }
 
 
+// This test verifies that a container termination status for a terminated
+// nested container is available via `wait()` and `destroy()` methods for
+// both mesos and composing containerizers.
+TEST_F(NestedMesosContainerizerTest, ROOT_CGROUPS_TerminatedNestedStatus)
+{
+  slave::Flags flags = CreateSlaveFlags();
+  flags.launcher = "linux";
+  flags.isolation = "cgroups/cpu,filesystem/linux,namespaces/pid";
+
+  Fetcher fetcher(flags);
+
+  Try create = MesosContainerizer::create(
+  flags,
+  true,
+  );
+
+  ASSERT_SOME(create);
+
+  MesosContainerizer* mesosContainerizer(create.get());
+
+  Try composing =
+slave::ComposingContainerizer::create({mesosContainerizer});
+
+  ASSERT_SOME(composing);
+
+  Owned containerizer(composing.get());
+
+  SlaveID slaveId = SlaveID();
+
+  // Launch a top-level container.
+  ContainerID containerId;
+  containerId.set_value(id::UUID::random().toString());
+
+  Try directory = environment->mkdtemp();
+  ASSERT_SOME(directory);
+
+  Future launch = containerizer->launch(
+containerId,
+createContainerConfig(
+None(),
+createExecutorInfo("executor", "sleep 1000", "cpus:1"),
+directory.get()),
+map(),
+None());
+
+  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);
+
+  // Launch a nested container.
+  ContainerID nestedContainerId;
+  nestedContainerId.mutable_parent()->CopyFrom(containerId);
+  nestedContainerId.set_value(id::UUID::random().toString());
+
+  launch = containerizer->launch(
+  nestedContainerId,
+  createContainerConfig(createCommandInfo("exit 42")),
+  map(),
+  None());
+
+  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);
+
+  // Verify that both `wait` and `destroy` methods of composing containerizer
+  // return the same container termination for a terminated nested container.
+  Future nestedWait =
+containerizer->wait(nestedContainerId);
+
+  AWAIT_READY(nestedWait);
+  ASSERT_SOME(nestedWait.get());
+  ASSERT_TRUE(nestedWait.get()->has_status());
+  EXPECT_WEXITSTATUS_EQ(42, nestedWait.get()->status());
+
+  Future nestedTermination =
+containerizer->destroy(nestedContainerId);
+
+  AWAIT_READY(nestedTermination);
+  ASSERT_SOME(nestedTermination.get());
+  ASSERT_TRUE(nestedTermination.get()->has_status());
+  EXPECT_WEXITSTATUS_EQ(42, nestedTermination.get()->status());
+
+  // Verify that both `wait` and `destroy` methods of mesos containerizer
+  // return the same container termination for a terminated nested container.
+  nestedWait = mesosContainerizer->wait(nestedContainerId);
+
+  AWAIT_READY(nestedWait);
+  ASSERT_SOME(nestedWait.get());
+  ASSERT_TRUE(nestedWait.get()->has_status());
+  EXPECT_WEXITSTATUS_EQ(42, nestedWait.get()->status());
+
+  nestedTermination = mesosContainerizer->destroy(nestedContainerId);
+
+  AWAIT_READY(nestedTermination);
+  ASSERT_SOME(nestedTermination.get());
+  

[3/8] mesos git commit: Updated composing containerizer tests.

[qianzhang]

Updated composing containerizer tests.

This patch updates composing containerizer tests in order to be
consistent with the unification of `destroy()` and `wait()` return
types.

Review: https://reviews.apache.org/r/66671/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bf901dad
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bf901dad
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bf901dad

Branch: refs/heads/master
Commit: bf901dad031e4f9d55832fa9eb38455ba9639809
Parents: a4492f7
Author: Andrei Budnik 
Authored: Fri May 25 09:08:08 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:08 2018 +0800

--
 .../composing_containerizer_tests.cpp   | 40 
 1 file changed, 16 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/bf901dad/src/tests/containerizer/composing_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/composing_containerizer_tests.cpp 
b/src/tests/containerizer/composing_containerizer_tests.cpp
index 6964ac2..4236bd4 100644
--- a/src/tests/containerizer/composing_containerizer_tests.cpp
+++ b/src/tests/containerizer/composing_containerizer_tests.cpp
@@ -63,8 +63,7 @@ class ComposingContainerizerTest : public MesosTest {};
 // underlying containerizer's destroy (because it's not sure
 // if the containerizer can handle the type of container being
 // launched). If the launch is not supported by the 1st containerizer,
-// the composing containerizer should stop the launch loop and
-// set the value of destroy future to true.
+// the composing containerizer should stop the launch.
 TEST_F(ComposingContainerizerTest, DestroyDuringUnsupportedLaunchLoop)
 {
   vector containerizers;
@@ -80,7 +79,6 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringUnsupportedLaunchLoop)
   containerId.set_value("container");
   TaskInfo taskInfo;
   ExecutorInfo executorInfo;
-  SlaveID slaveId;
   map environment;
 
   Promise launchPromise;
@@ -100,8 +98,6 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringUnsupportedLaunchLoop)
   environment,
   None());
 
-  Resources resources = Resources::parse("cpus:1;mem:256").get();
-
   EXPECT_TRUE(launched.isPending());
 
   Future destroyed =
@@ -118,13 +114,13 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringUnsupportedLaunchLoop)
   launchPromise.set(Containerizer::LaunchResult::NOT_SUPPORTED);
   destroyPromise.set(Option::none());
 
-  // `launched` should be a failure and `destroyed` should be true
-  // because the launch was stopped from being tried on the 2nd
-  // containerizer because of the destroy.
+  // `launched` should be a failure and `destroyed` should be `None`
+  // because there was no container in `RUNNING` or `LAUNCHING` state
+  // at the moment `destroy()` was called.
   AWAIT_FAILED(launched);
 
   AWAIT_READY(destroyed);
-  EXPECT_SOME(destroyed.get());
+  EXPECT_NONE(destroyed.get());
 }
 
 
@@ -132,8 +128,9 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringUnsupportedLaunchLoop)
 // launch loop. The composing containerizer still calls the
 // underlying containerizer's destroy (because it's not sure
 // if the containerizer can handle the type of container being
-// launched). If the launch is successful the destroy future
-// value depends on the containerizer's destroy.
+// launched). If the launch is successful, the composing
+// containerizer's destroy future value depends on the underlying
+// containerizer's destroy.
 TEST_F(ComposingContainerizerTest, DestroyDuringSupportedLaunchLoop)
 {
   vector containerizers;
@@ -149,7 +146,6 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringSupportedLaunchLoop)
   containerId.set_value("container");
   TaskInfo taskInfo;
   ExecutorInfo executorInfo;
-  SlaveID slaveId;
   map environment;
 
   Promise launchPromise;
@@ -169,8 +165,6 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringSupportedLaunchLoop)
   environment,
   None());
 
-  Resources resources = Resources::parse("cpus:1;mem:256").get();
-
   EXPECT_TRUE(launched.isPending());
 
   Future destroyed =
@@ -185,14 +179,14 @@ TEST_F(ComposingContainerizerTest, 
DestroyDuringSupportedLaunchLoop)
   AWAIT_READY(destroy);
 
   launchPromise.set(Containerizer::LaunchResult::SUCCESS);
-  destroyPromise.set(Option::none());
+  destroyPromise.set(Option(ContainerTermination()));
 
-  // `launched` should return true and `destroyed` should return false
-  // because the launch succeeded and `destroyPromise` was set to false.
+  // `launched` should return `SUCCESS` and `destroyed` should return `Some`,
+  // because both 

[2/8] mesos git commit: Ensured that `wait()` and `destroy()` return the same result.

[qianzhang]

Ensured that `wait()` and `destroy()` return the same result.

We need to return the same `ContainerTermination` result for both
`wait()` and `destroy()` for a terminated container. This patch
ensures that for a terminated nested container `destroy()` returns
the same result as for `wait()`.

Review: https://reviews.apache.org/r/66670/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a4492f77
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a4492f77
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a4492f77

Branch: refs/heads/master
Commit: a4492f7767ef056bc4ea11f17d61521d547f38aa
Parents: 896c593
Author: Andrei Budnik 
Authored: Fri May 25 09:08:02 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:02 2018 +0800

--
 src/slave/containerizer/composing.cpp   | 5 -
 src/slave/containerizer/mesos/containerizer.cpp | 5 -
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a4492f77/src/slave/containerizer/composing.cpp
--
diff --git a/src/slave/containerizer/composing.cpp 
b/src/slave/containerizer/composing.cpp
index 3c03d66..7144ba7 100644
--- a/src/slave/containerizer/composing.cpp
+++ b/src/slave/containerizer/composing.cpp
@@ -605,7 +605,10 @@ Future 
ComposingContainerizerProcess::destroy(
 // Move this logging into the callers.
 LOG(WARNING) << "Attempted to destroy unknown container " << containerId;
 
-return None();
+// A nested container might have already been terminated, therefore
+// `containers_` might not contain it, but its exit status might have
+// been checkpointed.
+return wait(containerId);
   }
 
   Container* container = containers_.at(containerId);

http://git-wip-us.apache.org/repos/asf/mesos/blob/a4492f77/src/slave/containerizer/mesos/containerizer.cpp
--
diff --git a/src/slave/containerizer/mesos/containerizer.cpp 
b/src/slave/containerizer/mesos/containerizer.cpp
index 51edb58..6620e94 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -2383,7 +2383,10 @@ Future 
MesosContainerizerProcess::destroy(
 // Move this logging into the callers.
 LOG(WARNING) << "Attempted to destroy unknown container " << containerId;
 
-return None();
+// A nested container might have already been terminated, therefore
+// `containers_` might not contain it, but its exit status might have
+// been checkpointed.
+return wait(containerId);
   }
 
   const Owned& container = containers_.at(containerId);

[7/8] mesos git commit: Restored `WaitAfterDestroy` test for a nested container.

[qianzhang]

Restored `WaitAfterDestroy` test for a nested container.

This test was removed in fd4b9af147, but it's important to check that
after termination of a nested container, its termination status is
available. This property is used in default executor.

Review: https://reviews.apache.org/r/65505/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a1ce9ad6
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a1ce9ad6
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a1ce9ad6

Branch: refs/heads/master
Commit: a1ce9ad6227d7871d8409fcfee519a63dc812a0c
Parents: b549c9c
Author: Andrei Budnik 
Authored: Fri May 25 09:08:37 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:37 2018 +0800

--
 .../nested_mesos_containerizer_tests.cpp| 87 
 1 file changed, 87 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a1ce9ad6/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
--
diff --git a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp 
b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
index 8664f59..3481726 100644
--- a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
+++ b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
@@ -2579,6 +2579,93 @@ TEST_F(NestedMesosContainerizerTest,
   ASSERT_FALSE(containers->contains(containerId));
 }
 
+
+// This test verifies that termination status of a nested container is
+// available until its parent is terminated.
+TEST_F(NestedMesosContainerizerTest, ROOT_CGROUPS_WaitAfterDestroy)
+{
+  slave::Flags flags = CreateSlaveFlags();
+  flags.launcher = "linux";
+  flags.isolation = "cgroups/cpu,filesystem/linux,namespaces/pid";
+
+  Fetcher fetcher(flags);
+
+  Try create = MesosContainerizer::create(
+  flags,
+  true,
+  );
+
+  ASSERT_SOME(create);
+
+  Owned containerizer(create.get());
+
+  SlaveID slaveId = SlaveID();
+
+  // Launch a top-level container.
+  ContainerID containerId;
+  containerId.set_value(id::UUID::random().toString());
+
+  Try directory = environment->mkdtemp();
+  ASSERT_SOME(directory);
+
+  Future launch = containerizer->launch(
+containerId,
+createContainerConfig(
+None(),
+createExecutorInfo("executor", "sleep 1000", "cpus:1"),
+directory.get()),
+map(),
+None());
+
+  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);
+
+  // Launch a nested container.
+  ContainerID nestedContainerId;
+  nestedContainerId.mutable_parent()->CopyFrom(containerId);
+  nestedContainerId.set_value(id::UUID::random().toString());
+
+  launch = containerizer->launch(
+  nestedContainerId,
+  createContainerConfig(createCommandInfo("exit 42")),
+  map(),
+  None());
+
+  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);
+
+  // Wait once for a nested container completion, then wait again
+  // to make sure that its termination status is still available.
+  Future nestedWait =
+containerizer->wait(nestedContainerId);
+
+  AWAIT_READY(nestedWait);
+  ASSERT_SOME(nestedWait.get());
+  ASSERT_TRUE(nestedWait.get()->has_status());
+  EXPECT_WEXITSTATUS_EQ(42, nestedWait.get()->status());
+
+  nestedWait = containerizer->wait(nestedContainerId);
+
+  AWAIT_READY(nestedWait);
+  ASSERT_SOME(nestedWait.get());
+  ASSERT_TRUE(nestedWait.get()->has_status());
+  EXPECT_WEXITSTATUS_EQ(42, nestedWait.get()->status());
+
+  // Destroy the top-level container.
+  Future termination =
+containerizer->destroy(containerId);
+
+  AWAIT_READY(termination);
+  ASSERT_SOME(termination.get());
+  ASSERT_TRUE(termination.get()->has_status());
+  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
+
+  // Wait on nested container again.
+  nestedWait = containerizer->wait(nestedContainerId);
+
+  AWAIT_READY(nestedWait);
+  ASSERT_NONE(nestedWait.get());
+}
+
+
 // This test verifies that agent environment variables are not leaked
 // to the nested container, and the environment variables specified in
 // the command for the nested container will be honored.

[6/8] mesos git commit: Updated comments related to `wait`, `destroy` containerizer methods.

[qianzhang]

Updated comments related to `wait`, `destroy` containerizer methods.

This patch updates description of `wait()` and `destroy()` methods
of the containerizer API.

Review: https://reviews.apache.org/r/67130/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b549c9ca
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b549c9ca
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b549c9ca

Branch: refs/heads/master
Commit: b549c9cac15100cc0497aaa41073be16f678e14a
Parents: a8eac5b
Author: Andrei Budnik 
Authored: Fri May 25 09:08:28 2018 +0800
Committer: Qian Zhang 
Committed: Fri May 25 09:08:28 2018 +0800

--
 src/slave/containerizer/containerizer.hpp | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/b549c9ca/src/slave/containerizer/containerizer.hpp
--
diff --git a/src/slave/containerizer/containerizer.hpp 
b/src/slave/containerizer/containerizer.hpp
index 5817b1d..675dfad 100644
--- a/src/slave/containerizer/containerizer.hpp
+++ b/src/slave/containerizer/containerizer.hpp
@@ -128,18 +128,18 @@ public:
 
   // Wait on the 'ContainerTermination'. If the executor terminates,
   // the containerizer should also destroy the containerized context.
-  // Returns None if the container cannot be found.
   // The future may be failed if an error occurs during termination of
   // the executor or destruction of the container.
+  //
+  // Returns `None` if the container cannot be found.
+  // NOTE: For terminated nested containers, whose parent container is
+  // still running, the checkpointed `ContainerTermination` must be returned.
   virtual process::Future wait(
   const ContainerID& containerId) = 0;
 
-  // Destroy a running container, killing all processes and releasing
-  // all resources. Returns None when the container cannot be found,
-  // or a failure if something went wrong.
-  //
-  // NOTE: You cannot wait() on containers that have been destroyed,
-  // so you should always call wait() before destroy().
+  // Destroy a starting or running container, killing all processes and
+  // releasing all resources. Returns the same result as `wait()` method,
+  // therefore calling `wait()` right before `destroy()` is not required.
   virtual process::Future destroy(
   const ContainerID& containerId) = 0;
 

mesos git commit: Made CNI isolator recovery waits until unknown orphan cleanup is done.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 0da49ee9b -> 77aba9a27


Made CNI isolator recovery waits until unknown orphan cleanup is done.

Review: https://reviews.apache.org/r/67769


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/77aba9a2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/77aba9a2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/77aba9a2

Branch: refs/heads/master
Commit: 77aba9a27ab8aac30f54cea6e465269e9c083fb2
Parents: 0da49ee
Author: Qian Zhang 
Authored: Thu Jun 28 16:11:31 2018 +0800
Committer: Qian Zhang 
Committed: Mon Jul 2 07:56:19 2018 +0800

--
 .../mesos/isolators/network/cni/cni.cpp | 24 ++--
 1 file changed, 22 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/77aba9a2/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
--
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp 
b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index dfa26c6..f46c962 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -403,6 +403,9 @@ Future NetworkCniIsolatorProcess::recover(
 rootDir.get() + "': " + entries.error());
   }
 
+  vector unknownOrphans;
+  vector> cleanups;
+
   foreach (const string& entry, entries.get()) {
 ContainerID containerId =
   protobuf::parseContainerId(Path(entry).basename());
@@ -432,12 +435,29 @@ Future NetworkCniIsolatorProcess::recover(
   if (!orphans.contains(containerId)) {
 LOG(INFO) << "Removing unknown orphaned container " << containerId;
 
-cleanup(containerId);
+unknownOrphans.push_back(containerId);
+cleanups.push_back(cleanup(containerId));
   }
 }
   }
 
-  return Nothing();
+  return await(cleanups)
+.then([unknownOrphans](const vector>& cleanups) {
+  CHECK_EQ(cleanups.size(), unknownOrphans.size());
+
+  int i = 0;
+  foreach (const Future& cleanup, cleanups) {
+if (!cleanup.isReady()) {
+  LOG(ERROR) << "Failed to cleanup unknown orphaned container "
+ << unknownOrphans.at(i) << ": "
+ << (cleanup.isFailed() ? cleanup.failure() : "discarded");
+}
+
+i++;
+  }
+
+  return Nothing();
+});
 }
 
 

[4/4] mesos git commit: Detached the virtual paths regardless of the result of gc.

[qianzhang]

Detached the virtual paths regardless of the result of gc.

Previously we only detach the following paths when the gc for the
executor's sandbox succeeds.
  1. /agent_workdir/frameworks/FID/executors/EID/runs/CID
  2. /agent_workdir/frameworks/FID/executors/EID/runs/latest
  3. /frameworks/FID/executors/EID/runs/latest

But the problem is, such gc may not always succeed, e.g., it may fail
due to the parent directory of the executor's sandbox already gc'ed.

Now in this patch, we will detach those paths regardless of the result
of gc.

Review: https://reviews.apache.org/r/65156


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5225a49c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5225a49c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5225a49c

Branch: refs/heads/master
Commit: 5225a49c495bc7e3362bcee2d460d8c99111c7f4
Parents: 9e2f9a2
Author: Qian Zhang 
Authored: Sun Jan 14 22:02:33 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 10:04:30 2018 +0800

--
 src/slave/slave.cpp | 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/5225a49c/src/slave/slave.cpp
--
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 5ad6410..155d9f0 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -5873,7 +5873,7 @@ void Slave::removeExecutor(Framework* framework, 
Executor* executor)
 
   os::utime(path); // Update the modification time.
   garbageCollect(path)
-.then(defer(self(), ::detachFile, path));
+.onAny(defer(self(), ::detachFile, path));
 
   // Schedule the top level executor work directory, only if the
   // framework doesn't have any 'pending' tasks for this executor.
@@ -5897,10 +5897,9 @@ void Slave::removeExecutor(Framework* framework, 
Executor* executor)
 
 os::utime(path); // Update the modification time.
 garbageCollect(path)
-  .then(defer(self(), [=]() {
+  .onAny(defer(self(), [=](const Future& future) {
 detachFile(latestPath);
 detachFile(virtualLatestPath);
-return Nothing();
   }));
   }
 
@@ -8615,7 +8614,7 @@ void Framework::recoverExecutor(
 slave->flags.work_dir, slave->info.id(), id(), state.id, runId);
 
 slave->garbageCollect(path)
-   .then(defer(slave, ::detachFile, path));
+   .onAny(defer(slave, ::detachFile, path));
 
 // GC the executor run's meta directory.
 slave->garbageCollect(paths::getExecutorRunPath(
@@ -8624,7 +8623,7 @@ void Framework::recoverExecutor(
 // GC the top level executor work directory.
 slave->garbageCollect(paths::getExecutorPath(
 slave->flags.work_dir, slave->info.id(), id(), state.id))
-.then(defer(slave, ::detachFile, latestPath));
+.onAny(defer(slave, ::detachFile, latestPath));
 
 // GC the top level executor meta directory.
 slave->garbageCollect(paths::getExecutorPath(

[2/4] mesos git commit: Made task's volume directory visible in the /files endpoints.

[qianzhang]

 // GC the executor run's meta directory.
 slave->garbageCollect(paths::getExecutorRunPath(
@@ -8917,6 +8950,10 @@ Task* Executor::addLaunchedTask(const TaskInfo& task)
 
   launchedTasks[task.task_id()] = t;
 
+  if (info.has_type() && info.type() == ExecutorInfo::DEFAULT) {
+attachTaskVolumeDirectory(*t);
+  }
+
   return t;
 }
 
@@ -8928,6 +8965,17 @@ void Executor::completeTask(const TaskID& taskId)
   CHECK(terminatedTasks.contains(taskId))
 << "Failed to find terminated task " << taskId;
 
+  // If `completedTasks` is full and this is a default executor, we need
+  // to detach the volume directory for the first task in `completedTasks`
+  // before pushing a task into it, otherwise, we will never have chance
+  // to do the detach for that task which would be a leak.
+  if (info.has_type() &&
+  info.type() == ExecutorInfo::DEFAULT &&
+  completedTasks.full()) {
+const shared_ptr& firstTask = completedTasks.front();
+detachTaskVolumeDirectory(*firstTask);
+  }
+
   Task* task = terminatedTasks[taskId];
   completedTasks.push_back(shared_ptr(task));
   terminatedTasks.erase(taskId);
@@ -8998,6 +9046,10 @@ void Executor::recoverTask(const TaskState& state, bool 
recheckpointTask)
 
   launchedTasks[state.id] = task;
 
+  if (info.has_type() && info.type() == ExecutorInfo::DEFAULT) {
+attachTaskVolumeDirectory(*task);
+  }
+
   // Read updates to get the latest state of the task.
   foreach (const StatusUpdate& update, state.updates) {
 Try updated = updateTaskState(update.status());
@@ -9103,6 +9155,73 @@ bool Executor::incompleteTasks()
 }
 
 
+void Executor::attachTaskVolumeDirectory(const Task& task)
+{
+  CHECK(info.has_type() && info.type() == ExecutorInfo::DEFAULT);
+
+  foreach (const Resource& resource, task.resources()) {
+// Ignore if there are no disk resources or if the
+// disk resources did not specify a volume mapping.
+if (!resource.has_disk() || !resource.disk().has_volume()) {
+  continue;
+}
+
+const Volume& volume = resource.disk().volume();
+
+const string executorVolumePath =
+  path::join(directory, volume.container_path());
+
+const string taskPath = paths::getTaskPath(
+slave->flags.work_dir,
+slave->info.id(),
+frameworkId,
+id,
+containerId,
+task.task_id());
+
+const string taskVolumePath =
+  path::join(taskPath, volume.container_path());
+
+slave->files->attach(executorVolumePath, taskVolumePath)
+  .onAny(defer(
+  slave,
+  ::fileAttached,
+  lambda::_1,
+  executorVolumePath,
+  taskVolumePath));
+  }
+}
+
+
+void Executor::detachTaskVolumeDirectory(const Task& task)
+{
+  CHECK(info.has_type() && info.type() == ExecutorInfo::DEFAULT);
+
+  foreach (const Resource& resource, task.resources()) {
+// Ignore if there are no disk resources or if the
+// disk resources did not specify a volume mapping.
+if (!resource.has_disk() || !resource.disk().has_volume()) {
+  continue;
+}
+
+const Volume& volume = resource.disk().volume();
+
+const string taskPath = paths::getTaskPath(
+slave->flags.work_dir,
+slave->info.id(),
+frameworkId,
+id,
+containerId,
+task.task_id());
+
+const string taskVolumePath =
+  path::join(taskPath, volume.container_path());
+
+slave->files->detach(taskVolumePath);
+  }
+}
+
+
 bool Executor::isGeneratedForCommandTask() const
 {
   return isGeneratedForCommandTask_;

http://git-wip-us.apache.org/repos/asf/mesos/blob/e126254e/src/slave/slave.hpp
--
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index ef0eae2..a07f046 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -833,6 +833,29 @@ public:
   // Returns true if there are any queued/launched/terminated tasks.
   bool incompleteTasks();
 
+  // TODO(qianzhang): This is a workaround to make the default executor
+  // task's volume directory visible in MESOS UI. In MESOS-7225, we made
+  // sure a task can access any volumes specified in its disk resources
+  // from its sandbox by introducing a workaround to the default executor,
+  // i.e., adding a `SANDBOX_PATH` volume with type `PARENT` to the
+  // corresponding nested container. This volume gets translated into a
+  // bind mount in the nested container's mount namespace, which is is not
+  // visible in Mesos UI because it operates in the host namespace. See
+  // Mesos-8279 for details.
+  //
+  // To make the task's volume directory visible in Mesos UI, here we
+  // attach the executor's volume directory to it, so when users browse
+  // task's volume directory in Mesos UI, what they actually browse is the
+  // executor's volume directory. Note when calling 

[1/4] mesos git commit: Detached `virtualLatestPath` when recovering the executor.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 9e2f9a240 -> 2c5da1b66


Detached `virtualLatestPath` when recovering the executor.

Previously we miss to detach `/frameworks/FID/executors/EID/runs/latest`
when we find the latest run of the executor was completed in the method
`Framework::recoverExecutor`, that is a leak.

Review: https://reviews.apache.org/r/65167


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2c5da1b6
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2c5da1b6
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2c5da1b6

Branch: refs/heads/master
Commit: 2c5da1b668de91e33831caafb18a3b4d71b26c69
Parents: 9585a21
Author: Qian Zhang 
Authored: Mon Jan 15 16:40:00 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 10:04:30 2018 +0800

--
 src/slave/slave.cpp | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/2c5da1b6/src/slave/slave.cpp
--
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 45e6f9b..1672c06 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -8656,7 +8656,10 @@ void Framework::recoverExecutor(
 // GC the top level executor work directory.
 slave->garbageCollect(paths::getExecutorPath(
 slave->flags.work_dir, slave->info.id(), id(), state.id))
-.onAny(defer(slave, ::detachFile, latestPath));
+.onAny(defer(slave->self(), [=](const Future& future) {
+  slave->detachFile(latestPath);
+  slave->detachFile(virtualLatestPath);
+}));
 
 // GC the top level executor meta directory.
 slave->garbageCollect(paths::getExecutorPath(

[3/4] mesos git commit: Updated `ROOT_TaskSandboxPersistentVolume` to check `/files` endpoint.

[qianzhang]

Updated `ROOT_TaskSandboxPersistentVolume` to check `/files` endpoint.

Review: https://reviews.apache.org/r/65070


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9585a217
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9585a217
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9585a217

Branch: refs/heads/master
Commit: 9585a2173970589f91858301c66479827c1370a9
Parents: e126254
Author: Qian Zhang 
Authored: Wed Jan 10 21:26:45 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 10:04:30 2018 +0800

--
 src/tests/default_executor_tests.cpp | 82 +++
 1 file changed, 73 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/9585a217/src/tests/default_executor_tests.cpp
--
diff --git a/src/tests/default_executor_tests.cpp 
b/src/tests/default_executor_tests.cpp
index d2cf5fd..065eae6 100644
--- a/src/tests/default_executor_tests.cpp
+++ b/src/tests/default_executor_tests.cpp
@@ -2413,7 +2413,8 @@ TEST_P_TEMP_DISABLED_ON_WINDOWS(
 
 
 // This test verifies that the default executor mounts the persistent volume
-// in the task container when it is set on a task in the task group.
+// in the task container when it is set on a task in the task group, and the
+// task's volume directory can be accessed from the `/files` endpoint.
 TEST_P_TEMP_DISABLED_ON_WINDOWS(
 PersistentVolumeDefaultExecutor, ROOT_TaskSandboxPersistentVolume)
 {
@@ -2489,7 +2490,7 @@ TEST_P_TEMP_DISABLED_ON_WINDOWS(
   v1::TaskInfo taskInfo = v1::createTask(
   offer.agent_id(),
   reserved.apply(v1::CREATE(volume)).get(),
-  "echo abc > task_volume_path/file");
+  "echo abc > task_volume_path/file && sleep 1000");
 
   v1::Offer::Operation reserve = v1::RESERVE(reserved);
   v1::Offer::Operation create = v1::CREATE(volume);
@@ -2499,7 +2500,6 @@ TEST_P_TEMP_DISABLED_ON_WINDOWS(
 
   Future updateStarting;
   Future updateRunning;
-  Future updateFinished;
   EXPECT_CALL(*scheduler, update(_, _))
 .WillOnce(DoAll(FutureArg<1>(),
 v1::scheduler::SendAcknowledge(
@@ -2508,8 +2508,7 @@ TEST_P_TEMP_DISABLED_ON_WINDOWS(
 .WillOnce(DoAll(FutureArg<1>(),
 v1::scheduler::SendAcknowledge(
 frameworkId,
-offer.agent_id(
-.WillOnce(FutureArg<1>());
+offer.agent_id(;
 
   mesos.send(v1::createCallAccept(
   frameworkId,
@@ -2524,18 +2523,83 @@ TEST_P_TEMP_DISABLED_ON_WINDOWS(
   ASSERT_EQ(TASK_RUNNING, updateRunning->status().state());
   ASSERT_EQ(taskInfo.task_id(), updateRunning->status().task_id());
 
-  AWAIT_READY(updateFinished);
-  ASSERT_EQ(TASK_FINISHED, updateFinished->status().state());
-  ASSERT_EQ(taskInfo.task_id(), updateFinished->status().task_id());
-
   string volumePath = slave::paths::getPersistentVolumePath(
   flags.work_dir,
   devolve(volume));
 
   string filePath = path::join(volumePath, "file");
 
+  // Wait up to 10 seconds for the task to write a file into the volume.
+  Duration waited = Duration::zero();
+  do {
+if (os::exists(filePath)) {
+  break;
+}
+
+os::sleep(Seconds(1));
+waited += Seconds(1);
+  } while (waited < Seconds(10));
+
   // Ensure that the task was able to write to the persistent volume.
+  EXPECT_TRUE(os::exists(filePath));
   EXPECT_SOME_EQ("abc\n", os::read(filePath));
+
+  v1::ContainerStatus status = updateRunning->status().container_status();
+
+  ASSERT_TRUE(status.has_container_id());
+  EXPECT_TRUE(status.container_id().has_parent());
+
+  v1::ContainerID executorContainerId = status.container_id().parent();
+
+  string taskPath = slave::paths::getTaskPath(
+  flags.work_dir,
+  devolve(offer.agent_id()),
+  devolve(frameworkId),
+  devolve(executorInfo.executor_id()),
+  devolve(executorContainerId),
+  devolve(taskInfo.task_id()));
+
+  string taskVolumePath =
+path::join(taskPath, volume.disk().volume().container_path());
+
+  // Ensure the task's volume directory can be accessed from
+  // the `/files` endpoint.
+  process::UPID files("files", slave.get()->pid.address);
+
+  {
+string query = string("path=") + taskVolumePath;
+Future response = process::http::get(
+files,
+"browse",
+query,
+createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+AWAIT_ASSERT_RESPONSE_STATUS_EQ(OK().status, response);
+AWAIT_ASSERT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", 
response);
+
+Try parse = JSON::parse(response->body);
+ASSERT_SOME(parse);
+EXPECT_NE(0, parse->values.size());
+  }
+
+  {
+string query =
+  string("path=") + 

[1/4] mesos git commit: Made task's volume directory visible in the /files endpoints.

[qianzhang]

xecutor->detachTaskVolumeDirectory(*task);
+  }
+}
+  }));
 
 // GC the executor run's meta directory.
 slave->garbageCollect(paths::getExecutorRunPath(
@@ -8901,6 +8934,10 @@ Task* Executor::addLaunchedTask(const TaskInfo& task)
 
   launchedTasks[task.task_id()] = t;
 
+  if (info.has_type() && info.type() == ExecutorInfo::DEFAULT) {
+attachTaskVolumeDirectory(*t);
+  }
+
   return t;
 }
 
@@ -8912,6 +8949,17 @@ void Executor::completeTask(const TaskID& taskId)
   CHECK(terminatedTasks.contains(taskId))
 << "Failed to find terminated task " << taskId;
 
+  // If `completedTasks` is full and this is a default executor, we need
+  // to detach the volume directory for the first task in `completedTasks`
+  // before pushing a task into it, otherwise, we will never have chance
+  // to do the detach for that task which would be a leak.
+  if (info.has_type() &&
+  info.type() == ExecutorInfo::DEFAULT &&
+  completedTasks.full()) {
+const shared_ptr& firstTask = completedTasks.front();
+detachTaskVolumeDirectory(*firstTask);
+  }
+
   Task* task = terminatedTasks[taskId];
   completedTasks.push_back(shared_ptr(task));
   terminatedTasks.erase(taskId);
@@ -8982,6 +9030,10 @@ void Executor::recoverTask(const TaskState& state, bool 
recheckpointTask)
 
   launchedTasks[state.id] = task;
 
+  if (info.has_type() && info.type() == ExecutorInfo::DEFAULT) {
+attachTaskVolumeDirectory(*task);
+  }
+
   // Read updates to get the latest state of the task.
   foreach (const StatusUpdate& update, state.updates) {
 Try updated = updateTaskState(update.status());
@@ -9087,6 +9139,73 @@ bool Executor::incompleteTasks()
 }
 
 
+void Executor::attachTaskVolumeDirectory(const Task& task)
+{
+  CHECK(info.has_type() && info.type() == ExecutorInfo::DEFAULT);
+
+  foreach (const Resource& resource, task.resources()) {
+// Ignore if there are no disk resources or if the
+// disk resources did not specify a volume mapping.
+if (!resource.has_disk() || !resource.disk().has_volume()) {
+  continue;
+}
+
+const Volume& volume = resource.disk().volume();
+
+const string executorVolumePath =
+  path::join(directory, volume.container_path());
+
+const string taskPath = paths::getTaskPath(
+slave->flags.work_dir,
+slave->info.id(),
+frameworkId,
+id,
+containerId,
+task.task_id());
+
+const string taskVolumePath =
+  path::join(taskPath, volume.container_path());
+
+slave->files->attach(executorVolumePath, taskVolumePath)
+  .onAny(defer(
+  slave,
+  ::fileAttached,
+  lambda::_1,
+  executorVolumePath,
+  taskVolumePath));
+  }
+}
+
+
+void Executor::detachTaskVolumeDirectory(const Task& task)
+{
+  CHECK(info.has_type() && info.type() == ExecutorInfo::DEFAULT);
+
+  foreach (const Resource& resource, task.resources()) {
+// Ignore if there are no disk resources or if the
+// disk resources did not specify a volume mapping.
+if (!resource.has_disk() || !resource.disk().has_volume()) {
+  continue;
+}
+
+const Volume& volume = resource.disk().volume();
+
+const string taskPath = paths::getTaskPath(
+slave->flags.work_dir,
+slave->info.id(),
+frameworkId,
+id,
+containerId,
+task.task_id());
+
+const string taskVolumePath =
+  path::join(taskPath, volume.container_path());
+
+slave->files->detach(taskVolumePath);
+  }
+}
+
+
 bool Executor::isGeneratedForCommandTask() const
 {
   return isGeneratedForCommandTask_;

http://git-wip-us.apache.org/repos/asf/mesos/blob/f2466a76/src/slave/slave.hpp
--
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index ef0eae2..a07f046 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -833,6 +833,29 @@ public:
   // Returns true if there are any queued/launched/terminated tasks.
   bool incompleteTasks();
 
+  // TODO(qianzhang): This is a workaround to make the default executor
+  // task's volume directory visible in MESOS UI. In MESOS-7225, we made
+  // sure a task can access any volumes specified in its disk resources
+  // from its sandbox by introducing a workaround to the default executor,
+  // i.e., adding a `SANDBOX_PATH` volume with type `PARENT` to the
+  // corresponding nested container. This volume gets translated into a
+  // bind mount in the nested container's mount namespace, which is is not
+  // visible in Mesos UI because it operates in the host namespace. See
+  // Mesos-8279 for details.
+  //
+  // To make the task's volume directory visible in Mesos UI, here we
+  // attach the executor's volume directory to it, so when users browse
+  // task's volume directory in Mesos UI,

[3/4] mesos git commit: Detached `virtualLatestPath` when recovering the executor.

[qianzhang]

Detached `virtualLatestPath` when recovering the executor.

Previously we miss to detach `/frameworks/FID/executors/EID/runs/latest`
when we find the latest run of the executor was completed in the method
`Framework::recoverExecutor`, that is a leak.

Review: https://reviews.apache.org/r/65167


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f25c9b3f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f25c9b3f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f25c9b3f

Branch: refs/heads/1.5.x
Commit: f25c9b3f77df9f4edc250e18be7392391d19d024
Parents: f2466a7
Author: Qian Zhang 
Authored: Mon Jan 15 16:40:00 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 12:08:49 2018 +0800

--
 src/slave/slave.cpp | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/f25c9b3f/src/slave/slave.cpp
--
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 4a37041..956f79d 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -8640,7 +8640,10 @@ void Framework::recoverExecutor(
 // GC the top level executor work directory.
 slave->garbageCollect(paths::getExecutorPath(
 slave->flags.work_dir, slave->info.id(), id(), state.id))
-.onAny(defer(slave, ::detachFile, latestPath));
+.onAny(defer(slave->self(), [=](const Future& future) {
+  slave->detachFile(latestPath);
+  slave->detachFile(virtualLatestPath);
+}));
 
 // GC the top level executor meta directory.
 slave->garbageCollect(paths::getExecutorPath(

[4/4] mesos git commit: Added MESOS-8279, MESOS-8444 and MESOS-8446 to the 1.5.0 CHANGELOG.

[qianzhang]

Added MESOS-8279, MESOS-8444 and MESOS-8446 to the 1.5.0 CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1d9b5de7
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1d9b5de7
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1d9b5de7

Branch: refs/heads/1.5.x
Commit: 1d9b5de7d6ad796f5b0ea8e9c4e5541dda6984a4
Parents: f25c9b3
Author: Qian Zhang 
Authored: Wed Jan 17 12:06:32 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 12:08:49 2018 +0800

--
 CHANGELOG | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/1d9b5de7/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index fa0d03a..47e1150 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -61,6 +61,14 @@ Feature Graduations:
 All Experimental Features:
 
 
+All Resolved Issues:
+
+** Bug
+  * [MESOS-8279] - Persistent volumes are not visible in Mesos UI using 
default executor on Linux.
+  * [MESOS-8444] - GC failure causes agent miss to detach virtual paths for 
the executor's sandbox.
+  * [MESOS-8446] - Agent miss to detach `virtualLatestPath` for the executor's 
sandbox during recovery.
+
+
 Release Notes - Mesos - Version 1.4.2 (WIP)
 ---
 * This is a bug fix release.

[2/4] mesos git commit: Detached the virtual paths regardless of the result of gc.

[qianzhang]

Detached the virtual paths regardless of the result of gc.

Previously we only detach the following paths when the gc for the
executor's sandbox succeeds.
  1. /agent_workdir/frameworks/FID/executors/EID/runs/CID
  2. /agent_workdir/frameworks/FID/executors/EID/runs/latest
  3. /frameworks/FID/executors/EID/runs/latest

But the problem is, such gc may not always succeed, e.g., it may fail
due to the parent directory of the executor's sandbox already gc'ed.

Now in this patch, we will detach those paths regardless of the result
of gc.

Review: https://reviews.apache.org/r/65156


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a198637d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a198637d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a198637d

Branch: refs/heads/1.5.x
Commit: a198637d3805765d11a2474b3a66097592c9b065
Parents: af64bcb
Author: Qian Zhang 
Authored: Sun Jan 14 22:02:33 2018 +0800
Committer: Qian Zhang 
Committed: Wed Jan 17 11:55:36 2018 +0800

--
 src/slave/slave.cpp | 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a198637d/src/slave/slave.cpp
--
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 3e7281f..aefea9c 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -5871,7 +5871,7 @@ void Slave::removeExecutor(Framework* framework, 
Executor* executor)
 
   os::utime(path); // Update the modification time.
   garbageCollect(path)
-.then(defer(self(), ::detachFile, path));
+.onAny(defer(self(), ::detachFile, path));
 
   // Schedule the top level executor work directory, only if the
   // framework doesn't have any 'pending' tasks for this executor.
@@ -5895,10 +5895,9 @@ void Slave::removeExecutor(Framework* framework, 
Executor* executor)
 
 os::utime(path); // Update the modification time.
 garbageCollect(path)
-  .then(defer(self(), [=]() {
+  .onAny(defer(self(), [=](const Future& future) {
 detachFile(latestPath);
 detachFile(virtualLatestPath);
-return Nothing();
   }));
   }
 
@@ -8599,7 +8598,7 @@ void Framework::recoverExecutor(
 slave->flags.work_dir, slave->info.id(), id(), state.id, runId);
 
 slave->garbageCollect(path)
-   .then(defer(slave, ::detachFile, path));
+   .onAny(defer(slave, ::detachFile, path));
 
 // GC the executor run's meta directory.
 slave->garbageCollect(paths::getExecutorRunPath(
@@ -8608,7 +8607,7 @@ void Framework::recoverExecutor(
 // GC the top level executor work directory.
 slave->garbageCollect(paths::getExecutorPath(
 slave->flags.work_dir, slave->info.id(), id(), state.id))
-.then(defer(slave, ::detachFile, latestPath));
+.onAny(defer(slave, ::detachFile, latestPath));
 
 // GC the top level executor meta directory.
 slave->garbageCollect(paths::getExecutorPath(

mesos git commit: Updated Qian Zhang's entry in the contributors list.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master bc9fe6480 -> a3e1f9b82


Updated Qian Zhang's entry in the contributors list.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a3e1f9b8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a3e1f9b8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a3e1f9b8

Branch: refs/heads/master
Commit: a3e1f9b82ec966017ffe1f9767b93965a7af9670
Parents: bc9fe64
Author: Qian Zhang <zhq527...@gmail.com>
Authored: Thu Jan 18 11:05:18 2018 +0800
Committer: Qian Zhang <zhq527...@gmail.com>
Committed: Thu Jan 18 11:05:18 2018 +0800

--
 docs/contributors.yaml | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/a3e1f9b8/docs/contributors.yaml
--
diff --git a/docs/contributors.yaml b/docs/contributors.yaml
index 8bfeb89..8fc6d2b 100644
--- a/docs/contributors.yaml
+++ b/docs/contributors.yaml
@@ -618,9 +618,12 @@
 
 - name: Qian Zhang
   affiliations:
-- {organization: IBM}
+- {organization: Mesosphere, start_date: 2017-10-01}
+- {organization: IBM, end_date: 2017-09-30}
   emails:
-- zhang...@cn.ibm.com
+- qianzh...@apache.org
+- zhq527...@gmail.com
+- qzh...@mesosphere.com
   jira_user: qianzhang
   reviewboard_user: qianzhang
 

mesos git commit: Updated the way to output error messages in `NetworkCniIsolatorSetup`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 635c33431 -> 2cdbec02e


Updated the way to output error messages in `NetworkCniIsolatorSetup`.

Review: https://reviews.apache.org/r/65306


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2cdbec02
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2cdbec02
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2cdbec02

Branch: refs/heads/master
Commit: 2cdbec02e37c794627204f0e1fadf09e5325507d
Parents: 635c334
Author: Qian Zhang 
Authored: Tue Jan 23 15:54:58 2018 +0800
Committer: Qian Zhang 
Committed: Thu Jan 25 08:53:56 2018 +0800

--
 .../mesos/isolators/network/cni/cni.cpp   | 18 ++
 1 file changed, 14 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/2cdbec02/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
--
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp 
b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index af1d477..697f998 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -36,6 +36,7 @@
 #include 
 
 #include "common/protobuf_utils.hpp"
+#include "common/status_utils.hpp"
 
 #include "linux/fs.hpp"
 #include "linux/ns.hpp"
@@ -2076,10 +2077,19 @@ int NetworkCniIsolatorSetup::execute()
   return EXIT_FAILURE;
 }
 
-if (os::spawn("ifconfig", {"ifconfig", "lo", "up"}) != 0) {
-  cerr << "Failed to bring up the loopback interface in the new "
-   << "network namespace of pid " << flags.pid.get()
-   << ": " << os::strerror(errno) << endl;
+int status = os::spawn("ifconfig", {"ifconfig", "lo", "up"});
+
+const string message =
+  "Failed to bring up the loopback interface in the new "
+  "network namespace of pid " + stringify(flags.pid.get());
+
+if (status == -1) {
+  cerr << message << ": " << "os::spawn failed" << endl;
+  return EXIT_FAILURE;
+}
+
+if (!WSUCCEEDED(status)) {
+  cerr << message << ": 'ifconfig lo up' " << WSTRINGIFY(status) << endl;
   return EXIT_FAILURE;
 }
   }

mesos git commit: Fixed a minor issue in the log message of `network/cni` isolator.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 2bb7189e9 -> 7f148e00a


Fixed a minor issue in the log message of `network/cni` isolator.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7f148e00
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7f148e00
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7f148e00

Branch: refs/heads/master
Commit: 7f148e00a5e7713d752bb3f9e50132612f8357bf
Parents: 2bb7189
Author: Qian Zhang 
Authored: Tue Jan 30 15:29:04 2018 +0800
Committer: Qian Zhang 
Committed: Tue Jan 30 15:29:04 2018 +0800

--
 src/slave/containerizer/mesos/isolators/network/cni/cni.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/7f148e00/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
--
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp 
b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index 697f998..8966569 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -2084,7 +2084,8 @@ int NetworkCniIsolatorSetup::execute()
   "network namespace of pid " + stringify(flags.pid.get());
 
 if (status == -1) {
-  cerr << message << ": " << "os::spawn failed" << endl;
+  cerr << message << ": " << "os::spawn failed: "
+   << os::strerror(errno) << endl;
   return EXIT_FAILURE;
 }
 

mesos git commit: Fixed a typo in `Slave::recover`.

[qianzhang]

Repository: mesos
Updated Branches:
  refs/heads/master 15fc434e4 -> 25d269605


Fixed a typo in `Slave::recover`.


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/25d26960
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/25d26960
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/25d26960

Branch: refs/heads/master
Commit: 25d269605d52fe55910302fefb558ce39e813974
Parents: 15fc434
Author: Qian Zhang 
Authored: Sun Jan 28 20:59:07 2018 +0800
Committer: Qian Zhang 
Committed: Sun Jan 28 20:59:07 2018 +0800

--
 src/slave/slave.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/mesos/blob/25d26960/src/slave/slave.cpp
--
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 43c7955..a6a5c93 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -6626,7 +6626,7 @@ Future Slave::recover(const Try& 
state)
 } else if (state->rebooted) {
   // Prior to Mesos 1.4 we directly bypass the state recovery and
   // start as a new agent upon reboot (introduced in MESOS-844).
-  // This unncessarily discards the existing agent ID (MESOS-6223).
+  // This unnecessarily discards the existing agent ID (MESOS-6223).
   // Starting in Mesos 1.4 we'll attempt to recover the slave state
   // even after reboot but in case of an incompatible slave info change
   // we'll fall back to recovering as a new agent (existing behavior).