[mesos] 01/08: Added 'prettyjws' option to docker manifest V2 Schema1 accept header.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 10d57420ccd27b13a02e48a475e7c77d4f4ed100
Author: Gilbert Song
AuthorDate: Sat Mar 23 20:54:08 2019 -0700
Added 'prettyjws' option to docker manifest V2 Schema1 accept header.
Review: https://reviews.apache.org/r/70287
---
src/uri/fetchers/docker.cpp | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/uri/fetchers/docker.cpp b/src/uri/fetchers/docker.cpp
index a87d7f0..ffb5194 100644
--- a/src/uri/fetchers/docker.cpp
+++ b/src/uri/fetchers/docker.cpp
@@ -687,7 +687,10 @@ Future DockerFetcherPluginProcess::fetch(
// Note: The 'Accept' header is required for Amazon ECR. See:
// https://forums.aws.amazon.com/message.jspa?messageID=780440
http::Headers manifestHeaders = {
-{"Accept", "application/vnd.docker.distribution.manifest.v1+json"}
+{"Accept",
+ "application/vnd.docker.distribution.manifest.v1+json,"
+ "application/vnd.docker.distribution.manifest.v1+prettyjws"
+}
};
return curl(manifestUri, manifestHeaders + basicAuthHeaders, stallTimeout)
[mesos] 02/08: Refactored the UCR docker store to construct 'Image' proto at pullers.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit ed1587bc0ee4a94329b379c6c45430062bfd8e37
Author: Gilbert Song
AuthorDate: Mon Apr 1 16:11:43 2019 -0700
Refactored the UCR docker store to construct 'Image' proto at pullers.
This refactoring is needed for supporting docker manifest v2s2 because
the puller has to let the docker store knows the v1 config, and this
is also needed for garbage collecting the v1 config in the image layer
store.
Review: https://reviews.apache.org/r/70354
---
.../mesos/provisioner/docker/image_tar_puller.cpp | 22 +++--
.../mesos/provisioner/docker/image_tar_puller.hpp | 2 +-
.../mesos/provisioner/docker/message.proto | 4 +++-
.../mesos/provisioner/docker/metadata_manager.cpp | 28 ++
.../mesos/provisioner/docker/metadata_manager.hpp | 12 ++
.../mesos/provisioner/docker/puller.hpp| 4 +++-
.../mesos/provisioner/docker/registry_puller.cpp | 28 +-
.../mesos/provisioner/docker/registry_puller.hpp | 2 +-
.../mesos/provisioner/docker/store.cpp | 16 ++---
.../containerizer/provisioner_docker_tests.cpp | 20 +++-
10 files changed, 71 insertions(+), 67 deletions(-)
diff --git
a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
index 3a33388..3e6741d 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
@@ -66,13 +66,13 @@ public:
~ImageTarPullerProcess() override {}
- Future> pull(
+ Future pull(
const spec::ImageReference& reference,
const string& directory,
const string& backend);
private:
- Future> _pull(
+ Future _pull(
const spec::ImageReference& reference,
const string& directory,
const string& backend);
@@ -152,7 +152,7 @@ ImageTarPuller::~ImageTarPuller()
}
-Future> ImageTarPuller::pull(
+Future ImageTarPuller::pull(
const spec::ImageReference& reference,
const string& directory,
const string& backend,
@@ -167,7 +167,7 @@ Future> ImageTarPuller::pull(
}
-Future> ImageTarPullerProcess::pull(
+Future ImageTarPullerProcess::pull(
const spec::ImageReference& reference,
const string& directory,
const string& backend)
@@ -186,7 +186,7 @@ Future> ImageTarPullerProcess::pull(
<< "' to '" << directory << "' using HDFS uri fetcher";
return fetcher->fetch(uri, directory)
- .then(defer(self(), [=]() -> Future> {
+ .then(defer(self(), [=]() -> Future {
const string source = paths::getImageArchiveTarPath(directory, image);
VLOG(1) << "Untarring image '" << reference
@@ -216,7 +216,7 @@ Future> ImageTarPullerProcess::pull(
}
-Future> ImageTarPullerProcess::_pull(
+Future ImageTarPullerProcess::_pull(
const spec::ImageReference& reference,
const string& directory,
const string& backend)
@@ -294,7 +294,15 @@ Future> ImageTarPullerProcess::_pull(
}
return extractLayers(directory, layerIds, backend)
-.then([layerIds]() -> vector { return layerIds; });
+.then([reference, layerIds]() -> Image {
+ Image image;
+ image.mutable_reference()->CopyFrom(reference);
+ foreach (const string& layerId, layerIds) {
+image.add_layer_ids(layerId);
+ }
+
+ return image;
+});
}
diff --git
a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
index b5ec924..6c0fa83 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
@@ -50,7 +50,7 @@ public:
~ImageTarPuller() override;
- process::Future> pull(
+ process::Future pull(
const ::docker::spec::ImageReference& reference,
const std::string& directory,
const std::string& backend,
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.proto
b/src/slave/containerizer/mesos/provisioner/docker/message.proto
index a55ac90..612c591 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.proto
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.proto
@@ -28,7 +28,9 @@ package mesos.internal.slave.docker;
message Image {
required .docker.spec.ImageReference reference = 1;
- // The order of the layers represents the dependency between layers.
+ // The order of the layers represents the dependency between layers,
+ // where the root layer's id (no parent layer) is first and the leaf
+ // layer's id is last.
repeated string layer_ids = 2;
}
diff --git
[mesos] 07/08: Added gcr registry test.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 363e753c199faf5e928fc3ad732d43e036a97ee0
Author: Gilbert Song
AuthorDate: Fri Mar 22 10:37:46 2019 -0700
Added gcr registry test.
Review: https://reviews.apache.org/r/70291
---
src/tests/containerizer/provisioner_docker_tests.cpp | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 3da6628..5e30ed0 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -639,15 +639,18 @@ TEST_P(ProvisionerDockerHdfsTest,
ROOT_ImageTarPullerHdfsFetcherSimpleCommand)
// when specifying the repository name (e.g., 'busybox'). The registry
// puller normalize docker official images if necessary.
INSTANTIATE_TEST_CASE_P(
-ImageAlpine,
+ContainerImage,
ProvisionerDockerTest,
::testing::ValuesIn(vector({
"alpine", // Verifies the normalization of the Docker repository name.
"library/alpine",
+"gcr.io/google-containers/busybox:1.24", // manifest.v1+prettyjws
+"gcr.io/google-containers/busybox:1.27", // manifest.v2+json
// TODO(alexr): The registry below is unreliable and hence disabled.
// Consider re-enabling shall it become more stable.
// "registry.cn-hangzhou.aliyuncs.com/acs-sample/alpine",
-"quay.io/coreos/alpine-sh"})));
+"quay.io/coreos/alpine-sh" // manifest.v1+prettyjws
+ })));
// TODO(jieyu): This is a ROOT test because of MESOS-4757. Remove the
[mesos] 08/08: Added a unit test for Mesos containerizer image force pulling.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e6a3f0d0165fc48d70e9fe920b0273250b2d5e85
Author: Gilbert Song
AuthorDate: Wed Mar 27 17:33:19 2019 -0700
Added a unit test for Mesos containerizer image force pulling.
Review: https://reviews.apache.org/r/70366
---
.../containerizer/provisioner_docker_tests.cpp | 127 +
1 file changed, 127 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 5e30ed0..1c5d720 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -740,6 +740,133 @@ TEST_P(ProvisionerDockerTest,
ROOT_INTERNET_CURL_SimpleCommand)
}
+// This test verifies the functionality of image `cached` option
+// for image force pulling.
+TEST_F(ProvisionerDockerTest, ROOT_INTERNET_CURL_ImageForcePulling)
+{
+ Try> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "docker/runtime,filesystem/linux";
+ flags.image_providers = "docker";
+
+ // Image pulling time may be long, depending on the location of
+ // the registry server.
+ flags.executor_registration_timeout = Minutes(10);
+
+ Owned detector = master.get()->createDetector();
+ Try> slave = StartSlave(detector.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ , DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(, _, _));
+
+ Future> offers1, offers2;
+ EXPECT_CALL(sched, resourceOffers(, _))
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>())
+.WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers1);
+ ASSERT_EQ(1u, offers1->size());
+
+ const Offer& offer1 = offers1.get()[0];
+
+ // NOTE: We use a non-shell command here because 'sh' might not be
+ // in the PATH. 'alpine' does not specify env PATH in the image. On
+ // some linux distribution, '/bin' is not in the PATH by default.
+ CommandInfo command;
+ command.set_shell(false);
+ command.set_value("/bin/ls");
+ command.add_arguments("ls");
+ command.add_arguments("-al");
+ command.add_arguments("/");
+
+ TaskInfo task = createTask(
+ offer1.slave_id(),
+ Resources::parse("cpus:1;mem:128").get(),
+ command);
+
+ Image image;
+ image.set_type(Image::DOCKER);
+ image.mutable_docker()->set_name("alpine");
+
+ ContainerInfo* container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+ container->mutable_mesos()->mutable_image()->CopyFrom(image);
+
+ Future statusStarting1;
+ Future statusRunning1;
+ Future statusFinished1;
+ EXPECT_CALL(sched, statusUpdate(, _))
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>());
+
+ driver.launchTasks(offer1.id(), {task});
+
+ AWAIT_READY_FOR(statusStarting1, Minutes(10));
+ EXPECT_EQ(task.task_id(), statusStarting1->task_id());
+ EXPECT_EQ(TASK_STARTING, statusStarting1->state());
+
+ AWAIT_READY(statusRunning1);
+ EXPECT_EQ(task.task_id(), statusRunning1->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning1->state());
+
+ AWAIT_READY(statusFinished1);
+ EXPECT_EQ(task.task_id(), statusFinished1->task_id());
+ EXPECT_EQ(TASK_FINISHED, statusFinished1->state());
+
+ AWAIT_READY(offers2);
+ ASSERT_EQ(1u, offers2->size());
+
+ const Offer& offer2 = offers2.get()[0];
+
+ task = createTask(
+ offer2.slave_id(),
+ Resources::parse("cpus:1;mem:128").get(),
+ command);
+
+ // Image force pulling.
+ image.set_cached(false);
+
+ container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+ container->mutable_mesos()->mutable_image()->CopyFrom(image);
+
+ Future statusStarting2;
+ Future statusRunning2;
+ Future statusFinished2;
+ EXPECT_CALL(sched, statusUpdate(, _))
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>())
+.WillOnce(FutureArg<1>());
+
+ driver.launchTasks(offer2.id(), {task});
+
+ AWAIT_READY_FOR(statusStarting2, Minutes(10));
+ EXPECT_EQ(task.task_id(), statusStarting2->task_id());
+ EXPECT_EQ(TASK_STARTING, statusStarting2->state());
+
+ AWAIT_READY(statusRunning2);
+ EXPECT_EQ(task.task_id(), statusRunning2->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning2->state());
+
+ AWAIT_READY(statusFinished2);
+ EXPECT_EQ(task.task_id(), statusFinished2->task_id());
+ EXPECT_EQ(TASK_FINISHED, statusFinished2->state());
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test verifies that the scratch based docker image (that
// only contain a single binary and its dependencies) can be
// launched correctly.
[mesos] 05/08: Added a TODO for additional URLs support.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 0c6975345cf647838240f90ad97c0b7b49a9bba1
Author: Gilbert Song
AuthorDate: Sat Mar 23 21:14:15 2019 -0700
Added a TODO for additional URLs support.
Review: https://reviews.apache.org/r/70289
---
src/uri/fetchers/docker.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/uri/fetchers/docker.cpp b/src/uri/fetchers/docker.cpp
index e11e54f..631db2d 100644
--- a/src/uri/fetchers/docker.cpp
+++ b/src/uri/fetchers/docker.cpp
@@ -968,6 +968,8 @@ Future DockerFetcherPluginProcess::urlFetchBlob(
return Failure("Schema 2 manifest does not exist");
}
+ // TODO(gilbert): Support v2s2 additional urls for non-windows platforms.
+ // We should avoid parsing the manifest for each layer.
Try manifest = spec::v2_2::parse(_manifest.get());
if (manifest.isError()) {
return Failure(
[mesos] branch master updated (003e47d -> e6a3f0d)
This is an automated email from the ASF dual-hosted git repository. gilbert pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 003e47d Added CSI headers to CentOS devel RPM. new 10d5742 Added 'prettyjws' option to docker manifest V2 Schema1 accept header. new ed1587b Refactored the UCR docker store to construct 'Image' proto at pullers. new a64ba10 Added protobuf for docker v2 schema2 config_digest in 'Image'. new 9f04006 Supported docker manifest v2 schema2. new 0c69753 Added a TODO for additional URLs support. new 899740a Fixed docker fetcher plugin unit test for v2s2 change. new 363e753 Added gcr registry test. new e6a3f0d Added a unit test for Mesos containerizer image force pulling. The 8 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mesos/provisioner/docker/image_tar_puller.cpp | 22 +- .../mesos/provisioner/docker/image_tar_puller.hpp | 2 +- .../mesos/provisioner/docker/message.proto | 8 +- .../mesos/provisioner/docker/metadata_manager.cpp | 28 +- .../mesos/provisioner/docker/metadata_manager.hpp | 12 +- .../mesos/provisioner/docker/puller.hpp| 4 +- .../mesos/provisioner/docker/registry_puller.cpp | 294 +--- .../mesos/provisioner/docker/registry_puller.hpp | 2 +- .../mesos/provisioner/docker/store.cpp | 66 +++-- .../containerizer/provisioner_docker_tests.cpp | 154 ++- src/tests/uri_fetcher_tests.cpp| 60 ++-- src/uri/fetchers/docker.cpp| 303 + 12 files changed, 630 insertions(+), 325 deletions(-)
[mesos] 06/08: Fixed docker fetcher plugin unit test for v2s2 change.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 899740a625ae7a2756031023b997703844ecfe7b
Author: Gilbert Song
AuthorDate: Wed Mar 27 12:03:00 2019 -0700
Fixed docker fetcher plugin unit test for v2s2 change.
Review: https://reviews.apache.org/r/70290
---
src/tests/uri_fetcher_tests.cpp | 60 ++---
1 file changed, 26 insertions(+), 34 deletions(-)
diff --git a/src/tests/uri_fetcher_tests.cpp b/src/tests/uri_fetcher_tests.cpp
index f8bacc0..bb224d8 100644
--- a/src/tests/uri_fetcher_tests.cpp
+++ b/src/tests/uri_fetcher_tests.cpp
@@ -302,29 +302,17 @@ TEST_F(DockerFetcherPluginTest,
INTERNET_CURL_FetchManifest)
AWAIT_READY_FOR(fetcher.get()->fetch(uri, dir), Seconds(60));
- // Version 2 schema 1 image manifest test
- Try _s1Manifest = os::read(path::join(dir, "manifest"));
- ASSERT_SOME(_s1Manifest);
-
- Try s1Manifest =
-docker::spec::v2::parse(_s1Manifest.get());
-
- ASSERT_SOME(s1Manifest);
- EXPECT_EQ(1u, s1Manifest->schemaversion());
- EXPECT_EQ(TEST_REPOSITORY, s1Manifest->name());
- EXPECT_EQ("latest", s1Manifest->tag());
-
-#ifdef __WINDOWS__
// Version 2 schema 2 image manifest test
- Try _s2Manifest = os::read(path::join(dir, "manifest_v2s2"));
- ASSERT_SOME(_s2Manifest);
+ Try _manifest = os::read(path::join(dir, "manifest"));
+ ASSERT_SOME(_manifest);
- Try s2Manifest =
-docker::spec::v2_2::parse(_s2Manifest.get());
+ Try manifest =
+docker::spec::v2_2::parse(_manifest.get());
- ASSERT_SOME(s2Manifest);
- EXPECT_EQ(2u, s2Manifest->schemaversion());
-#endif // __WINDOWS__
+ ASSERT_SOME(manifest);
+ EXPECT_EQ(2u, manifest->schemaversion());
+ EXPECT_EQ("application/vnd.docker.distribution.manifest.v2+json",
+manifest->mediatype());
}
@@ -360,19 +348,21 @@ TEST_F(DockerFetcherPluginTest, INTERNET_CURL_FetchImage)
Try _manifest = os::read(path::join(dir, "manifest"));
ASSERT_SOME(_manifest);
- Try manifest =
- docker::spec::v2::parse(_manifest.get());
+ Try manifest =
+ docker::spec::v2_2::parse(_manifest.get());
ASSERT_SOME(manifest);
- EXPECT_EQ(1u, manifest->schemaversion());
- EXPECT_EQ(TEST_REPOSITORY, manifest->name());
- EXPECT_EQ("latest", manifest->tag());
+ EXPECT_EQ(2u, manifest->schemaversion());
+ EXPECT_EQ("application/vnd.docker.distribution.manifest.v2+json",
+manifest->mediatype());
- for (int i = 0; i < manifest->fslayers_size(); i++) {
+ EXPECT_TRUE(os::exists(path::join(dir, manifest->config().digest(;
+
+ for (int i = 0; i < manifest->layers_size(); i++) {
EXPECT_TRUE(os::exists(
DockerFetcherPlugin::getBlobPath(
dir,
-manifest->fslayers(i).blobsum(;
+manifest->layers(i).digest(;
}
}
@@ -395,19 +385,21 @@ TEST_F(DockerFetcherPluginTest,
INTERNET_CURL_InvokeFetchByName)
Try _manifest = os::read(path::join(dir, "manifest"));
ASSERT_SOME(_manifest);
- Try manifest =
- docker::spec::v2::parse(_manifest.get());
+ Try manifest =
+ docker::spec::v2_2::parse(_manifest.get());
ASSERT_SOME(manifest);
- EXPECT_EQ(1u, manifest->schemaversion());
- EXPECT_EQ(TEST_REPOSITORY, manifest->name());
- EXPECT_EQ("latest", manifest->tag());
+ EXPECT_EQ(2u, manifest->schemaversion());
+ EXPECT_EQ("application/vnd.docker.distribution.manifest.v2+json",
+manifest->mediatype());
+
+ EXPECT_TRUE(os::exists(path::join(dir, manifest->config().digest(;
- for (int i = 0; i < manifest->fslayers_size(); i++) {
+ for (int i = 0; i < manifest->layers_size(); i++) {
EXPECT_TRUE(os::exists(
DockerFetcherPlugin::getBlobPath(
dir,
-manifest->fslayers(i).blobsum(;
+manifest->layers(i).digest(;
}
}
[mesos] 04/08: Supported docker manifest v2 schema2.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 9f040066eaf0c6b742b0d86384fd71aae2036f26
Author: Gilbert Song
AuthorDate: Fri Mar 22 00:32:30 2019 -0700
Supported docker manifest v2 schema2.
Review: https://reviews.apache.org/r/70288
---
.../mesos/provisioner/docker/registry_puller.cpp | 266 +++---
.../mesos/provisioner/docker/store.cpp | 52 +++-
src/uri/fetchers/docker.cpp| 296 -
3 files changed, 392 insertions(+), 222 deletions(-)
diff --git
a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
index 7778976..35b6afb 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
@@ -25,6 +25,7 @@
#include
#include
+#include
#include
#include
@@ -92,13 +93,34 @@ private:
const hashset& blobSums,
const string& backend);
- Future> fetchBlobs(
+ Future pull(
const spec::ImageReference& reference,
const string& directory,
+const spec::v2_2::ImageManifest& manifest,
+const hashset& digests,
+const string& backend);
+
+ Future> fetchBlobs(
+const spec::ImageReference& normalizedRef,
+const string& directory,
const spec::v2::ImageManifest& manifest,
const string& backend,
const Option& config);
+ Future> fetchBlobs(
+ const spec::ImageReference& normalizedRef,
+ const string& directory,
+ const spec::v2_2::ImageManifest& manifest,
+ const string& backend,
+ const Option& config);
+
+ Future> fetchBlobs(
+ const spec::ImageReference& normalizedRef,
+ const string& directory,
+ const hashset& digests,
+ const string& backend,
+ const Option& config);
+
RegistryPullerProcess(const RegistryPullerProcess&) = delete;
RegistryPullerProcess& operator=(const RegistryPullerProcess&) = delete;
@@ -237,31 +259,31 @@ Future RegistryPullerProcess::pull(
Future RegistryPullerProcess::_pull(
-const spec::ImageReference& _reference,
+const spec::ImageReference& reference,
const string& directory,
const string& backend,
const Option& config)
{
- spec::ImageReference reference = normalize(_reference, defaultRegistryUrl);
+ spec::ImageReference normalizedRef = normalize(reference,
defaultRegistryUrl);
URI manifestUri;
- if (reference.has_registry()) {
-Result port = spec::getRegistryPort(reference.registry());
+ if (normalizedRef.has_registry()) {
+Result port = spec::getRegistryPort(normalizedRef.registry());
if (port.isError()) {
return Failure("Failed to get registry port: " + port.error());
}
-Try scheme = spec::getRegistryScheme(reference.registry());
+Try scheme = spec::getRegistryScheme(normalizedRef.registry());
if (scheme.isError()) {
return Failure("Failed to get registry scheme: " + scheme.error());
}
manifestUri = uri::docker::manifest(
-reference.repository(),
-(reference.has_digest()
- ? reference.digest()
- : (reference.has_tag() ? reference.tag() : "latest")),
-spec::getRegistryHost(reference.registry()),
+normalizedRef.repository(),
+(normalizedRef.has_digest()
+ ? normalizedRef.digest()
+ : (normalizedRef.has_tag() ? normalizedRef.tag() : "latest")),
+spec::getRegistryHost(normalizedRef.registry()),
scheme.get(),
port.isSome() ? port.get() : Option());
} else {
@@ -274,19 +296,24 @@ Future RegistryPullerProcess::_pull(
: Option();
manifestUri = uri::docker::manifest(
-reference.repository(),
-(reference.has_digest()
- ? reference.digest()
- : (reference.has_tag() ? reference.tag() : "latest")),
+normalizedRef.repository(),
+(normalizedRef.has_digest()
+ ? normalizedRef.digest()
+ : (normalizedRef.has_tag() ? normalizedRef.tag() : "latest")),
registry,
defaultRegistryUrl.scheme,
port);
}
- VLOG(1) << "Pulling image '" << reference
+ VLOG(1) << "Pulling image '" << normalizedRef
<< "' from '" << manifestUri
<< "' to '" << directory << "'";
+ // Pass the original 'reference' along to subsequent methods
+ // because metadata manager may already has this reference in
+ // cache. This is necessary to ensure the backward compatibility
+ // after upgrading to the version including MESOS-9675 for
+ // docker manifest v2 schema2 support.
return fetcher->fetch(
manifestUri,
directory,
@@ -306,21 +333,57 @@ Future RegistryPullerProcess::__pull(
return Failure("Failed to read the manifest: " + _manifest.error());
}
- Try
[mesos] 03/08: Added protobuf for docker v2 schema2 config_digest in 'Image'.
This is an automated email from the ASF dual-hosted git repository.
gilbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit a64ba106fbe1e0ea075f523a09b9831f8ad31387
Author: Gilbert Song
AuthorDate: Mon Apr 1 22:42:53 2019 -0700
Added protobuf for docker v2 schema2 config_digest in 'Image'.
Review: https://reviews.apache.org/r/70365
---
src/slave/containerizer/mesos/provisioner/docker/message.proto | 4
1 file changed, 4 insertions(+)
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.proto
b/src/slave/containerizer/mesos/provisioner/docker/message.proto
index 612c591..42fc4fa 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.proto
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.proto
@@ -32,6 +32,10 @@ message Image {
// where the root layer's id (no parent layer) is first and the leaf
// layer's id is last.
repeated string layer_ids = 2;
+
+ // The digest of Docker V2 Schema2 manifest config. Only exists when
+ // pulling an image via V2 Schema2 manifest.
+ optional string config_digest = 3;
}
[mesos] branch master updated: Added CSI headers to CentOS devel RPM.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 003e47d Added CSI headers to CentOS devel RPM.
003e47d is described below
commit 003e47d852f6eb9a1cd7c86750d91b37269aec26
Author: Chun-Hung Hsiao
AuthorDate: Thu Apr 4 11:39:33 2019 -0700
Added CSI headers to CentOS devel RPM.
---
support/packaging/centos/mesos.spec | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/support/packaging/centos/mesos.spec
b/support/packaging/centos/mesos.spec
index de905fd..54f7393 100644
--- a/support/packaging/centos/mesos.spec
+++ b/support/packaging/centos/mesos.spec
@@ -93,7 +93,6 @@ This package provides files for developing Mesos
frameworks/modules.
--enable-install-module-dependencies \
--enable-libevent \
--enable-ssl \
---enable-grpc \
--enable-hardening \
--enable-xfs-disk-isolator=%{_with_xfs}
@@ -160,6 +159,7 @@ install -m 0644 src/java/target/mesos-*.jar
%{buildroot}%{_datadir}/java/
##
%files devel
%doc LICENSE NOTICE
+%{_includedir}/csi/
%{_includedir}/elfio/
%{_includedir}/mesos/
%{_includedir}/stout/
[mesos] branch master updated: Updated the master to allocate recovered orphan operation resources.
This is an automated email from the ASF dual-hosted git repository. grag pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 6bf9582 Updated the master to allocate recovered orphan operation resources. 6bf9582 is described below commit 6bf9582c35cce5fe38ae559d2d44b03a8383b9ec Author: Greg Mann AuthorDate: Thu Apr 4 11:59:37 2019 -0700 Updated the master to allocate recovered orphan operation resources. This patch updates the master's framework recovery code to use the allocator's `addResourceProvider()` method rather than `updateSlave()` when recovering orphan operations, which has the benefit of tracking the allocation of the operations' consumed resources, avoiding situations in which those resources would be incorrectly offered to frameworks while the operation is still in a pending state. Review: https://reviews.apache.org/r/70325/ --- src/master/master.cpp | 21 ++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/master/master.cpp b/src/master/master.cpp index cf5caa0..ad54ae2 100644 --- a/src/master/master.cpp +++ b/src/master/master.cpp @@ -10543,15 +10543,30 @@ void Master::recoverFramework( slave->totalResources += consumedUnallocated; slave->usedResources[framework->id()] += consumed.get(); - allocator->updateSlave(slave->id, slave->info, slave->totalResources); + hashmap usedResources; + usedResources.put(framework->id(), consumed.get()); - // NOTE: The allocation of these orphan operation resources will be - // updated in `addFramework` below. + // This call to `addResourceProvider()` adds orphan operation + // resources back to the agent's total and used resources. This + // prevents these resources from being offered while the operation is + // still pending. + // + // NOTE: We intentionally call `addResourceProvider()` before we call + // `addFramework()` below because if the order were reversed, these + // resources would be incorrectly tracked twice in the allocator. + allocator->addResourceProvider( + slave->id, + consumedUnallocated, + usedResources); } } } } + // NOTE: We intentionally call `addFramework()` after we called + // `addResourceProvider()` above because if the order were reversed, the + // resources of orphan operations would be incorrectly tracked twice in the + // allocator. addFramework(framework, suppressedRoles); }
[mesos-site] branch asf-site updated: Updated the website built from mesos SHA: 995b243.
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/mesos-site.git The following commit(s) were added to refs/heads/asf-site by this push: new ed04675 Updated the website built from mesos SHA: 995b243. ed04675 is described below commit ed04675b57a33e8191b1b4ef8a8d8169e50fce29 Author: jenkins AuthorDate: Thu Apr 4 14:46:38 2019 + Updated the website built from mesos SHA: 995b243. --- content/documentation/committers/index.html| 7 +++ content/documentation/latest/committers/index.html | 7 +++ 2 files changed, 14 insertions(+) diff --git a/content/documentation/committers/index.html b/content/documentation/committers/index.html index 9c29362..8be2ee7 100644 --- a/content/documentation/committers/index.html +++ b/content/documentation/committers/index.html @@ -197,6 +197,13 @@ canonical list is located on +1 + Andrei Budnik + Mesosphere + + abud...@apache.org + + + +1 Benno Evers Mesosphere diff --git a/content/documentation/latest/committers/index.html b/content/documentation/latest/committers/index.html index e74eb52..3cf9287 100644 --- a/content/documentation/latest/committers/index.html +++ b/content/documentation/latest/committers/index.html @@ -197,6 +197,13 @@ canonical list is located on +1 + Andrei Budnik + Mesosphere + + abud...@apache.org + + + +1 Benno Evers Mesosphere
[mesos] branch master updated: Added Andrei Budnik to the committers list.
This is an automated email from the ASF dual-hosted git repository. abudnik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 995b243 Added Andrei Budnik to the committers list. 995b243 is described below commit 995b24307a6c679d89af11cb1185fe8b27b5ef86 Author: Andrei Budnik AuthorDate: Thu Apr 4 16:09:02 2019 +0200 Added Andrei Budnik to the committers list. --- docs/committers.md | 7 +++ 1 file changed, 7 insertions(+) diff --git a/docs/committers.md b/docs/committers.md index f3c6220..f7b2d03 100644 --- a/docs/committers.md +++ b/docs/committers.md @@ -95,6 +95,13 @@ canonical list is located on +1 + Andrei Budnik + Mesosphere + + abud...@apache.org + + + +1 Benno Evers Mesosphere
