Repository: mesos Updated Branches: refs/heads/master d9b9bcfc1 -> 9b9730493
Removed docker puller flag. Review: https://reviews.apache.org/r/41731 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9b973049 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9b973049 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9b973049 Branch: refs/heads/master Commit: 9b97304934b611b384957a11b7e45778433aab2c Parents: d9b9bcf Author: Timothy Chen <[email protected]> Authored: Sat Dec 26 10:19:07 2015 -0800 Committer: Timothy Chen <[email protected]> Committed: Tue Jan 5 17:29:28 2016 -0800 ---------------------------------------------------------------------- docs/configuration.md | 5 +++- docs/mesos-provisioner.md | 10 +++---- .../mesos/provisioner/docker/local_puller.cpp | 29 ++++++++++++++++---- .../mesos/provisioner/docker/local_puller.hpp | 7 +++-- .../mesos/provisioner/docker/puller.cpp | 20 +++++++------- src/slave/flags.cpp | 15 +++------- src/slave/flags.hpp | 2 -- .../containerizer/provisioner_docker_tests.cpp | 27 ++++++++---------- 8 files changed, 64 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index c689e22..fb6f678 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1047,7 +1047,10 @@ file:///path/to/file (where file contains one of the above)</code></pre> --docker_registry=VALUE </td> <td> - Default Docker image registry server. + The default url for pulling Docker images. It could either be a Docker + registry server url (i.e: https://registry.docker.io), or a local path + (i.e: file:///tmp/docker/images) in which Docker image archives + (result of 'docker save') are stored. (default: https://registry-1.docker.io) </td> </tr> http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/docs/mesos-provisioner.md ---------------------------------------------------------------------- diff --git a/docs/mesos-provisioner.md b/docs/mesos-provisioner.md index fdb298c..06094f5 100644 --- a/docs/mesos-provisioner.md +++ b/docs/mesos-provisioner.md @@ -52,11 +52,11 @@ Image can both be configured in a Volume and in ContainerInfo at the same time. ## Setup and Agent Flags -To run the agent to enable Mesos containerizer, you must launch the agent with mesos as a containerizer option, which is the default containerizer for the mesos agent. It also has to enable linux/filesystem as part of the enabled isolators. The supported image providers can also be configured through agent flags, as well as the supported image provider backend. +To run the agent to enable Mesos containerizer, you must launch the agent with mesos as a containerizer option, which is the default containerizer for the mesos agent. It also has to enable filesystem/linux as part of the enabled isolators. The supported image providers can also be configured through agent flags, as well as the supported image provider backend. Mesos agent also needs to be running under linux with root permissions. -* Example: `mesos-slave --containerizers=mesos --image_providers=appc,docker --image_provisioner_backend=copy --isolation=linux/filesystem` +* Example: `mesos-slave --containerizers=mesos --image_providers=appc,docker --image_provisioner_backend=copy --isolation=filesystem/linux` Each Image provider can have additional configurations that can be set. @@ -79,11 +79,11 @@ TODO(tnachen): Add Appc information. https://github.com/docker/docker/blob/master/image/spec/v1.md -Docker image provider supports two different methods of finding and pulling images: local and registry. The `docker_puller` agent flag allows the slave to choose between these methods. +Docker image provider supports two different methods of finding and pulling images: local and registry. The `docker_registry` agent flag allows the slave to choose between these methods based on the URL scheme. -Local puller finds docker images based on image name and tag in the host filesystem, in the directory configured by `docker_local_archives_dir`. +By specifying a URL in `docker_registry` agent flag pointing to a local directory (file:///tmp/mesos/images/docker), the provisioner will use the Local puller to find docker images based on image name and tag in the host filesystem. -Registry puller finds and downloads images by contacting Docker registry, and by default contacts server configured by `docker_registry` and `docker_registry_port` agent flag when no custom registry is specified. +If the URL configured in `docker_registry` isn't pointing to a local directory, it will be assumed to be a registry referring to a Docker registry. The Registry puller will find and download images by contacting Docker registry with the configured URL when no custom registry is specified. Note that to run the Registry puller Mesos agent must be running with SSL enabled. http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp index f71b572..6ae920a 100644 --- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp +++ b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp @@ -23,6 +23,7 @@ #include <stout/json.hpp> #include <stout/os.hpp> #include <stout/result.hpp> +#include <stout/strings.hpp> #include <process/collect.hpp> #include <process/defer.hpp> @@ -51,7 +52,7 @@ namespace docker { class LocalPullerProcess : public Process<LocalPullerProcess> { public: - LocalPullerProcess(const Flags& _flags) : flags(_flags) {} + LocalPullerProcess(const string& _archivesDir) : archivesDir(_archivesDir) {} ~LocalPullerProcess() {} @@ -68,13 +69,31 @@ private: const string& directory, const vector<string>& layerIds); - const Flags flags; + const string archivesDir; }; -LocalPuller::LocalPuller(const Flags& flags) +Try<Owned<Puller>> LocalPuller::create(const Flags& flags) +{ + // This should already been verified at puller.cpp. + if (!strings::startsWith(flags.docker_registry, "file://")) { + return Error("Expecting registry url to have file:// scheme"); + } + + const string archivesDir = strings::remove( + flags.docker_registry, + "file://", + strings::Mode::PREFIX); + + Owned<LocalPullerProcess> process(new LocalPullerProcess(archivesDir)); + + return Owned<Puller>(new LocalPuller(process)); +} + + +LocalPuller::LocalPuller(Owned<LocalPullerProcess>& _process) + : process(_process) { - process = Owned<LocalPullerProcess>(new LocalPullerProcess(flags)); spawn(process.get()); } @@ -99,7 +118,7 @@ Future<list<pair<string, string>>> LocalPullerProcess::pull( const string& directory) { const string tarPath = paths::getImageArchiveTarPath( - flags.docker_local_archives_dir, + archivesDir, stringify(name)); if (!os::exists(tarPath)) { http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp index 025d96c..7f441de 100644 --- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp +++ b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp @@ -41,7 +41,7 @@ class LocalPullerProcess; class LocalPuller : public Puller { public: - explicit LocalPuller(const Flags& flags); + static Try<process::Owned<Puller>> create(const Flags& flags); ~LocalPuller(); @@ -50,9 +50,12 @@ public: const Path& directory); private: - LocalPuller& operator=(const LocalPuller&) = delete; // Not assignable. + explicit LocalPuller(process::Owned<LocalPullerProcess>& _process); + LocalPuller(const LocalPuller&) = delete; // Not copyable. + LocalPuller& operator=(const LocalPuller&) = delete; // Not assignable. + process::Owned<LocalPullerProcess> process; }; http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/slave/containerizer/mesos/provisioner/docker/puller.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp index dd17acf..5650f2f 100644 --- a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp +++ b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp @@ -50,22 +50,22 @@ namespace docker { Try<Owned<Puller>> Puller::create(const Flags& flags) { - const string puller = flags.docker_puller; - - if (puller == "local") { - return Owned<Puller>(new LocalPuller(flags)); - } - - if (puller == "registry") { - Try<Owned<Puller>> puller = RegistryPuller::create(flags); + // TODO(tnachen): Support multiple registries in the puller. + if (strings::startsWith(flags.docker_registry, "file://")) { + Try<Owned<Puller>> puller = LocalPuller::create(flags); if (puller.isError()) { - return Error("Failed to create registry puller: " + puller.error()); + return Error("Failed to create local puller: " + puller.error()); } return puller.get(); } - return Error("Unknown or unsupported docker puller: " + puller); + Try<Owned<Puller>> puller = RegistryPuller::create(flags); + if (puller.isError()) { + return Error("Failed to create registry puller: " + puller.error()); + } + + return puller.get(); } http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/slave/flags.cpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp index a60d3c8..19c2996 100644 --- a/src/slave/flags.cpp +++ b/src/slave/flags.cpp @@ -123,16 +123,6 @@ mesos::internal::slave::Flags::Flags() "Docker authentication server used to authenticate with Docker registry", "https://auth.docker.io";); - add(&Flags::docker_local_archives_dir, - "docker_local_archives_dir", - "Directory for Docker local puller to look in for image archives", - "/tmp/mesos/images/docker"); - - add(&Flags::docker_puller, - "docker_puller", - "Strategy for Docker puller to fetch images", - "local"); - add(&Flags::docker_puller_timeout_secs, "docker_puller_timeout", "Timeout in seconds for pulling images from the Docker registry", @@ -140,7 +130,10 @@ mesos::internal::slave::Flags::Flags() add(&Flags::docker_registry, "docker_registry", - "Default Docker image registry server", + "The default url for pulling Docker images. It could either be a Docker\n" + "registry server url (i.e: https://registry.docker.io), or a local path\n" + "(i.e: file:///tmp/docker/images) in which Docker image archives\n" + "(result of 'docker save') are stored.", "https://registry-1.docker.io";); add(&Flags::docker_store_dir, http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index 2b2679c..6857fde 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -52,8 +52,6 @@ public: std::string appc_store_dir; std::string docker_auth_server; - std::string docker_local_archives_dir; - std::string docker_puller; std::string docker_puller_timeout_secs; std::string docker_registry; std::string docker_store_dir; http://git-wip-us.apache.org/repos/asf/mesos/blob/9b973049/src/tests/containerizer/provisioner_docker_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp index 8635eee..8d6a060 100644 --- a/src/tests/containerizer/provisioner_docker_tests.cpp +++ b/src/tests/containerizer/provisioner_docker_tests.cpp @@ -1072,9 +1072,9 @@ protected: { TemporaryDirectoryTest::SetUp(); - const string imageDir = path::join(os::getcwd(), "images"); - const string image = path::join(imageDir, "abc:latest"); - ASSERT_SOME(os::mkdir(imageDir)); + const string archivesDir = path::join(os::getcwd(), "images"); + const string image = path::join(archivesDir, "abc:latest"); + ASSERT_SOME(os::mkdir(archivesDir)); ASSERT_SOME(os::mkdir(image)); JSON::Value repositories = JSON::parse( @@ -1133,15 +1133,14 @@ protected: // stored in the proper locations accessible to the Docker provisioner. TEST_F(ProvisionerDockerLocalStoreTest, LocalStoreTestWithTar) { - const string imageDir = path::join(os::getcwd(), "images"); - const string image = path::join(imageDir, "abc:latest"); - ASSERT_SOME(os::mkdir(imageDir)); + const string archivesDir = path::join(os::getcwd(), "images"); + const string image = path::join(archivesDir, "abc:latest"); + ASSERT_SOME(os::mkdir(archivesDir)); ASSERT_SOME(os::mkdir(image)); slave::Flags flags; - flags.docker_puller = "local"; + flags.docker_registry = "file://" + archivesDir; flags.docker_store_dir = path::join(os::getcwd(), "store"); - flags.docker_local_archives_dir = imageDir; Try<Owned<slave::Store>> store = slave::docker::Store::create(flags); ASSERT_SOME(store); @@ -1162,9 +1161,8 @@ TEST_F(ProvisionerDockerLocalStoreTest, LocalStoreTestWithTar) TEST_F(ProvisionerDockerLocalStoreTest, MetadataManagerInitialization) { slave::Flags flags; - flags.docker_puller = "local"; + flags.docker_registry = "file://" + path::join(os::getcwd(), "images"); flags.docker_store_dir = path::join(os::getcwd(), "store"); - flags.docker_local_archives_dir = path::join(os::getcwd(), "images"); Try<Owned<slave::Store>> store = slave::docker::Store::create(flags); ASSERT_SOME(store); @@ -1222,15 +1220,14 @@ public: // when multiple requests for the same image is in flight. TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously) { - const string imageDir = path::join(os::getcwd(), "images"); - const string image = path::join(imageDir, "abc:latest"); - ASSERT_SOME(os::mkdir(imageDir)); + const string archivesDir = path::join(os::getcwd(), "images"); + const string image = path::join(archivesDir, "abc:latest"); + ASSERT_SOME(os::mkdir(archivesDir)); ASSERT_SOME(os::mkdir(image)); slave::Flags flags; - flags.docker_puller = "local"; + flags.docker_registry = "file://" + archivesDir; flags.docker_store_dir = path::join(os::getcwd(), "store"); - flags.docker_local_archives_dir = imageDir; MockPuller* puller = new MockPuller(); Future<Nothing> pull;
