This is an automated email from the ASF dual-hosted git repository. qianzhang pushed a commit to branch 1.5.x in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e068da28fcdac66c1a4654d818f43407906f6bb3 Author: Liangyu Zhao <[email protected]> AuthorDate: Tue Aug 28 13:32:17 2018 -0700 Windows: Parse version 2 schema 2 Docker image manifest. Added support to parse V2S2 Docker image manifest (https://docs.docker.com/registry/spec/manifest-v2-2/). Adopted the validation code from patch 53850. Review: https://reviews.apache.org/r/68451/ --- include/mesos/docker/spec.hpp | 18 ++++ include/mesos/docker/v2_2.hpp | 23 +++++ include/mesos/docker/v2_2.proto | 45 ++++++++++ src/CMakeLists.txt | 1 + src/Makefile.am | 11 ++- src/docker/spec.cpp | 66 ++++++++++++++ src/tests/CMakeLists.txt | 2 +- src/tests/containerizer/docker_spec_tests.cpp | 121 ++++++++++++++++++++++++++ 8 files changed, 284 insertions(+), 3 deletions(-) diff --git a/include/mesos/docker/spec.hpp b/include/mesos/docker/spec.hpp index 2879414..56827fe 100644 --- a/include/mesos/docker/spec.hpp +++ b/include/mesos/docker/spec.hpp @@ -30,6 +30,7 @@ #include <mesos/docker/v1.hpp> #include <mesos/docker/v2.hpp> +#include <mesos/docker/v2_2.hpp> namespace docker { namespace spec { @@ -119,6 +120,23 @@ Try<ImageManifest> parse(const JSON::Object& json); Try<ImageManifest> parse(const std::string& s); } // namespace v2 { + + +namespace v2_2 { + +// Validates if the specified docker v2 s2 image manifest conforms to the +// Docker v2 s2 spec. Returns the error if the validation fails. +Option<Error> validate(const ImageManifest& manifest); + + +// Returns the docker v2 s2 image manifest from the given JSON object. +Try<ImageManifest> parse(const JSON::Object& json); + + +// Returns the docker v2 s2 image manifest from the given string. +Try<ImageManifest> parse(const std::string& s); + +} // namespace v2_2 { } // namespace spec { } // namespace docker { diff --git a/include/mesos/docker/v2_2.hpp b/include/mesos/docker/v2_2.hpp new file mode 100644 index 0000000..19f710f --- /dev/null +++ b/include/mesos/docker/v2_2.hpp @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __MESOS_DOCKER_V2_2_HPP__ +#define __MESOS_DOCKER_V2_2_HPP__ + +// ONLY USEFUL AFTER RUNNING PROTOC. +#include <mesos/docker/v2_2.pb.h> + +#endif // __MESOS_DOCKER_V2_2_HPP__ diff --git a/include/mesos/docker/v2_2.proto b/include/mesos/docker/v2_2.proto new file mode 100644 index 0000000..17bb11a --- /dev/null +++ b/include/mesos/docker/v2_2.proto @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package docker.spec.v2_2; + +/** + * Protobuf for the Docker version 2 schema 2 image manifest JSON schema: + * https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md + */ +message ImageManifest { + required uint32 schemaVersion = 1; + required string mediaType = 2; + + message Config { + required string mediaType = 1; + required uint32 size = 2; + required string digest = 3; + } + + required Config config = 3; + + message Layer { + required string mediaType = 1; + required uint32 size = 2; + required string digest = 3; + repeated string urls = 4; + } + + repeated Layer layers = 4; +} \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6532308..c825855 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ PROTOC_GENERATE(TARGET mesos/authorizer/authorizer) PROTOC_GENERATE(TARGET mesos/docker/spec) PROTOC_GENERATE(TARGET mesos/docker/v1) PROTOC_GENERATE(TARGET mesos/docker/v2) +PROTOC_GENERATE(TARGET mesos/docker/v2_2) PROTOC_GENERATE(TARGET mesos/maintenance/maintenance) PROTOC_GENERATE(TARGET mesos/master/master) PROTOC_GENERATE(TARGET mesos/module/hook) diff --git a/src/Makefile.am b/src/Makefile.am index 34f0d86..3d6751f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -270,6 +270,7 @@ CONTAINERIZER_PROTO = $(top_srcdir)/include/mesos/slave/containerizer.proto DOCKER_SPEC_PROTO = $(top_srcdir)/include/mesos/docker/spec.proto DOCKER_V1_PROTO = $(top_srcdir)/include/mesos/docker/v1.proto DOCKER_V2_PROTO = $(top_srcdir)/include/mesos/docker/v2.proto +DOCKER_V2_2_PROTO = $(top_srcdir)/include/mesos/docker/v2_2.proto EXECUTOR_PROTO = $(top_srcdir)/include/mesos/executor/executor.proto FETCHER_PROTO = $(top_srcdir)/include/mesos/fetcher/fetcher.proto HOOK_PROTO = $(top_srcdir)/include/mesos/module/hook.proto @@ -316,6 +317,8 @@ CXX_PROTOS = \ ../include/mesos/docker/v1.pb.h \ ../include/mesos/docker/v2.pb.cc \ ../include/mesos/docker/v2.pb.h \ + ../include/mesos/docker/v2_2.pb.cc \ + ../include/mesos/docker/v2_2.pb.h \ ../include/mesos/executor/executor.pb.cc \ ../include/mesos/executor/executor.pb.h \ ../include/mesos/fetcher/fetcher.pb.cc \ @@ -673,12 +676,15 @@ docker_HEADERS = \ $(top_srcdir)/include/mesos/docker/v1.hpp \ $(top_srcdir)/include/mesos/docker/v1.proto \ $(top_srcdir)/include/mesos/docker/v2.hpp \ - $(top_srcdir)/include/mesos/docker/v2.proto + $(top_srcdir)/include/mesos/docker/v2.proto \ + $(top_srcdir)/include/mesos/docker/v2_2.hpp \ + $(top_srcdir)/include/mesos/docker/v2_2.proto nodist_docker_HEADERS = \ ../include/mesos/docker/spec.pb.h \ ../include/mesos/docker/v1.pb.h \ - ../include/mesos/docker/v2.pb.h + ../include/mesos/docker/v2.pb.h \ + ../include/mesos/docker/v2_2.pb.h executordir = $(pkgincludedir)/executor @@ -1633,6 +1639,7 @@ libmesos_la_SOURCES = \ $(DOCKER_SPEC_PROTO) \ $(DOCKER_V1_PROTO) \ $(DOCKER_V2_PROTO) \ + $(DOCKER_V2_2_PROTO) \ $(FETCHER_PROTO) \ $(HOOK_PROTO) \ $(MAINTENANCE_PROTO) \ diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp index 538cf18..4253520 100644 --- a/src/docker/spec.cpp +++ b/src/docker/spec.cpp @@ -408,5 +408,71 @@ Try<ImageManifest> parse(const string& s) } } // namespace v2 { + +namespace v2_2 { + +Option<Error> validate(const ImageManifest& manifest) +{ + // Validate required fields are present, + // e.g., repeated fields that has to be >= 1. + if (manifest.layers_size() <= 0) { + return Error("'layers' field size must be at least one"); + } + + // Verify 'config' field. + if (!strings::contains(manifest.config().digest(), ":")) { + return Error("Incorrect 'digest' format: " + manifest.config().digest()); + } + + // Verify 'layers' field. + for (int i = 0; i < manifest.layers_size(); ++i) { + if (!strings::contains(manifest.layers(i).digest(), ":")) { + return Error("Incorrect 'digest' format: " + manifest.layers(i).digest()); + } + } + + if (manifest.schemaversion() != 2) { + return Error("'schemaVersion' field must be 2"); + } + + if (manifest.mediatype() != + "application/vnd.docker.distribution.manifest.v2+json") { + return Error( + "'mediaType' field must be " + "'application/vnd.docker.distribution.manifest.v2+json'"); + } + + return None(); +} + + +Try<ImageManifest> parse(const JSON::Object& json) +{ + Try<ImageManifest> manifest = protobuf::parse<ImageManifest>(json); + if (manifest.isError()) { + return Error("Protobuf parse failed: " + manifest.error()); + } + + Option<Error> error = validate(manifest.get()); + if (error.isSome()) { + return Error( + "Docker v2 s2 image manifest validation failed: " + error->message); + } + + return manifest.get(); +} + + +Try<ImageManifest> parse(const string& s) +{ + Try<JSON::Object> json = JSON::parse<JSON::Object>(s); + if (json.isError()) { + return Error("JSON parse failed: " + json.error()); + } + + return parse(json.get()); +} + +} // namespace v2_2 { } // namespace spec { } // namespace docker { diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 2585b6c..2a614cb 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -151,6 +151,7 @@ list(APPEND MESOS_TESTS_SRC containerizer/containerizer_tests.cpp containerizer/cpu_isolator_tests.cpp containerizer/memory_isolator_tests.cpp + containerizer/docker_spec_tests.cpp containerizer/docker_tests.cpp) if (NOT WIN32) @@ -191,7 +192,6 @@ if (NOT WIN32) containerizer/appc_spec_tests.cpp containerizer/composing_containerizer_tests.cpp containerizer/docker_containerizer_tests.cpp - containerizer/docker_spec_tests.cpp containerizer/environment_secret_isolator_tests.cpp containerizer/io_switchboard_tests.cpp containerizer/isolator_tests.cpp diff --git a/src/tests/containerizer/docker_spec_tests.cpp b/src/tests/containerizer/docker_spec_tests.cpp index 5fde49a..ffd5bcc 100644 --- a/src/tests/containerizer/docker_spec_tests.cpp +++ b/src/tests/containerizer/docker_spec_tests.cpp @@ -619,6 +619,127 @@ TEST_F(DockerSpecTest, ValidateV2ImageManifestSignaturesNonEmpty) EXPECT_ERROR(manifest); } + +TEST_F(DockerSpecTest, ParseV2_2ImageManifest) +{ + Try<JSON::Object> json = JSON::parse<JSON::Object>( + R"~( + { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ] + })~"); + + ASSERT_SOME(json); + + Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get()); + ASSERT_SOME(manifest); + + EXPECT_EQ(2u, manifest->schemaversion()); + EXPECT_EQ( + "application/vnd.docker.distribution.manifest.v2+json", + manifest->mediatype()); + + EXPECT_EQ( + "application/vnd.docker.container.image.v1+json", + manifest->config().mediatype()); + EXPECT_EQ(7023u, manifest->config().size()); + EXPECT_EQ( + "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + manifest->config().digest()); + + EXPECT_EQ(3, manifest->layers_size()); + + EXPECT_EQ( + "application/vnd.docker.image.rootfs.diff.tar.gzip", + manifest->layers(0).mediatype()); + EXPECT_EQ(32654u, manifest->layers(0).size()); + EXPECT_EQ( + "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + manifest->layers(0).digest()); + + EXPECT_EQ( + "application/vnd.docker.image.rootfs.diff.tar.gzip", + manifest->layers(1).mediatype()); + EXPECT_EQ(16724u, manifest->layers(1).size()); + EXPECT_EQ( + "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + manifest->layers(1).digest()); + + EXPECT_EQ( + "application/vnd.docker.image.rootfs.diff.tar.gzip", + manifest->layers(2).mediatype()); + EXPECT_EQ(73109u, manifest->layers(2).size()); + EXPECT_EQ( + "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + manifest->layers(2).digest()); +} + + +TEST_F(DockerSpecTest, ParseInvalidV2_2ImageManifest) +{ + // This is an invalid manifest. The size of the repeated fields + // 'layers' must be >= 1. The 'signatures' and + // 'schemaVersion' fields are not set. + Try<JSON::Object> json = JSON::parse<JSON::Object>( + R"~( + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + } + })~"); + + ASSERT_SOME(json); + + Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get()); + EXPECT_ERROR(manifest); +} + + +TEST_F(DockerSpecTest, ValidateV2_2ImageManifestLayersNonEmpty) +{ + Try<JSON::Object> json = JSON::parse<JSON::Object>( + R"~( + { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + } + })~"); + + ASSERT_SOME(json); + + Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get()); + EXPECT_ERROR(manifest); +} + } // namespace tests { } // namespace internal { } // namespace mesos {
