Repository: hadoop Updated Branches: refs/heads/trunk c07715e37 -> 67173b484
HDDS-441. Create new s3gateway daemon. Contributed by Elek Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67173b48 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67173b48 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67173b48 Branch: refs/heads/trunk Commit: 67173b4847908cdf27df143ca61a785aef9b835a Parents: c07715e Author: Bharat Viswanadham <[email protected]> Authored: Mon Sep 24 13:42:55 2018 -0700 Committer: Bharat Viswanadham <[email protected]> Committed: Mon Sep 24 13:42:55 2018 -0700 ---------------------------------------------------------------------- hadoop-ozone/common/src/main/bin/ozone | 7 +- hadoop-ozone/dist/pom.xml | 11 ++ .../main/compose/ozones3/docker-compose.yaml | 56 ++++++ .../dist/src/main/compose/ozones3/docker-config | 73 ++++++++ hadoop-ozone/pom.xml | 1 + hadoop-ozone/s3gateway/pom.xml | 173 +++++++++++++++++++ .../org/apache/hadoop/ozone/s3/Gateway.java | 55 ++++++ hadoop-project/pom.xml | 5 + 8 files changed, 380 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/common/src/main/bin/ozone ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index d173ec6..4b50771 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -44,6 +44,7 @@ function hadoop_usage hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" hadoop_add_subcommand "om" daemon "Ozone Manager" hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" + hadoop_add_subcommand "s3g" daemon "run the S3 compatible REST gateway" hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager" hadoop_add_subcommand "sh" client "command line interface for object store operations" hadoop_add_subcommand "version" client "print the version" @@ -114,7 +115,11 @@ function ozonecmd_case hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" - + ;; + s3g) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway' + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-s3gateway" ;; fs) HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index ba9f14f..7d11372 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -62,6 +62,13 @@ </artifactItem> <artifactItem> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-s3gateway</artifactId> + <version>${ozone.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-ozone-s3gateway.classpath</destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-ozone-manager</artifactId> <version>${ozone.version}</version> <classifier>classpath</classifier> @@ -166,6 +173,10 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-s3gateway</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-ozone-manager</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml new file mode 100644 index 0000000..10dc61e --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9864 + command: ["ozone","datanode"] + env_file: + - ./docker-config + ozoneManager: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION + env_file: + - ./docker-config + command: ["ozone","om"] + scm: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone","scm"] + s3g: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + env_file: + - ./docker-config + command: ["ozone","s3g"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/dist/src/main/compose/ozones3/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config new file mode 100644 index 0000000..2b22874 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=true +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 8d3e161..075c674 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -43,6 +43,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <module>integration-test</module> <module>objectstore-service</module> <module>datanode</module> + <module>s3gateway</module> <module>dist</module> <module>docs</module> </modules> http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/s3gateway/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml new file mode 100644 index 0000000..7b02c85 --- /dev/null +++ b/hadoop-ozone/s3gateway/pom.xml @@ -0,0 +1,173 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>hadoop-ozone-s3gateway</artifactId> + <name>Apache Hadoop Ozone S3 Gatway</name> + <packaging>jar</packaging> + <version>0.3.0-SNAPSHOT</version> + <properties> + <file.encoding>UTF-8</file.encoding> + <downloadSources>true</downloadSources> + </properties> + + <build> + <plugins> + + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-common</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.jboss.weld.servlet</groupId> + <artifactId>weld-servlet</artifactId> + <version>2.4.7.Final</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet-core</artifactId> + <version>2.27</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.ext.cdi</groupId> + <artifactId>jersey-cdi1x</artifactId> + <version>2.27</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.inject</groupId> + <artifactId>jersey-hk2</artifactId> + <version>2.27</version> + </dependency> + <dependency> + <groupId>javax.enterprise</groupId> + <artifactId>cdi-api</artifactId> + <version>1.2</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <type>test-jar</type> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-client</artifactId> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>2.15.0</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java new file mode 100644 index 0000000..4e3e48e --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; + +/** + * This class is used to start/stop S3 compatible rest server. + */ +@Command(name = "ozone s3g", + hidden = true, description = "S3 compatible rest server.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +public class Gateway extends GenericCli { + + private static final Logger LOG = LoggerFactory.getLogger(Gateway.class); + + public static void main(String[] args) throws Exception { + new Gateway().run(args); + } + + @Override + public Void call() throws Exception { + start(); + return null; + } + + public void start() { + LOG.info("Starting Ozone S3 gateway"); + } + + public void stop() { + LOG.info("Stoping Ozone S3 gateway"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/67173b48/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8d08703..e7baee0 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -623,6 +623,11 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-s3gateway</artifactId> + <version>${hdds.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-integration-test</artifactId> <version>${hdds.version}</version> <type>test-jar</type> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
