Repository: incubator-gearpump Updated Branches: refs/heads/master 1b9889121 -> a01809b25
[GEARPUMP-222] remove DFSJarStore from gearpump-daemon Author: huafengw <fvunic...@gmail.com> Closes #96 from huafengw/jarclass. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/a01809b2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/a01809b2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/a01809b2 Branch: refs/heads/master Commit: a01809b2588a5dd1038b7afa8dca5148729e634c Parents: 1b98891 Author: huafengw <fvunic...@gmail.com> Authored: Wed Oct 12 16:06:15 2016 +0800 Committer: manuzhang <owenzhang1...@gmail.com> Committed: Wed Oct 12 16:06:15 2016 +0800 ---------------------------------------------------------------------- .../org.apache.gearpump.jarstore.JarStore | 19 ++++++ .../org.apache.gearpump.jarstore.JarStore | 19 ++++++ .../org.apache.gearpump.jarstore.JarStore | 20 ------ .../gearpump/jarstore/dfs/DFSJarStore.scala | 67 -------------------- .../org.apache.gearpump.jarstore.JarStore | 20 ------ .../gearpump/cluster/utils/SystemOperation.java | 7 +- .../org.apache.gearpump.jarstore.JarStore | 19 ++++++ .../gearpump/jarstore/dfs/DFSJarStore.scala | 66 +++++++++++++++++++ project/Build.scala | 23 +++++-- project/Pack.scala | 3 +- 10 files changed, 145 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore new file mode 100644 index 0000000..c64d444 --- /dev/null +++ b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.gearpump.jarstore.local.LocalJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore new file mode 100644 index 0000000..c64d444 --- /dev/null +++ b/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.gearpump.jarstore.local.LocalJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore deleted file mode 100644 index e173a8a..0000000 --- a/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.apache.gearpump.jarstore.local.LocalJarStore -org.apache.gearpump.jarstore.dfs.DFSJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala ---------------------------------------------------------------------- diff --git a/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala b/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala deleted file mode 100644 index ebaf354..0000000 --- a/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gearpump.jarstore.dfs - -import java.io.{InputStream, OutputStream} - -import com.typesafe.config.Config -import org.apache.gearpump.jarstore.JarStore -import org.apache.gearpump.util.Constants -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.hadoop.fs.permission.{FsAction, FsPermission} - -/** - * DFSJarStore store the uploaded jar on HDFS - */ -class DFSJarStore extends JarStore { - private var rootPath: Path = null - override val scheme: String = "hdfs" - - override def init(config: Config): Unit = { - rootPath = new Path(config.getString(Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH)) - val fs = rootPath.getFileSystem(new Configuration()) - if (!fs.exists(rootPath)) { - fs.mkdirs(rootPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) - } - } - - /** - * Creates the file on JarStore. - * - * @param fileName name of the file to be created on JarStore. - * @return OutputStream returns a stream into which the data can be written. - */ - override def createFile(fileName: String): OutputStream = { - val filePath = new Path(rootPath, fileName) - val fs = filePath.getFileSystem(new Configuration()) - fs.create(filePath) - } - - /** - * Gets the InputStream to read the file - * - * @param fileName name of the file to be read on JarStore. - * @return InputStream returns a stream from which the data can be read. - */ - override def getFile(fileName: String): InputStream = { - val filePath = new Path(rootPath, fileName) - val fs = filePath.getFileSystem(new Configuration()) - fs.open(filePath) - } -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore deleted file mode 100644 index e173a8a..0000000 --- a/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.apache.gearpump.jarstore.local.LocalJarStore -org.apache.gearpump.jarstore.dfs.DFSJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java ---------------------------------------------------------------------- diff --git a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java index 5b2a890..4983dde 100644 --- a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java +++ b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java @@ -17,11 +17,12 @@ */ package org.apache.gearpump.cluster.utils; -import org.apache.commons.io.IOUtils; +import org.apache.gearpump.google.common.io.CharStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStreamReader; public class SystemOperation { @@ -44,8 +45,8 @@ public class SystemOperation { Process process = new ProcessBuilder(new String[]{"/bin/bash", "-c", cmd}).start(); try { process.waitFor(); - String output = IOUtils.toString(process.getInputStream()); - String errorOutput = IOUtils.toString(process.getErrorStream()); + String output = CharStreams.toString(new InputStreamReader(process.getInputStream())); + String errorOutput = CharStreams.toString(new InputStreamReader(process.getErrorStream())); LOG.debug("Shell Output: " + output); if (errorOutput.length() != 0) { LOG.error("Shell Error Output: " + errorOutput); http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore new file mode 100644 index 0000000..2809247 --- /dev/null +++ b/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.gearpump.jarstore.dfs.DFSJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala ---------------------------------------------------------------------- diff --git a/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala b/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala new file mode 100644 index 0000000..ae4cf46 --- /dev/null +++ b/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gearpump.jarstore.dfs + +import java.io.{InputStream, OutputStream} +import org.apache.gearpump.util.Constants +import org.apache.gearpump.jarstore.JarStore +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import com.typesafe.config.Config +import org.apache.hadoop.fs.permission.{FsAction, FsPermission} + +/** + * DFSJarStore store the uploaded jar on HDFS + */ +class DFSJarStore extends JarStore { + private var rootPath: Path = null + override val scheme: String = "hdfs" + + override def init(config: Config): Unit = { + rootPath = new Path(config.getString(Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH)) + val fs = rootPath.getFileSystem(new Configuration()) + if (!fs.exists(rootPath)) { + fs.mkdirs(rootPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + } + } + + /** + * Creates the file on JarStore. + * + * @param fileName name of the file to be created on JarStore. + * @return OutputStream returns a stream into which the data can be written. + */ + override def createFile(fileName: String): OutputStream = { + val filePath = new Path(rootPath, fileName) + val fs = filePath.getFileSystem(new Configuration()) + fs.create(filePath) + } + + /** + * Gets the InputStream to read the file + * + * @param fileName name of the file to be read on JarStore. + * @return InputStream returns a stream from which the data can be read. + */ + override def getFile(fileName: String): InputStream = { + val filePath = new Path(rootPath, fileName) + val fs = filePath.getFileSystem(new Configuration()) + fs.open(filePath) + } +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/project/Build.scala ---------------------------------------------------------------------- diff --git a/project/Build.scala b/project/Build.scala index 34f0ae2..4552a64 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -145,8 +145,7 @@ object Build extends sbt.Build { "com.typesafe.akka" %% "akka-cluster" % akkaVersion, "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion, "commons-logging" % "commons-logging" % commonsLoggingVersion, - "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion, - "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided" + "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion ) ) @@ -258,7 +257,7 @@ object Build extends sbt.Build { base = file("."), settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting) .aggregate(shaded, core, daemon, streaming, services, external_kafka, external_monoid, - external_serializer, examples, storm, yarn, external_hbase, packProject, + external_serializer, examples, storm, yarn, external_hbase, gearpumpHadoop, packProject, external_hadoopfs, integration_test).settings(Defaults.itSettings: _*) .disablePlugins(sbtassembly.AssemblyPlugin) @@ -463,14 +462,24 @@ object Build extends sbt.Build { .dependsOn (streaming % "test->test; compile->compile") .disablePlugins(sbtassembly.AssemblyPlugin) + lazy val gearpumpHadoop = Project( + id = "gearpump-hadoop", + base = file("gearpump-hadoop"), + settings = commonSettings ++ noPublish ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion, + "org.apache.hadoop" % "hadoop-common" % hadoopVersion + ) + ) + ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin) + lazy val yarn = Project( id = "gearpump-experiments-yarn", base = file("experiments/yarn"), settings = commonSettings ++ noPublish ++ Seq( libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion, - "org.apache.hadoop" % "hadoop-common" % hadoopVersion, "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion, "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion, "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion, @@ -480,8 +489,8 @@ object Build extends sbt.Build { "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided" ) )) - .dependsOn(services % "test->test;compile->compile", daemon % "provided", core % "provided") - .disablePlugins(sbtassembly.AssemblyPlugin) + .dependsOn(services % "test->test;compile->compile", daemon % "provided", + core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin) lazy val external_hbase = Project( id = "gearpump-external-hbase", http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/project/Pack.scala ---------------------------------------------------------------------- diff --git a/project/Pack.scala b/project/Pack.scala index 1c87653..13e53de 100644 --- a/project/Pack.scala +++ b/project/Pack.scala @@ -114,7 +114,8 @@ object Pack extends sbt.Build { packLibDir := Map( "lib" -> new ProjectsToPack(core.id, streaming.id), "lib/daemon" -> new ProjectsToPack(daemon.id, cgroup.id).exclude(core.id, streaming.id), - "lib/yarn" -> new ProjectsToPack(yarn.id).exclude(services.id, daemon.id), + "lib/yarn" -> new ProjectsToPack(gearpumpHadoop.id, yarn.id). + exclude(services.id, daemon.id, core.id), "lib/services" -> new ProjectsToPack(services.id).exclude(daemon.id), "lib/storm" -> new ProjectsToPack(storm.id).exclude(streaming.id) ),