Author: jghoman
Date: Fri Feb 10 00:13:04 2012
New Revision: 1242614
URL: http://svn.apache.org/viewvc?rev=1242614&view=rev
Log:
GIRAPH-142: _hadoopBsp should be prefixable via configuration.
Added:
incubator/giraph/trunk/src/test/java/zk/
incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java
Modified:
incubator/giraph/trunk/CHANGELOG
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
Modified: incubator/giraph/trunk/CHANGELOG
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Fri Feb 10 00:13:04 2012
@@ -2,6 +2,8 @@ Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-142: _hadoopBsp should be prefixable via configuration. (jghoman)
+
GIRAPH-145. Change partition request log level to debug rather than info.
(jghoman)
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
---
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
(original)
+++
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
Fri Feb 10 00:13:04 2012
@@ -23,6 +23,7 @@ import org.apache.giraph.graph.partition
import org.apache.giraph.zk.BspEvent;
import org.apache.giraph.zk.PredicateLock;
import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.giraph.zk.ZooKeeperManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -137,7 +138,7 @@ public abstract class BspService <
/** Unset application attempt */
public static final long UNSET_APPLICATION_ATTEMPT = Long.MIN_VALUE;
- public static final String BASE_DIR = "/_hadoopBsp";
+ private static final String BASE_DIR = "/_hadoopBsp";
public static final String MASTER_JOB_STATE_NODE = "/_masterJobState";
public static final String INPUT_SPLIT_DIR = "/_inputSplitDir";
public static final String INPUT_SPLIT_DONE_DIR = "/_inputSplitDoneDir";
@@ -629,7 +630,7 @@ public abstract class BspService <
conf.getInt(GiraphJob.CHECKPOINT_FREQUENCY,
GiraphJob.CHECKPOINT_FREQUENCY_DEFAULT);
- BASE_PATH = BASE_DIR + "/" + jobId;
+ BASE_PATH = ZooKeeperManager.getBasePath(conf) + BASE_DIR + "/" +
jobId;
MASTER_JOB_STATE_PATH = BASE_PATH + MASTER_JOB_STATE_NODE;
INPUT_SPLIT_PATH = BASE_PATH + INPUT_SPLIT_DIR;
INPUT_SPLITS_ALL_READY_PATH = BASE_PATH + INPUT_SPLITS_ALL_READY_NODE;
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
(original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
Fri Feb 10 00:13:04 2012
@@ -280,6 +280,12 @@ public class GiraphJob extends Job {
public static final String RESTART_SUPERSTEP = "giraph.restartSuperstep";
/**
+ * Base ZNode for Giraph's state in the ZooKeeper cluster. Must be a root
+ * znode on the cluster beginning with "/"
+ */
+ public static final String BASE_ZNODE_KEY = "giraph.zkBaseZNode";
+
+ /**
* If ZOOKEEPER_LIST is not set, then use this directory to manage
* ZooKeeper
*/
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
---
incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
(original)
+++
incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
Fri Feb 10 00:13:04 2012
@@ -18,6 +18,19 @@
package org.apache.giraph.zk;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.Closeables;
+import org.apache.commons.io.FileUtils;
+import org.apache.giraph.graph.GiraphJob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
@@ -35,18 +48,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Closeables;
-import org.apache.commons.io.FileUtils;
-import org.apache.giraph.graph.GiraphJob;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.log4j.Logger;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import static org.apache.giraph.graph.GiraphJob.BASE_ZNODE_KEY;
+
/**
* Manages the election of ZooKeeper servers, starting/stopping the services,
@@ -128,6 +131,24 @@ public class ZooKeeperManager {
return GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT + "/" + jobId;
}
+ /**
+ * Return the base ZooKeeper ZNode from which all other ZNodes Giraph creates
+ * should be sited, for instance in a multi-tenant ZooKeeper, the znode
+ * reserved for Giraph
+ *
+ * @param conf Necessary to access user-provided values
+ * @return String of path without trailing slash
+ */
+ public static String getBasePath(Configuration conf) {
+ String result = conf.get(BASE_ZNODE_KEY, "");
+ if(!result.equals("") && !result.startsWith("/")) {
+ throw new IllegalArgumentException("Value for " +
+ BASE_ZNODE_KEY + " must start with /: " + result);
+ }
+
+ return result;
+ }
+
/**
* Collects the output of a stream and dumps it to the log.
*/
Added: incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java?rev=1242614&view=auto
==============================================================================
--- incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java (added)
+++ incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java Fri Feb
10 00:13:04 2012
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package zk;
+
+import junit.framework.TestCase;
+import org.apache.giraph.graph.GiraphJob;
+import org.apache.giraph.zk.ZooKeeperManager;
+import org.apache.hadoop.conf.Configuration;
+
+public class TestZooKeeperManager extends TestCase {
+ public void testGetBasePath() {
+ Configuration conf = new Configuration();
+
+ // Default is empty, everything goes in root znode
+ assertEquals("Default value for base path should be empty",
+ "", ZooKeeperManager.getBasePath(conf));
+
+ conf.set(GiraphJob.BASE_ZNODE_KEY, "/howdy");
+ assertEquals("Base path should reflect value of " +
GiraphJob.BASE_ZNODE_KEY,
+ "/howdy", ZooKeeperManager.getBasePath(conf));
+
+ conf.set(GiraphJob.BASE_ZNODE_KEY, "no_slash");
+ try {
+ ZooKeeperManager.getBasePath(conf);
+ fail("Should not have allowed path without starting slash");
+ } catch (IllegalArgumentException iae) {
+ assertTrue(iae.getMessage().contains(GiraphJob.BASE_ZNODE_KEY));
+ }
+ }
+}
