Repository: incubator-blur Updated Branches: refs/heads/master b72ea64d5 -> 3c164d62f
Fixing broken build. Needed to fix imports and statis util class. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3c164d62 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3c164d62 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3c164d62 Branch: refs/heads/master Commit: 3c164d62f4a9a5b5762d8cf9e32af74fc0e04fdd Parents: b72ea64 Author: Aaron McCurry <[email protected]> Authored: Mon Dec 22 07:48:45 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Dec 22 07:48:45 2014 -0500 ---------------------------------------------------------------------- .../blur/mapreduce/lib/BlurOutputCommitter.java | 7 +++---- .../blur/mapreduce/lib/GenericBlurRecordWriter.java | 4 ++-- .../lib/BlurOutputFormatMiniClusterTest.java | 8 +++++--- .../blur/mapreduce/lib/BlurOutputFormatTest.java | 14 +++++++------- .../blur/mapreduce/lib/BlurOutputCommitter.java | 6 +++--- .../blur/mapreduce/lib/GenericBlurRecordWriter.java | 4 ++-- .../lib/BlurOutputFormatMiniClusterTest.java | 4 ++-- .../blur/mapreduce/lib/BlurOutputFormatTest.java | 14 +++++++------- 8 files changed, 31 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java index b5902bf..4e44461 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java @@ -20,10 +20,9 @@ import java.io.IOException; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.mapreduce.lib.BlurOutputFormat; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -86,7 +85,7 @@ public class BlurOutputCommitter extends OutputCommitter { int shardId = attemptId % shardCount; _taskAttemptID = context.getTaskAttemptID(); Path tableOutput = BlurOutputFormat.getOutputPath(_configuration); - String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); + String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); _indexPath = new Path(tableOutput, shardName); _newIndex = new Path(_indexPath, _taskAttemptID.toString() + ".tmp"); } @@ -185,7 +184,7 @@ public class BlurOutputCommitter extends OutputCommitter { TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration); int shardCount = tableDescriptor.getShardCount(); for (int i = 0; i < shardCount; i++) { - String shardName = BlurUtil.getShardName(i); + String shardName = ShardUtil.getShardName(i); fileSystem.mkdirs(new Path(tableOutput, shardName)); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index 8b90331..fcbe98e 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -34,8 +34,8 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.RowDocumentUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -107,7 +107,7 @@ public class GenericBlurRecordWriter { int shardId = attemptId % shardCount; Path tableOutput = BlurOutputFormat.getOutputPath(_configuration); - String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); + String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); Path indexPath = new Path(tableOutput, shardName); _newIndex = new Path(indexPath, tmpDirName); _finalDir = new ProgressableDirectory(new HdfsDirectory(_configuration, _newIndex), getProgressable()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java index f2a53d7..dc30ca4 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java @@ -16,7 +16,9 @@ package org.apache.blur.mapreduce.lib; * See the License for the specific language governing permissions and * limitations under the License. */ -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.DataInputStream; @@ -35,8 +37,8 @@ import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; -import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.GCWatcher; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -162,7 +164,7 @@ public class BlurOutputFormatMiniClusterTest { BlurOutputFormat.setOutputPath(job, output); Path tablePath = new Path(tableUri); - Path shardPath = new Path(tablePath, BlurUtil.getShardName(0)); + Path shardPath = new Path(tablePath, ShardUtil.getShardName(0)); FileStatus[] listStatus = fileSystem.listStatus(shardPath); assertEquals(3, listStatus.length); System.out.println("======" + listStatus.length); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index a2937ec..03364b1 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -36,7 +36,7 @@ import org.apache.blur.server.TableContext; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -142,7 +142,7 @@ public class BlurOutputFormatTest { Counters ctrs = job.getCounters(); System.out.println("Counters: " + ctrs); - Path path = new Path(output, BlurUtil.getShardName(0)); + Path path = new Path(output, ShardUtil.getShardName(0)); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next())); @@ -194,7 +194,7 @@ public class BlurOutputFormatTest { Counters ctrs = job.getCounters(); System.out.println("Counters: " + ctrs); - Path path = new Path(output, BlurUtil.getShardName(0)); + Path path = new Path(output, ShardUtil.getShardName(0)); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); @@ -267,7 +267,7 @@ public class BlurOutputFormatTest { long total = 0; for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(output, BlurUtil.getShardName(i)); + Path path = new Path(output, ShardUtil.getShardName(i)); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); @@ -313,7 +313,7 @@ public class BlurOutputFormatTest { long total = 0; for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(output, BlurUtil.getShardName(i)); + Path path = new Path(output, ShardUtil.getShardName(i)); Collection<Path> commitedTasks = getCommitedTasks(path); assertTrue(multiple >= commitedTasks.size()); for (Path p : commitedTasks) { @@ -399,7 +399,7 @@ public class BlurOutputFormatTest { assertFalse(job.isSuccessful()); for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(tableUri, BlurUtil.getShardName(i)); + Path path = new Path(tableUri, ShardUtil.getShardName(i)); FileSystem fileSystem = path.getFileSystem(job.getConfiguration()); FileStatus[] listStatus = fileSystem.listStatus(path); assertEquals(toString(listStatus), 0, listStatus.length); @@ -453,7 +453,7 @@ public class BlurOutputFormatTest { private void createShardDirectories(Path outDir, int shardCount) throws IOException { localFs.mkdirs(outDir); for (int i = 0; i < shardCount; i++) { - localFs.mkdirs(new Path(outDir, BlurUtil.getShardName(i))); + localFs.mkdirs(new Path(outDir, ShardUtil.getShardName(i))); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java index ab866fd..2db0483 100644 --- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java +++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputCommitter.java @@ -22,7 +22,7 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -69,7 +69,7 @@ public class BlurOutputCommitter extends OutputCommitter { TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration); int shardCount = tableDescriptor.getShardCount(); for (int i = 0; i < shardCount; i++) { - String shardName = BlurUtil.getShardName(i); + String shardName = ShardUtil.getShardName(i); fileSystem.mkdirs(new Path(tableOutput, shardName)); } } @@ -212,7 +212,7 @@ public class BlurOutputCommitter extends OutputCommitter { int shardId = attemptId % shardCount; conf._taskAttemptID = context.getTaskAttemptID(); Path tableOutput = BlurOutputFormat.getOutputPath(conf._configuration); - String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); + String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); conf._indexPath = new Path(tableOutput, shardName); conf._newIndex = new Path(conf._indexPath, conf._taskAttemptID.toString() + ".tmp"); return conf; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index 1077440..1211988 100644 --- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -34,8 +34,8 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.RowDocumentUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -105,7 +105,7 @@ public class GenericBlurRecordWriter { int shardId = attemptId % shardCount; Path tableOutput = BlurOutputFormat.getOutputPath(_configuration); - String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); + String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId); Path indexPath = new Path(tableOutput, shardName); _newIndex = new Path(indexPath, tmpDirName); _finalDir = new ProgressableDirectory(new HdfsDirectory(_configuration, _newIndex), getProgressable()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java index 14daa27..43ff57f 100644 --- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java +++ b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java @@ -37,8 +37,8 @@ import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; -import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.GCWatcher; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -169,7 +169,7 @@ public class BlurOutputFormatMiniClusterTest { BlurOutputFormat.setOutputPath(job, output); Path tablePath = new Path(tableUri); - Path shardPath = new Path(tablePath, BlurUtil.getShardName(0)); + Path shardPath = new Path(tablePath, ShardUtil.getShardName(0)); FileStatus[] listStatus = fileSystem.listStatus(shardPath); assertEquals(3, listStatus.length); System.out.println("======" + listStatus.length); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c164d62/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index fbcee34..976a096 100644 --- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -35,7 +35,7 @@ import org.apache.blur.server.TableContext; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -150,7 +150,7 @@ public class BlurOutputFormatTest { Counters ctrs = job.getCounters(); System.out.println("Counters: " + ctrs); - Path path = new Path(output, BlurUtil.getShardName(0)); + Path path = new Path(output, ShardUtil.getShardName(0)); dump(path, conf); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); @@ -216,7 +216,7 @@ public class BlurOutputFormatTest { Counters ctrs = job.getCounters(); System.out.println("Counters: " + ctrs); - Path path = new Path(output, BlurUtil.getShardName(0)); + Path path = new Path(output, ShardUtil.getShardName(0)); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); @@ -262,7 +262,7 @@ public class BlurOutputFormatTest { long total = 0; for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(output, BlurUtil.getShardName(i)); + Path path = new Path(output, ShardUtil.getShardName(i)); Collection<Path> commitedTasks = getCommitedTasks(path); assertEquals(1, commitedTasks.size()); @@ -310,7 +310,7 @@ public class BlurOutputFormatTest { long total = 0; for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(output, BlurUtil.getShardName(i)); + Path path = new Path(output, ShardUtil.getShardName(i)); Collection<Path> commitedTasks = getCommitedTasks(path); assertTrue(multiple >= commitedTasks.size()); for (Path p : commitedTasks) { @@ -401,7 +401,7 @@ public class BlurOutputFormatTest { assertFalse(job.isSuccessful()); for (int i = 0; i < tableDescriptor.getShardCount(); i++) { - Path path = new Path(output, BlurUtil.getShardName(i)); + Path path = new Path(output, ShardUtil.getShardName(i)); FileSystem fileSystem = path.getFileSystem(job.getConfiguration()); FileStatus[] listStatus = fileSystem.listStatus(path); assertEquals(toString(listStatus), 0, listStatus.length); @@ -455,7 +455,7 @@ public class BlurOutputFormatTest { private void createShardDirectories(Path outDir, int shardCount) throws IOException { localFs.mkdirs(outDir); for (int i = 0; i < shardCount; i++) { - localFs.mkdirs(new Path(outDir, BlurUtil.getShardName(i))); + localFs.mkdirs(new Path(outDir, ShardUtil.getShardName(i))); } }
