Apache-Phoenix | 3.0 | Hadoop1 | Build Successful
3.0 branch build status Successful Source repository https://git-wip-us.apache.org/repos/asf/phoenix.git Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastSuccessfulBuild/artifact/ Last Complete Test Report https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastCompletedBuild/testReport/ Changes [ravimagham] PHOENIX-1454 Map-Reduce-over-Phoenix-Tables
[2/3] phoenix git commit: PHOENIX-1454 Map-Reduce-over-Phoenix-Tables
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cabb16f7/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
--
diff --git
a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
index 5f6fcdd..5abc9c8 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.pig;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.List;
import java.util.Properties;
import org.apache.commons.cli.CommandLine;
@@ -28,13 +30,17 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.phoenix.pig.hadoop.PhoenixOutputFormat;
-import org.apache.phoenix.pig.hadoop.PhoenixRecord;
+import org.apache.phoenix.mapreduce.PhoenixOutputFormat;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import org.apache.phoenix.pig.writable.PhoenixPigDBWritable;
+import org.apache.phoenix.util.ColumnInfo;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.StoreFuncInterface;
@@ -71,74 +77,76 @@ import org.apache.pig.impl.util.UDFContext;
@SuppressWarnings("rawtypes")
public class PhoenixHBaseStorage implements StoreFuncInterface {
- private PhoenixPigConfiguration config;
- private RecordWriter writer;
- private String contextSignature = null;
- private ResourceSchema schema;
- private long batchSize;
- private final PhoenixOutputFormat outputFormat = new
PhoenixOutputFormat();
-
- // Set of options permitted
- private final static Options validOptions = new Options();
- private final static CommandLineParser parser = new GnuParser();
- private final static String SCHEMA = "_schema";
-
- private final CommandLine configuredOptions;
- private final String server;
-
- public PhoenixHBaseStorage(String server) throws ParseException {
- this(server, null);
- }
-
- public PhoenixHBaseStorage(String server, String optString)
- throws ParseException {
- populateValidOptions();
- this.server = server;
-
- String[] optsArr = optString == null ? new String[0] :
optString.split(" ");
- try {
- configuredOptions = parser.parse(validOptions, optsArr);
- } catch (ParseException e) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("[-batchSize]", validOptions);
- throw e;
- }
-
- batchSize =
Long.parseLong(configuredOptions.getOptionValue("batchSize"));
- }
-
- private static void populateValidOptions() {
- validOptions.addOption("batchSize", true, "Specify upsert batch
size");
- }
-
- /**
-* Returns UDFProperties based on contextSignature.
-*/
- private Properties getUDFProperties() {
- return
UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] {
contextSignature });
- }
-
-
- /**
-* Parse the HBase table name and configure job
-*/
- @Override
- public void setStoreLocation(String location, Job job) throws
IOException {
- URI locationURI;
+ private Configuration config;
+private RecordWriter writer;
+private List columnInfo = null;
+private String contextSignature = null;
+private ResourceSchema schema;
+private long batchSize;
+private final PhoenixOutputFormat outputFormat = new PhoenixOutputFormat();
+
+// Set of options permitted
+private final static Options validOptions = new Options();
+private final static CommandLineParser parser = new GnuParser();
+private final static String SCHEMA = "_schema";
+
+private final CommandLine configuredOptions;
+private final String server;
+
+public PhoenixHBaseStorage(String server) throws ParseException {
+this(server, null);
+}
+
+public PhoenixHBaseStorage(String server, String optString)
+throws ParseException {
+populateValidOptions();
+this.server = server;
+
+
[1/3] phoenix git commit: PHOENIX-1454 Map-Reduce-over-Phoenix-Tables
Repository: phoenix
Updated Branches:
refs/heads/3.0 d6f70d25a -> cabb16f7d
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cabb16f7/phoenix-pig/src/test/java/org/apache/phoenix/pig/PhoenixPigConfigurationTest.java
--
diff --git
a/phoenix-pig/src/test/java/org/apache/phoenix/pig/PhoenixPigConfigurationTest.java
b/phoenix-pig/src/test/java/org/apache/phoenix/pig/PhoenixPigConfigurationTest.java
deleted file mode 100644
index 0337563..000
---
a/phoenix-pig/src/test/java/org/apache/phoenix/pig/PhoenixPigConfigurationTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- *distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you maynot use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicablelaw or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.pig;
-
-import static org.junit.Assert.assertEquals;
-
-import java.sql.SQLException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-
-/**
- * Tests for PhoenixPigConfiguration.
- *
- */
-public class PhoenixPigConfigurationTest {
-
-
-@Test
-public void testBasicConfiguration() throws SQLException {
-Configuration conf = new Configuration();
-final PhoenixPigConfiguration phoenixConfiguration = new
PhoenixPigConfiguration(conf);
-final String zkQuorum = "localhost";
-final String tableName = "TABLE";
-final long batchSize = 100;
-phoenixConfiguration.configure(zkQuorum, tableName, batchSize);
-assertEquals(zkQuorum,phoenixConfiguration.getServer());
-assertEquals(tableName,phoenixConfiguration.getTableName());
-assertEquals(batchSize,phoenixConfiguration.getBatchSize());
- }
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cabb16f7/phoenix-pig/src/test/java/org/apache/phoenix/pig/util/ColumnInfoToStringEncoderDecoderTest.java
--
diff --git
a/phoenix-pig/src/test/java/org/apache/phoenix/pig/util/ColumnInfoToStringEncoderDecoderTest.java
b/phoenix-pig/src/test/java/org/apache/phoenix/pig/util/ColumnInfoToStringEncoderDecoderTest.java
deleted file mode 100644
index 9777bb5..000
---
a/phoenix-pig/src/test/java/org/apache/phoenix/pig/util/ColumnInfoToStringEncoderDecoderTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- *distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you maynot use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicablelaw or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.pig.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-
-import org.apache.phoenix.pig.util.ColumnInfoToStringEncoderDecoder;
-import org.apache.phoenix.schema.PDataType;
-import org.apache.phoenix.util.ColumnInfo;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- * Tests methods on {@link ColumnInfoToStringEncoderDecoder}
- */
-public class ColumnInfoToStringEncoderDecoderTest {
-
-@Test
-public void testEncode() {
-final ColumnInfo columnInfo = new ColumnInfo("col1",
PDataType.VARCHAR.getSqlType());
-final String encodedColumnInfo =
ColumnInfoToStringEncoderDecoder.encode(Lists.newArrayList(columnInfo));
-assertEquals(columnInfo.toString(),encodedColumnInfo);
-}
-
-@Test
-public void testDecode() {
-final ColumnInfo columnInfo = new ColumnInfo("col1",
PDataType.VARCHAR.getSqlType());
-final String encodedColumnInfo =
ColumnInfoToStringEncoderDecoder.encode(Lists.newArrayList(columnInfo));
-
[3/3] phoenix git commit: PHOENIX-1454 Map-Reduce-over-Phoenix-Tables
PHOENIX-1454 Map-Reduce-over-Phoenix-Tables Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cabb16f7 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cabb16f7 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cabb16f7 Branch: refs/heads/3.0 Commit: cabb16f7d08a4d9f5e0ae5027b1f9810edd69c1d Parents: d6f70d2 Author: ravimagham Authored: Tue Dec 9 16:51:22 2014 -0800 Committer: ravimagham Committed: Tue Dec 9 16:51:22 2014 -0800 -- .../phoenix/mapreduce/PhoenixInputFormat.java | 117 +++ .../phoenix/mapreduce/PhoenixInputSplit.java| 122 +++ .../mapreduce/PhoenixOutputCommitter.java | 54 +++ .../phoenix/mapreduce/PhoenixOutputFormat.java | 62 .../phoenix/mapreduce/PhoenixRecordReader.java | 140 .../phoenix/mapreduce/PhoenixRecordWriter.java | 91 + .../util/ColumnInfoToStringEncoderDecoder.java | 65 .../phoenix/mapreduce/util/ConnectionUtil.java | 49 +++ .../util/PhoenixConfigurationUtil.java | 299 .../mapreduce/util/PhoenixMapReduceUtil.java| 99 ++ .../org/apache/phoenix/util/PhoenixRuntime.java | 4 +- .../java/org/apache/phoenix/util/QueryUtil.java | 26 +- .../ColumnInfoToStringEncoderDecoderTest.java | 61 .../util/PhoenixConfigurationUtilTest.java | 124 +++ .../org/apache/phoenix/util/QueryUtilTest.java | 2 +- .../phoenix/pig/PhoenixPigConfigurationIT.java | 109 -- .../apache/phoenix/pig/PhoenixHBaseLoader.java | 45 +-- .../apache/phoenix/pig/PhoenixHBaseStorage.java | 218 ++-- .../phoenix/pig/PhoenixPigConfiguration.java| 340 --- .../phoenix/pig/hadoop/PhoenixInputFormat.java | 142 .../phoenix/pig/hadoop/PhoenixInputSplit.java | 127 --- .../pig/hadoop/PhoenixOutputCommitter.java | 111 -- .../phoenix/pig/hadoop/PhoenixOutputFormat.java | 94 - .../phoenix/pig/hadoop/PhoenixRecord.java | 112 -- .../phoenix/pig/hadoop/PhoenixRecordReader.java | 142 .../phoenix/pig/hadoop/PhoenixRecordWriter.java | 83 - .../util/ColumnInfoToStringEncoderDecoder.java | 69 .../phoenix/pig/util/PhoenixPigSchemaUtil.java | 29 +- .../pig/util/QuerySchemaParserFunction.java | 21 +- .../pig/util/SqlQueryToColumnInfoFunction.java | 49 ++- .../org/apache/phoenix/pig/util/TypeUtil.java | 248 +++--- .../pig/writable/PhoenixPigDBWritable.java | 121 +++ .../pig/PhoenixPigConfigurationTest.java| 49 --- .../ColumnInfoToStringEncoderDecoderTest.java | 61 .../pig/util/PhoenixPigSchemaUtilTest.java | 17 +- .../pig/util/QuerySchemaParserFunctionTest.java | 22 +- .../util/SqlQueryToColumnInfoFunctionTest.java | 21 +- 37 files changed, 1759 insertions(+), 1786 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/cabb16f7/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java new file mode 100644 index 000..7c67c2c --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.mapreduce; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apach
