Repository: incubator-blur Updated Branches: refs/heads/master 72799d839 -> 4784a6faf
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java deleted file mode 100644 index b35feec..0000000 --- a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.blur.hive; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.blur.BlurConfiguration; -import org.apache.blur.mapreduce.lib.BlurOutputFormat; -import org.apache.blur.mapreduce.lib.BlurRecord; -import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.BlurClient; -import org.apache.blur.thrift.generated.Blur.Iface; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.ColumnDefinition; -import org.apache.blur.thrift.generated.Schema; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.utils.BlurConstants; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.SerDeStats; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.io.Writable; - -public class BlurSerDe extends AbstractSerDe { - - public static final String BLUR_CONTROLLER_CONNECTION_STR = "BLUR_CONTROLLER_CONNECTION_STR"; - private static final String FAMILY = "blur.family"; - private static final String TABLE = "blur.table"; - private String _family; - private Map<String, ColumnDefinition> _schema; - private ObjectInspector _objectInspector; - private List<String> _columnNames; - private List<TypeInfo> _columnTypes; - private BlurSerializer _serializer; - - @Override - public void initialize(Configuration conf, Properties tbl) throws SerDeException { - String table = tbl.getProperty(TABLE); - _family = tbl.getProperty(FAMILY); - BlurConfiguration configuration; - try { - configuration = new BlurConfiguration(); - configuration.set(BlurConstants.BLUR_ZOOKEEPER_CONNECTION, - tbl.getProperty(BlurConstants.BLUR_ZOOKEEPER_CONNECTION)); - } catch (IOException e) { - throw new SerDeException(e); - } - - Iface client = BlurClient.getClient(configuration); - Schema schema; - try { - List<String> tableList = client.tableList(); - if (!tableList.contains(table)) { - throw new SerDeException("Table [" + table + "] does not exist."); - } - if (conf != null) { - TableDescriptor tableDescriptor = client.describe(table); - BlurOutputFormat.setTableDescriptor(conf, tableDescriptor); - conf.set(BLUR_CONTROLLER_CONNECTION_STR, getControllerConnectionStr(client)); - } - schema = client.schema(table); - } catch (BlurException e) { - throw new SerDeException(e); - } catch (TException e) { - throw new SerDeException(e); - } catch (IOException e) { - throw new SerDeException(e); - } - - Map<String, ColumnDefinition> columns = schema.getFamilies().get(_family); - if (columns == null) { - throw new SerDeException("Family [" + _family + "] does not exist in table [" + table + "]"); - } - - _schema = new HashMap<String, ColumnDefinition>(); - for (ColumnDefinition columnDefinition : columns.values()) { - String subColumnName = columnDefinition.getSubColumnName(); - if (subColumnName == null) { - _schema.put(columnDefinition.getColumnName(), columnDefinition); - } - } - - BlurObjectInspectorGenerator blurObjectInspectorGenerator = new BlurObjectInspectorGenerator(_schema.values()); - _objectInspector = blurObjectInspectorGenerator.getObjectInspector(); - _columnNames = blurObjectInspectorGenerator.getColumnNames(); - _columnTypes = blurObjectInspectorGenerator.getColumnTypes(); - - _serializer = new BlurSerializer(); - } - - private String getControllerConnectionStr(Iface client) throws BlurException, TException { - List<String> controllerServerList = client.controllerServerList(); - StringBuilder builder = new StringBuilder(); - for (String c : controllerServerList) { - if (builder.length() != 0) { - builder.append(','); - } - builder.append(c); - } - return builder.toString(); - } - - @Override - public Writable serialize(Object o, ObjectInspector oi) throws SerDeException { - return _serializer.serialize(o, oi, _columnNames, _columnTypes, _schema, _family); - } - - @Override - public Object deserialize(Writable writable) throws SerDeException { - throw new RuntimeException("Not Implemented"); - } - - @Override - public ObjectInspector getObjectInspector() throws SerDeException { - return _objectInspector; - } - - @Override - public SerDeStats getSerDeStats() { - return null; - } - - @Override - public Class<? extends Writable> getSerializedClass() { - return BlurRecord.class; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java deleted file mode 100644 index b65698f..0000000 --- a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.blur.hive; - -import java.util.List; -import java.util.Map; - -import org.apache.blur.mapreduce.lib.BlurRecord; -import org.apache.blur.thrift.generated.ColumnDefinition; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.lazy.LazyBoolean; -import org.apache.hadoop.hive.serde2.lazy.LazyByte; -import org.apache.hadoop.hive.serde2.lazy.LazyDate; -import org.apache.hadoop.hive.serde2.lazy.LazyDouble; -import org.apache.hadoop.hive.serde2.lazy.LazyFloat; -import org.apache.hadoop.hive.serde2.lazy.LazyHiveChar; -import org.apache.hadoop.hive.serde2.lazy.LazyHiveDecimal; -import org.apache.hadoop.hive.serde2.lazy.LazyHiveVarchar; -import org.apache.hadoop.hive.serde2.lazy.LazyInteger; -import org.apache.hadoop.hive.serde2.lazy.LazyLong; -import org.apache.hadoop.hive.serde2.lazy.LazyShort; -import org.apache.hadoop.hive.serde2.lazy.LazyString; -import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp; -import org.apache.hadoop.hive.serde2.lazy.LazyVoid; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.io.BooleanWritable; -import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; - -public class BlurSerializer { - - public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, - List<TypeInfo> columnTypes, Map<String, ColumnDefinition> schema, String family) throws SerDeException { - BlurRecord blurRecord = new BlurRecord(); - blurRecord.setFamily(family); - - StructObjectInspector soi = (StructObjectInspector) objectInspector; - - List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); - int size = columnNames.size(); - if (outputFieldRefs.size() != size) { - throw new SerDeException("Number of input columns was different than output columns (in = " + size + " vs out = " - + outputFieldRefs.size()); - } - List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); - for (int i = 0; i < size; i++) { - // StructField structFieldRef = outputFieldRefs.get(i); - Object structFieldData = structFieldsDataAsList.get(i); - if (structFieldData == null) { - continue; - } - // ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); - String columnName = columnNames.get(i); - String stringValue = toString(structFieldData); - if (stringValue == null) { - continue; - } - if (columnName.equals(BlurObjectInspectorGenerator.ROWID)) { - blurRecord.setRowId(stringValue); - } else if (columnName.equals(BlurObjectInspectorGenerator.RECORDID)) { - blurRecord.setRecordId(stringValue); - } else { - if (columnName.equals(BlurObjectInspectorGenerator.GEO_POINTVECTOR) - || columnName.equals(BlurObjectInspectorGenerator.GEO_RECURSIVEPREFIX) - || columnName.equals(BlurObjectInspectorGenerator.GEO_TERMPREFIX)) { - throw new SerDeException("Not supported yet."); - } else { - blurRecord.addColumn(columnName, stringValue); - } - } - } - return blurRecord; - } - - private String toString(Object o) { - if (o == null) { - return null; - } - if (o instanceof LazyBoolean) { - return lazyBoolean((LazyBoolean) o); - } else if (o instanceof LazyByte) { - return lazyByte((LazyByte) o); - } else if (o instanceof LazyDate) { - return lazyDate((LazyDate) o); - } else if (o instanceof LazyDouble) { - return lazyDouble((LazyDouble) o); - } else if (o instanceof LazyFloat) { - return lazyFloat((LazyFloat) o); - } else if (o instanceof LazyHiveChar) { - return lazyHiveChar((LazyHiveChar) o); - } else if (o instanceof LazyHiveDecimal) { - return lazyHiveDecimal((LazyHiveDecimal) o); - } else if (o instanceof LazyHiveVarchar) { - return lazyHiveVarchar((LazyHiveVarchar) o); - } else if (o instanceof LazyInteger) { - return lazyInteger((LazyInteger) o); - } else if (o instanceof LazyLong) { - return lazyLong((LazyLong) o); - } else if (o instanceof LazyShort) { - return lazyShort((LazyShort) o); - } else if (o instanceof LazyShort) { - return lazyString((LazyString) o); - } else if (o instanceof LazyTimestamp) { - return lazyTimestamp((LazyTimestamp) o); - } else if (o instanceof LazyVoid) { - return null; - } - return o.toString(); - } - - private String lazyInteger(LazyInteger o) { - IntWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - int i = writableObject.get(); - return Integer.toString(i); - } - - private String lazyLong(LazyLong o) { - LongWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - long l = writableObject.get(); - return Long.toString(l); - } - - private String lazyShort(LazyShort o) { - ShortWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - short s = writableObject.get(); - return Short.toString(s); - } - - private String lazyString(LazyString o) { - Text writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - return writableObject.toString(); - } - - private String lazyTimestamp(LazyTimestamp o) { - throw new RuntimeException("Not implemented."); - } - - private String lazyHiveVarchar(LazyHiveVarchar o) { - throw new RuntimeException("Not implemented."); - } - - private String lazyHiveDecimal(LazyHiveDecimal o) { - throw new RuntimeException("Not implemented."); - } - - private String lazyHiveChar(LazyHiveChar o) { - throw new RuntimeException("Not implemented."); - } - - private String lazyFloat(LazyFloat o) { - FloatWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - float f = writableObject.get(); - return Float.toString(f); - } - - private String lazyDouble(LazyDouble o) { - DoubleWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - double d = writableObject.get(); - return Double.toString(d); - } - - private String lazyDate(LazyDate o) { - throw new RuntimeException("Not implemented."); - } - - private String lazyByte(LazyByte o) { - ByteWritable writableObject = o.getWritableObject(); - if (writableObject == null) { - return null; - } - byte b = writableObject.get(); - return Integer.toString(b); - } - - private String lazyBoolean(LazyBoolean lazyBoolean) { - BooleanWritable writableObject = lazyBoolean.getWritableObject(); - if (writableObject == null) { - return null; - } - return Boolean.toString(writableObject.get()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java deleted file mode 100644 index 62eb656..0000000 --- a/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.blur.hive; - -import java.io.IOException; - -import groovy.lang.Writable; - -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; - -public class NullHiveInputFormat implements InputFormat<Writable, Writable> { - - @Override - public RecordReader<Writable, Writable> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2) - throws IOException { - return new RecordReader<Writable, Writable>() { - - @Override - public void close() throws IOException { - - } - - @Override - public Writable createKey() { - return null; - } - - @Override - public Writable createValue() { - return null; - } - - @Override - public long getPos() throws IOException { - return 0l; - } - - @Override - public float getProgress() throws IOException { - return 0.0f; - } - - @Override - public boolean next(Writable key, Writable value) throws IOException { - return false; - } - - }; - } - - @Override - public InputSplit[] getSplits(JobConf jobConf, int state) throws IOException { - return new InputSplit[] {}; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/test/java/org/apache/blur/hive/CreateData.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/test/java/org/apache/blur/hive/CreateData.java b/contrib/blur-hive/src/test/java/org/apache/blur/hive/CreateData.java deleted file mode 100644 index 63bf568..0000000 --- a/contrib/blur-hive/src/test/java/org/apache/blur/hive/CreateData.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.hive; - -import java.io.IOException; -import java.io.PrintWriter; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -public class CreateData { - - private static final String SEP = new String(new char[] { 1 }); - - public static void main(String[] args) throws IOException { - Path path = new Path("hdfs://localhost:9000/user/hive/warehouse/test.db/input_data/data"); - Configuration configuration = new Configuration(); - FileSystem fileSystem = path.getFileSystem(configuration); - FSDataOutputStream outputStream = fileSystem.create(path); - PrintWriter print = new PrintWriter(outputStream); - int rows = 100000; - for (int i = 0; i < rows; i++) { - String s = Integer.toString(i); - print.print(s); - print.print(SEP); - print.print(s + "-" + System.currentTimeMillis()); - for (int c = 0; c < 10; c++) { - print.print(SEP); - print.print(s); - } - print.println(); - } - print.close(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/test/java/org/apache/blur/hive/RunHiveTest.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/test/java/org/apache/blur/hive/RunHiveTest.java b/contrib/blur-hive/src/test/java/org/apache/blur/hive/RunHiveTest.java deleted file mode 100644 index 3451c1c..0000000 --- a/contrib/blur-hive/src/test/java/org/apache/blur/hive/RunHiveTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.hive; - -import org.apache.hadoop.hive.cli.CliDriver; - -public class RunHiveTest { - - public static void main(String[] args) throws Exception { - CliDriver.main(new String[] { "-f", "./src/test/java/test.hive" }); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/contrib/blur-hive/src/test/java/test.hive ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/test/java/test.hive b/contrib/blur-hive/src/test/java/test.hive deleted file mode 100644 index 5f950a8..0000000 --- a/contrib/blur-hive/src/test/java/test.hive +++ /dev/null @@ -1,55 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -set mapred.job.tracker=localhost:9001; -set hive.metastore.warehouse.dir=hdfs://localhost:9000/user/hive/warehouse; - -add jar file:///Users/amccurry/Development/incubator-blur/contrib/blur-hive/target/blur-hive-0.2.4-incubating-SNAPSHOT-hadoop1.jar; - -create database if not exists test; -use test; - -CREATE TABLE if not exists test -ROW FORMAT SERDE 'org.apache.blur.hive.BlurSerDe' -WITH SERDEPROPERTIES ( - 'blur.zookeeper.connection'='localhost', - 'blur.table'='test_hdfs', - 'blur.family'='fam0' -) -STORED BY 'org.apache.blur.hive.BlurHiveStorageHandler'; - -desc test; - -create table if not exists input_data ( - rowid string, - recordid string, - col0 string, - col1 string, - col2 string, - col3 string, - col4 string, - col5 string, - col6 string, - col7 string, - col8 string, - col9 string -); - -select * from input_data; - -insert overwrite table test select * from input_data; - - - http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4784a6fa/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8443a5b..127b320 100644 --- a/pom.xml +++ b/pom.xml @@ -172,8 +172,8 @@ under the License. </scm> <properties> - <commons-logging.version>1.1.3</commons-logging.version> - <commons-lang.version>2.4</commons-lang.version> + <commons-logging.version>1.1.3</commons-logging.version> + <commons-lang.version>2.4</commons-lang.version> <zookeeper.version>3.4.5</zookeeper.version> <log4j.version>1.2.15</log4j.version> <jersey.version>1.14</jersey.version> @@ -192,6 +192,7 @@ under the License. <mrunit.version>0.9.0-incubating</mrunit.version> <httpclient.version>4.1.3</httpclient.version> <servlet-api.version>3.0.1</servlet-api.version> + <hive.version>0.13.1</hive.version> </properties> <dependencyManagement> @@ -482,6 +483,7 @@ under the License. <module>blur-command</module> <module>blur-console</module> <module>blur-core</module> + <module>blur-hive</module> <module>blur-thrift</module> <module>blur-query</module> <module>blur-store</module> @@ -507,6 +509,7 @@ under the License. <module>blur-command</module> <module>blur-console</module> <module>blur-core</module> + <module>blur-hive</module> <module>blur-thrift</module> <module>blur-query</module> <module>blur-store</module> @@ -532,6 +535,7 @@ under the License. <module>blur-command</module> <module>blur-console</module> <module>blur-core</module> + <module>blur-hive</module> <module>blur-thrift</module> <module>blur-query</module> <module>blur-store</module>
