Starting to build hive output format, serde, and storagehandler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3c41ef12 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3c41ef12 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3c41ef12 Branch: refs/heads/blur-384-random-port-cleanup Commit: 3c41ef1251e40a80c90f6e2197eda49592893ba9 Parents: 89e10a0 Author: Aaron McCurry <[email protected]> Authored: Thu Oct 2 18:19:52 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Oct 2 18:19:52 2014 -0400 ---------------------------------------------------------------------- contrib/blur-hive/pom.xml | 164 +++++++++++++++++++ .../apache/blur/hive/BlurHiveOutputFormat.java | 53 ++++++ .../blur/hive/BlurHiveStorageHandler.java | 50 ++++++ .../blur/hive/BlurObjectInspectorGenerator.java | 45 +++++ .../java/org/apache/blur/hive/BlurSerDe.java | 130 +++++++++++++++ .../org/apache/blur/hive/BlurSerializer.java | 35 ++++ .../apache/blur/hive/NullHiveInputFormat.java | 43 +++++ 7 files changed, 520 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/pom.xml b/contrib/blur-hive/pom.xml new file mode 100644 index 0000000..dc02843 --- /dev/null +++ b/contrib/blur-hive/pom.xml @@ -0,0 +1,164 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <groupId>org.apache.blur</groupId> + <artifactId>blur</artifactId> + <version>0.2.4-incubating-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.blur</groupId> + <artifactId>blur-hive</artifactId> + <version>${projectVersion}</version> + <packaging>jar</packaging> + <name>Blur Hive</name> + <description>The Blur hive module contains a storage handler and serde for Hive so that it can integrate with Hive.</description> + + <dependencies> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>0.13.1</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-thrift</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>javax.mail</groupId> + <artifactId>mail</artifactId> + </exclusion> + <exclusion> + <groupId>javax.jms</groupId> + <artifactId>jms</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jdmk</groupId> + <artifactId>jmxtools</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jmx</groupId> + <artifactId>jmxri</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>hadoop1</id> + <activation> + <property> + <name>hadoop1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-mapred-hadoop1</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + </profiles> + + +</project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java new file mode 100644 index 0000000..21a01b0 --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java @@ -0,0 +1,53 @@ +package org.apache.blur.hive; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.blur.mapreduce.lib.BlurRecord; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class BlurHiveOutputFormat implements HiveOutputFormat<Text, BlurRecord> { + + @Override + public void checkOutputSpecs(FileSystem arg0, JobConf arg1) throws IOException { + throw new RuntimeException("Not Implemented"); + } + + @Override + public RecordWriter<Text, BlurRecord> getRecordWriter(FileSystem arg0, JobConf arg1, String arg2, Progressable arg3) + throws IOException { + throw new RuntimeException("Not Implemented"); + } + + @Override + public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jc, + Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, + Progressable progress) throws IOException { + throw new RuntimeException("Not Implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveStorageHandler.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveStorageHandler.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveStorageHandler.java new file mode 100644 index 0000000..727276f --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveStorageHandler.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.blur.hive; + +import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; + +public class BlurHiveStorageHandler extends DefaultStorageHandler { + + @Override + public Class<? extends InputFormat> getInputFormatClass() { + return NullHiveInputFormat.class; + } + + @Override + public Class<? extends OutputFormat> getOutputFormatClass() { + return BlurHiveOutputFormat.class; + } + + @Override + public Class<? extends SerDe> getSerDeClass() { + return BlurSerDe.class; + } + + @Override + public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { + // Will set setup Table Descriptor and Output Committer. + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurObjectInspectorGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurObjectInspectorGenerator.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurObjectInspectorGenerator.java new file mode 100644 index 0000000..9fbea50 --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurObjectInspectorGenerator.java @@ -0,0 +1,45 @@ +package org.apache.blur.hive; + +import java.util.List; +import java.util.Map; + +import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class BlurObjectInspectorGenerator { + + public BlurObjectInspectorGenerator(Map<String, ColumnDefinition> schema) { + // TODO Auto-generated constructor stub + } + + public ObjectInspector getObjectInspector() { + throw new RuntimeException("Not Implemented"); + } + + public List<String> getColumnNames() { + throw new RuntimeException("Not Implemented"); + } + + public List<TypeInfo> getColumnTypes() { + throw new RuntimeException("Not Implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java new file mode 100644 index 0000000..5581a61 --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java @@ -0,0 +1,130 @@ +package org.apache.blur.hive; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.blur.BlurConfiguration; +import org.apache.blur.mapreduce.lib.BlurRecord; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.Schema; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.utils.BlurConstants; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.Writable; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +public class BlurSerDe extends AbstractSerDe { + + private static final String FAMILY = "blur.family"; + private static final String TABLE = "blur.table"; + private String _family; + private Map<String, ColumnDefinition> _schema; + private ObjectInspector _objectInspector; + private List<String> _columnNames; + private List<TypeInfo> _columnTypes; + private BlurSerializer _serializer; + + @Override + public void initialize(Configuration conf, Properties tbl) throws SerDeException { + String table = tbl.getProperty(TABLE); + _family = tbl.getProperty(FAMILY); + BlurConfiguration configuration; + try { + configuration = new BlurConfiguration(); + configuration.set(BlurConstants.BLUR_ZOOKEEPER_CONNECTION, + tbl.getProperty(BlurConstants.BLUR_ZOOKEEPER_CONNECTION)); + } catch (IOException e) { + throw new SerDeException(e); + } + + Iface client = BlurClient.getClient(configuration); + // TableDescriptor tableDescriptor; + Schema schema; + try { + List<String> tableList = client.tableList(); + if (!tableList.contains(table)) { + throw new SerDeException("Table [" + table + "] does not exist."); + } + // tableDescriptor = client.describe(table); + schema = client.schema(table); + } catch (BlurException e) { + throw new SerDeException(e); + } catch (TException e) { + throw new SerDeException(e); + } + + Map<String, ColumnDefinition> columns = schema.getFamilies().get(_family); + if (columns == null) { + throw new SerDeException("Family [" + _family + "] does not exist in table [" + table + "]"); + } + + _schema = new HashMap<String, ColumnDefinition>(); + for (ColumnDefinition columnDefinition : columns.values()) { + String subColumnName = columnDefinition.getSubColumnName(); + if (subColumnName == null) { + _schema.put(columnDefinition.getColumnName(), columnDefinition); + } + } + + BlurObjectInspectorGenerator blurObjectInspectorGenerator = new BlurObjectInspectorGenerator(_schema); + _objectInspector = blurObjectInspectorGenerator.getObjectInspector(); + _columnNames = blurObjectInspectorGenerator.getColumnNames(); + _columnTypes = blurObjectInspectorGenerator.getColumnTypes(); + + _serializer = new BlurSerializer(); + } + + @Override + public Writable serialize(Object o, ObjectInspector oi) throws SerDeException { + return _serializer.serialize(o, oi, _columnNames, _columnTypes, _schema, _family); + } + + @Override + public Object deserialize(Writable writable) throws SerDeException { + throw new RuntimeException("Not Implemented"); + } + + @Override + public ObjectInspector getObjectInspector() throws SerDeException { + return _objectInspector; + } + + @Override + public SerDeStats getSerDeStats() { + return null; + } + + @Override + public Class<? extends Writable> getSerializedClass() { + return BlurRecord.class; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java new file mode 100644 index 0000000..2a3199c --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurSerializer.java @@ -0,0 +1,35 @@ +package org.apache.blur.hive; + +import java.util.List; +import java.util.Map; + +import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.Writable; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +public class BlurSerializer { + + public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, + List<TypeInfo> columnTypes, Map<String, ColumnDefinition> schema, String family) { + throw new RuntimeException("Not Implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3c41ef12/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java ---------------------------------------------------------------------- diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java b/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java new file mode 100644 index 0000000..feb02e4 --- /dev/null +++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/NullHiveInputFormat.java @@ -0,0 +1,43 @@ +package org.apache.blur.hive; + +import java.io.IOException; + +import groovy.lang.Writable; + +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class NullHiveInputFormat implements InputFormat<Writable, Writable> { + + @Override + public RecordReader<Writable, Writable> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2) + throws IOException { + throw new RuntimeException("Not Implemented"); + } + + @Override + public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException { + throw new RuntimeException("Not Implemented"); + } + +}
