TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, Committed by hyunsik)
Closes #470 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/facbf432 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/facbf432 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/facbf432 Branch: refs/heads/branch-0.10.1 Commit: facbf43235e07426696393e955ab6b3fd610d4ba Parents: 556498f Author: Hyunsik Choi <[email protected]> Authored: Fri May 8 00:52:09 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Fri May 8 01:15:10 2015 -0700 ---------------------------------------------------------------------- CHANGES | 8 +- .../apache/tajo/function/FunctionSignature.java | 10 +- .../org/apache/tajo/function/FunctionUtil.java | 41 + .../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 739 --------------- .../tajo/catalog/store/HCatalogStore.java | 891 ------------------- .../catalog/store/HCatalogStoreClientPool.java | 170 ---- .../apache/tajo/catalog/store/HCatalogUtil.java | 147 --- .../tajo/catalog/store/TestHCatalogStore.java | 402 --------- .../cli/tsql/commands/DescFunctionCommand.java | 18 +- .../main/java/org/apache/tajo/util/JSPUtil.java | 14 - .../main/resources/webapps/admin/functions.jsp | 3 +- 11 files changed, 59 insertions(+), 2384 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 47af73b..05ea1f2 100644 --- a/CHANGES +++ b/CHANGES @@ -9,9 +9,11 @@ Release 0.10.1 - unreleased IMPROVEMENT - TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates - an important kind of information. - (Contributed by Jongyoung Park, Committed by jihoon) + TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, + Committed by hyunsik) + + TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates + an important kind of information. (Contributed by Jongyoung Park, Committed by jihoon) TAJO-1381: Support multi-bytes delimiter for Text file. (Contributed by navis, Committed by jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java index fc3a056..89ee017 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java @@ -122,14 +122,20 @@ public class FunctionSignature implements Comparable<FunctionSignature>, ProtoOb return cmpVal; } - cmpVal = returnType.getType().compareTo(o.returnType.getType()); + cmpVal = functionType.name().compareTo(o.functionType.name()); + + if (cmpVal != 0) { + return cmpVal; + } + + cmpVal = returnType.getType().name().compareTo(o.returnType.getType().name()); if (cmpVal != 0) { return cmpVal; } for (int i = 0; i < Math.min(paramTypes.length, o.paramTypes.length); i++) { - cmpVal = paramTypes[i].getType().compareTo(o.paramTypes[i].getType()); + cmpVal = paramTypes[i].getType().name().compareTo(o.paramTypes[i].getType().name()); if (cmpVal != 0) { return cmpVal; http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java index dee5d1c..7011957 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java @@ -18,12 +18,53 @@ package org.apache.tajo.function; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; + import java.util.Collection; +import java.util.Comparator; import static org.apache.tajo.common.TajoDataTypes.DataType; public class FunctionUtil { + public static class FunctionDescProtoComparator implements Comparator<CatalogProtos.FunctionDescProto> + { + @Override + public int compare(CatalogProtos.FunctionDescProto f1, CatalogProtos.FunctionDescProto f2) { + CatalogProtos.FunctionSignatureProto s1 = f1.getSignature(); + CatalogProtos.FunctionSignatureProto s2 = f2.getSignature(); + + int cmpVal = s1.getName().compareTo(s2.getName()); + + if (cmpVal != 0) { + return cmpVal; + } + + cmpVal = s1.getType().name().compareTo(s2.getType().name()); + + if (cmpVal != 0) { + return cmpVal; + } + + cmpVal = s1.getReturnType().getType().name().compareTo(s2.getReturnType().getType().name()); + + if (cmpVal != 0) { + return cmpVal; + } + + for (int i = 0; i < Math.min(s1.getParameterTypesCount(), s2.getParameterTypesCount()); i++) { + cmpVal = s1.getParameterTypes(i).getType().name().compareTo(s2.getParameterTypes(i).getType().name()); + + if (cmpVal != 0) { + return cmpVal; + } + } + + return s2.getParameterTypesCount() - s1.getParameterTypesCount(); + } + } + public static String buildFQFunctionSignature(String funcName, DataType returnType, DataType... paramTypes) { return returnType.getType().name().toLowerCase() + " " + buildSimpleFunctionSignature(funcName, paramTypes); } http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml deleted file mode 100644 index fe8f34a..0000000 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml +++ /dev/null @@ -1,739 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>tajo-project</artifactId> - <groupId>org.apache.tajo</groupId> - <version>0.11.0-SNAPSHOT</version> - <relativePath>../../../tajo-project</relativePath> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>tajo-hcatalog</artifactId> - <packaging>jar</packaging> - <name>Tajo Catalog Drivers HCatalog</name> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - </properties> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <source>1.6</source> - <target>1.6</target> - <encoding>${project.build.sourceEncoding}</encoding> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - <configuration> - <excludes> - <exclude>derby.log</exclude> - </excludes> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <includeScope>runtime</includeScope> - <outputDirectory>${project.build.directory}/lib</outputDirectory> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <overWriteIfNewer>true</overWriteIfNewer> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - </plugin> - </plugins> - </build> - - <dependencies> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-catalog-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-catalog-client</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-catalog-server</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-rpc</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage-common</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - <version>0.9.0</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - <version>0.9.0</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - <scope>provided</scope> - </dependency> - </dependencies> - - <profiles> - <profile> - <id>hcatalog-0.12.0</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hive.version>0.12.0</hive.version> - <parquet.version>1.5.0</parquet.version> - <parquet.format.version>2.1.0</parquet.format.version> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-contrib</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-hbase-handler</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-testutils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shimss</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>jline</groupId> - <artifactId>jline</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive.hcatalog</groupId> - <artifactId>hcatalog-core</artifactId> - <version>${hive.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-hive-bundle</artifactId> - <version>${parquet.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>hcatalog-0.13.0</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hive.version>0.13.0</hive.version> - <parquet.version>1.5.0</parquet.version> - <parquet.format.version>2.1.0</parquet.format.version> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-contrib</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-hbase-handler</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-testutils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shimss</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>jline</groupId> - <artifactId>jline</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive.hcatalog</groupId> - <artifactId>hive-hcatalog-core</artifactId> - <version>${hive.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-hive-bundle</artifactId> - <version>${parquet.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>hcatalog-0.13.1</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hive.version>0.13.1</hive.version> - <parquet.version>1.5.0</parquet.version> - <parquet.format.version>2.1.0</parquet.format.version> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-contrib</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-hbase-handler</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-testutils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shimss</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libfb303</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - <version>${hive.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - <exclusion> - <groupId>com.jolbox</groupId> - <artifactId>bonecp</artifactId> - </exclusion> - <exclusion> - <groupId>jline</groupId> - <artifactId>jline</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hive.hcatalog</groupId> - <artifactId>hive-hcatalog-core</artifactId> - <version>${hive.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-cli</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-hive-bundle</artifactId> - <version>${parquet.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>docs</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <executions> - <execution> - <!-- build javadoc jars per jar for publishing to maven --> - <id>module-javadocs</id> - <phase>package</phase> - <goals> - <goal>jar</goal> - </goals> - <configuration> - <destDir>${project.build.directory}</destDir> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>src</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <executions> - <execution> - <!-- builds source jars and attaches them to the project for publishing --> - <id>hadoop-java-sources</id> - <phase>package</phase> - <goals> - <goal>jar-no-fork</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> - - <reporting> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - </plugin> - </plugins> - </reporting> - -</project> http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java deleted file mode 100644 index 2c3fc6a..0000000 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ /dev/null @@ -1,891 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog.store; - -import com.google.common.collect.Lists; - -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; -import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; -import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; -import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.exception.NotImplementedException; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.exception.InternalException; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.util.KeyValueSet; -import org.apache.thrift.TException; - -import java.io.IOException; -import java.util.*; - -import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType; - -public class HCatalogStore extends CatalogConstants implements CatalogStore { - protected final Log LOG = LogFactory.getLog(getClass()); - - private static String HIVE_WAREHOUSE_DIR_CONF_KEY = "hive.metastore.warehouse.dir"; - - protected Configuration conf; - private static final int CLIENT_POOL_SIZE = 2; - private final HCatalogStoreClientPool clientPool; - private final String defaultTableSpaceUri; - - public HCatalogStore(final Configuration conf) throws InternalException { - if (!(conf instanceof TajoConf)) { - throw new CatalogException("Invalid Configuration Type:" + conf.getClass().getSimpleName()); - } - this.conf = conf; - this.defaultTableSpaceUri = TajoConf.getWarehouseDir((TajoConf) conf).toString(); - this.clientPool = new HCatalogStoreClientPool(CLIENT_POOL_SIZE, conf); - } - - @Override - public boolean existTable(final String databaseName, final String tableName) throws CatalogException { - boolean exist = false; - org.apache.hadoop.hive.ql.metadata.Table table; - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - // get table - try { - client = clientPool.getClient(); - table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); - if (table != null) { - exist = true; - } - } catch (NoSuchObjectException nsoe) { - exist = false; - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - - return exist; - } - - @Override - public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName) throws CatalogException { - org.apache.hadoop.hive.ql.metadata.Table table = null; - HCatalogStoreClientPool.HCatalogStoreClient client = null; - Path path = null; - CatalogProtos.StoreType storeType = null; - org.apache.tajo.catalog.Schema schema = null; - KeyValueSet options = null; - TableStats stats = null; - PartitionMethodDesc partitions = null; - - ////////////////////////////////// - // set tajo table schema. - ////////////////////////////////// - try { - // get hive table schema - try { - client = clientPool.getClient(); - table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); - path = table.getPath(); - } catch (NoSuchObjectException nsoe) { - throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); - } catch (Exception e) { - throw new CatalogException(e); - } - - // convert hcatalog field schema into tajo field schema. - schema = new org.apache.tajo.catalog.Schema(); - HCatSchema tableSchema = null; - - try { - tableSchema = HCatUtil.getTableSchemaWithPtnCols(table); - } catch (IOException ioe) { - throw new CatalogException("Fail to get table schema. - tableName:" + tableName, ioe); - } - List<HCatFieldSchema> fieldSchemaList = tableSchema.getFields(); - boolean isPartitionKey = false; - for (HCatFieldSchema eachField : fieldSchemaList) { - isPartitionKey = false; - - if (table.getPartitionKeys() != null) { - for (FieldSchema partitionKey : table.getPartitionKeys()) { - if (partitionKey.getName().equals(eachField.getName())) { - isPartitionKey = true; - } - } - } - - if (!isPartitionKey) { - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); - TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(eachField.getType().toString()); - schema.addColumn(fieldName, dataType); - } - } - - // validate field schema. - try { - HCatalogUtil.validateHCatTableAndTajoSchema(tableSchema); - } catch (Exception e) { - throw new CatalogException("HCatalog cannot support schema. - schema:" + tableSchema.toString(), e); - } - - stats = new TableStats(); - options = new KeyValueSet(); - options.putAll(table.getParameters()); - options.remove("EXTERNAL"); - - Properties properties = table.getMetadata(); - if (properties != null) { - // set field delimiter - String fieldDelimiter = "", nullFormat = ""; - if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) { - fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM); - } else { - // if hive table used default row format delimiter, Properties doesn't have it. - // So, Tajo must set as follows: - fieldDelimiter = "\u0001"; - } - - // set null format - if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) { - nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT); - } else { - nullFormat = "\\N"; - } - options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); - - // set file output format - String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); - storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat)); - - if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) { - options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); - options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); - } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { - options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); - String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); - if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { - options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - } else if (ColumnarSerDe.class.getName().equals(serde)) { - options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); - } - } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) { - options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); - options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); - String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); - if (LazyBinarySerDe.class.getName().equals(serde)) { - options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - } else if (LazySimpleSerDe.class.getName().equals(serde)) { - options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); - } - } - - // set data size - long totalSize = 0; - if (properties.getProperty("totalSize") != null) { - totalSize = Long.parseLong(properties.getProperty("totalSize")); - } else { - try { - FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - totalSize = fs.getContentSummary(path).getLength(); - } - } catch (IOException ioe) { - throw new CatalogException("Fail to get path. - path:" + path.toString(), ioe); - } - } - stats.setNumBytes(totalSize); - } - - // set partition keys - List<FieldSchema> partitionKeys = table.getPartitionKeys(); - - if (null != partitionKeys) { - org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); - StringBuilder sb = new StringBuilder(); - if (partitionKeys.size() > 0) { - for (int i = 0; i < partitionKeys.size(); i++) { - FieldSchema fieldSchema = partitionKeys.get(i); - TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(fieldSchema.getType().toString()); - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); - expressionSchema.addColumn(new Column(fieldName, dataType)); - if (i > 0) { - sb.append(","); - } - sb.append(fieldSchema.getName()); - } - partitions = new PartitionMethodDesc( - databaseName, - tableName, - PartitionType.COLUMN, - sb.toString(), - expressionSchema); - } - } - } finally { - if(client != null) client.release(); - } - TableMeta meta = new TableMeta(storeType, options); - TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); - if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { - tableDesc.setExternal(true); - } - if (stats != null) { - tableDesc.setStats(stats); - } - if (partitions != null) { - tableDesc.setPartitionMethod(partitions); - } - return tableDesc.getProto(); - } - - - private TajoDataTypes.Type getDataType(final String typeStr) { - try { - return Enum.valueOf(TajoDataTypes.Type.class, typeStr); - } catch (IllegalArgumentException iae) { - LOG.error("Cannot find a matched type against from '" + typeStr + "'"); - return null; - } - } - - @Override - public final List<String> getAllTableNames(String databaseName) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - client = clientPool.getClient(); - return client.getHiveClient().getAllTables(databaseName); - } catch (TException e) { - throw new CatalogException(e); - } finally { - if(client != null) client.release(); - } - } - - @Override - public void createTablespace(String spaceName, String spaceUri) throws CatalogException { - // SKIP - } - - @Override - public boolean existTablespace(String spaceName) throws CatalogException { - // SKIP - return spaceName.equals(TajoConstants.DEFAULT_TABLESPACE_NAME); - } - - @Override - public void dropTablespace(String spaceName) throws CatalogException { - // SKIP - } - - @Override - public Collection<String> getAllTablespaceNames() throws CatalogException { - return Lists.newArrayList(TajoConstants.DEFAULT_TABLESPACE_NAME); - } - - @Override - public TablespaceProto getTablespace(String spaceName) throws CatalogException { - if (spaceName.equals(TajoConstants.DEFAULT_TABLESPACE_NAME)) { - TablespaceProto.Builder builder = TablespaceProto.newBuilder(); - builder.setSpaceName(TajoConstants.DEFAULT_TABLESPACE_NAME); - builder.setUri(defaultTableSpaceUri); - return builder.build(); - } else { - throw new CatalogException("tablespace concept is not supported in HCatalogStore"); - } - } - - @Override - public void updateTableStats(CatalogProtos.UpdateTableStatsProto statsProto) throws - CatalogException { - // TODO - not implemented yet - } - - @Override - public void alterTablespace(CatalogProtos.AlterTablespaceProto alterProto) throws CatalogException { - throw new CatalogException("tablespace concept is not supported in HCatalogStore"); - } - - @Override - public void createDatabase(String databaseName, String tablespaceName) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - Database database = new Database( - databaseName, - "", - defaultTableSpaceUri + "/" + databaseName, - new HashMap<String, String>()); - client = clientPool.getClient(); - client.getHiveClient().createDatabase(database); - } catch (AlreadyExistsException e) { - throw new AlreadyExistsDatabaseException(databaseName); - } catch (Throwable t) { - throw new CatalogException(t); - } finally { - if (client != null) { - client.release(); - } - } - } - - @Override - public boolean existDatabase(String databaseName) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - client = clientPool.getClient(); - List<String> databaseNames = client.getHiveClient().getAllDatabases(); - return databaseNames.contains(databaseName); - } catch (Throwable t) { - throw new CatalogException(t); - } finally { - if (client != null) { - client.release(); - } - } - } - - @Override - public void dropDatabase(String databaseName) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - client = clientPool.getClient(); - client.getHiveClient().dropDatabase(databaseName); - } catch (NoSuchObjectException e) { - throw new NoSuchDatabaseException(databaseName); - } catch (Throwable t) { - throw new CatalogException(databaseName); - } finally { - if (client != null) { - client.release(); - } - } - } - - @Override - public Collection<String> getAllDatabaseNames() throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - client = clientPool.getClient(); - return client.getHiveClient().getAllDatabases(); - } catch (TException e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - } - - @Override - public final void createTable(final CatalogProtos.TableDescProto tableDescProto) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - TableDesc tableDesc = new TableDesc(tableDescProto); - String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); - String databaseName = splitted[0]; - String tableName = splitted[1]; - - try { - client = clientPool.getClient(); - - org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api.Table(); - table.setDbName(databaseName); - table.setTableName(tableName); - table.setParameters(new HashMap<String, String>(tableDesc.getMeta().getOptions().getAllKeyValus())); - // TODO: set owner - //table.setOwner(); - - StorageDescriptor sd = new StorageDescriptor(); - sd.setSerdeInfo(new SerDeInfo()); - sd.getSerdeInfo().setParameters(new HashMap<String, String>()); - sd.getSerdeInfo().setName(table.getTableName()); - - // if tajo set location method, thrift client make exception as follows: - // Caused by: MetaException(message:java.lang.NullPointerException) - // If you want to modify table path, you have to modify on Hive cli. - if (tableDesc.isExternal()) { - table.setTableType(TableType.EXTERNAL_TABLE.name()); - table.putToParameters("EXTERNAL", "TRUE"); - - Path tablePath = new Path(tableDesc.getPath()); - FileSystem fs = tablePath.getFileSystem(conf); - if (fs.isFile(tablePath)) { - LOG.warn("A table path is a file, but HCatalog does not allow a file path."); - sd.setLocation(tablePath.getParent().toString()); - } else { - sd.setLocation(tablePath.toString()); - } - } - - // set column information - List<Column> columns = tableDesc.getSchema().getColumns(); - ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size()); - - for (Column eachField : columns) { - cols.add(new FieldSchema(eachField.getSimpleName(), - HCatalogUtil.getHiveFieldType(eachField.getDataType()), "")); - } - sd.setCols(cols); - - // set partition keys - if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { - List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>(); - for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) { - partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), - HCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); - } - table.setPartitionKeys(partitionKeys); - } - - if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) { - String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE); - sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); - sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); - if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { - sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); - } else { - sd.getSerdeInfo().setSerializationLib( - org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName()); - } - - if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) { - table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); - } - } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV) - || tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { - sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); - sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); - sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); - - String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.TEXT_DELIMITER, - StorageConstants.DEFAULT_FIELD_DELIMITER); - - // User can use an unicode for filed delimiter such as \u0001, \001. - // In this case, java console will convert this value into "\\u001". - // And hive will un-espace this value again. - // As a result, user can use right field delimiter. - // So, we have to un-escape this value. - sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, - StringEscapeUtils.unescapeJava(fieldDelimiter)); - sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, - StringEscapeUtils.unescapeJava(fieldDelimiter)); - table.getParameters().remove(StorageConstants.TEXT_DELIMITER); - - if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) { - table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL))); - table.getParameters().remove(StorageConstants.TEXT_NULL); - } - } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) { - String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); - sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()); - sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName()); - - if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { - sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); - - String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_DELIMITER, - StorageConstants.DEFAULT_FIELD_DELIMITER); - - // User can use an unicode for filed delimiter such as \u0001, \001. - // In this case, java console will convert this value into "\\u001". - // And hive will un-espace this value again. - // As a result, user can use right field delimiter. - // So, we have to un-escape this value. - sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, - StringEscapeUtils.unescapeJava(fieldDelimiter)); - sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, - StringEscapeUtils.unescapeJava(fieldDelimiter)); - table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER); - } else { - sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); - } - - if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) { - table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); - table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); - } - } else { - if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.PARQUET)) { - sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName()); - sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName()); - sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName()); - } else { - throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType - ().name())); - } - } - - sd.setSortCols(new ArrayList<Order>()); - - table.setSd(sd); - client.getHiveClient().createTable(table); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if(client != null) client.release(); - } - } - - @Override - public final void dropTable(String databaseName, final String tableName) throws CatalogException { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - client = clientPool.getClient(); - client.getHiveClient().dropTable(databaseName, tableName, false, false); - } catch (NoSuchObjectException nsoe) { - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - } - - - @Override - public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescProto) throws CatalogException { - final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName()); - - if (split.length == 1) { - throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" - + alterTableDescProto.getTableName() + "\"."); - } - - final String databaseName = split[0]; - final String tableName = split[1]; - - - switch (alterTableDescProto.getAlterTableType()) { - case RENAME_TABLE: - if (existTable(databaseName,alterTableDescProto.getNewTableName().toLowerCase())) { - throw new AlreadyExistsTableException(alterTableDescProto.getNewTableName()); - } - renameTable(databaseName, tableName, alterTableDescProto.getNewTableName().toLowerCase()); - break; - case RENAME_COLUMN: - if (existColumn(databaseName,tableName, alterTableDescProto.getAlterColumnName().getNewColumnName())) { - throw new ColumnNameAlreadyExistException(alterTableDescProto.getAlterColumnName().getNewColumnName()); - } - renameColumn(databaseName, tableName, alterTableDescProto.getAlterColumnName()); - break; - case ADD_COLUMN: - if (existColumn(databaseName,tableName, alterTableDescProto.getAddColumn().getName())) { - throw new ColumnNameAlreadyExistException(alterTableDescProto.getAddColumn().getName()); - } - addNewColumn(databaseName, tableName, alterTableDescProto.getAddColumn()); - break; - default: - //TODO - } - } - - - private void renameTable(String databaseName, String tableName, String newTableName) { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - try { - client = clientPool.getClient(); - Table newTable = client.getHiveClient().getTable(databaseName, tableName); - newTable.setTableName(newTableName); - client.getHiveClient().alter_table(databaseName, tableName, newTable); - - } catch (NoSuchObjectException nsoe) { - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - } - - private void renameColumn(String databaseName, String tableName, CatalogProtos.AlterColumnProto alterColumnProto) { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - try { - - client = clientPool.getClient(); - Table table = client.getHiveClient().getTable(databaseName, tableName); - List<FieldSchema> columns = table.getSd().getCols(); - - for (final FieldSchema currentColumn : columns) { - if (currentColumn.getName().equalsIgnoreCase(alterColumnProto.getOldColumnName())) { - currentColumn.setName(alterColumnProto.getNewColumnName()); - } - } - client.getHiveClient().alter_table(databaseName, tableName, table); - - } catch (NoSuchObjectException nsoe) { - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - } - - - private void addNewColumn(String databaseName, String tableName, CatalogProtos.ColumnProto columnProto) { - HCatalogStoreClientPool.HCatalogStoreClient client = null; - try { - - client = clientPool.getClient(); - Table table = client.getHiveClient().getTable(databaseName, tableName); - List<FieldSchema> columns = table.getSd().getCols(); - columns.add(new FieldSchema(columnProto.getName(), - HCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); - client.getHiveClient().alter_table(databaseName, tableName, table); - - - } catch (NoSuchObjectException nsoe) { - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - } - - @Override - public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMethodProto) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName) - throws CatalogException { - return null; // TODO - not implemented yet - } - - @Override - public boolean existPartitionMethod(String databaseName, String tableName) throws CatalogException { - return false; // TODO - not implemented yet - } - - @Override - public void dropPartitionMethod(String databaseName, String tableName) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public void addPartitions(CatalogProtos.PartitionsProto partitionsProto) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto partitionDescProto) throws CatalogException { - - } - - @Override - public CatalogProtos.PartitionsProto getPartitions(String tableName) throws CatalogException { - return null; // TODO - not implemented yet - } - - @Override - public CatalogProtos.PartitionDescProto getPartition(String partitionName) throws CatalogException { - return null; // TODO - not implemented yet - } - - @Override - public void delPartition(String partitionName) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public void dropPartitions(String tableName) throws CatalogException { - - } - - - @Override - public final void addFunction(final FunctionDesc func) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public final void deleteFunction(final FunctionDesc func) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public final void existFunction(final FunctionDesc func) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public final List<String> getAllFunctionNames() throws CatalogException { - // TODO - not implemented yet - return null; - } - - @Override - public void dropIndex(String databaseName, String indexName) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public boolean existIndexByName(String databaseName, String indexName) throws CatalogException { - // TODO - not implemented yet - return false; - } - - @Override - public CatalogProtos.IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException { - // TODO - not implemented yet - return null; - } - - @Override - public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException { - // TODO - not implemented yet - } - - @Override - public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException { - // TODO - not implemented yet - return null; - } - - @Override - public CatalogProtos.IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { - // TODO - not implemented yet - return null; - } - - @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) throws CatalogException { - // TODO - not implemented yet - return false; - } - - @Override - public final void close() { - clientPool.close(); - } - - private boolean existColumn(final String databaseName ,final String tableName , final String columnName) throws CatalogException { - boolean exist = false; - HCatalogStoreClientPool.HCatalogStoreClient client = null; - - try { - - client = clientPool.getClient(); - Table table = client.getHiveClient().getTable(databaseName, tableName); - List<FieldSchema> columns = table.getSd().getCols(); - - for (final FieldSchema currentColumn : columns) { - if (currentColumn.getName().equalsIgnoreCase(columnName)) { - exist = true; - } - } - client.getHiveClient().alter_table(databaseName, tableName, table); - - } catch (NoSuchObjectException nsoe) { - } catch (Exception e) { - throw new CatalogException(e); - } finally { - if (client != null) { - client.release(); - } - } - - return exist; - } - - @Override - public List<ColumnProto> getAllColumns() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<DatabaseProto> getAllDatabases() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<IndexProto> getAllIndexes() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<TablePartitionProto> getAllPartitions() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<TableOptionProto> getAllTableOptions() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<TableStatsProto> getAllTableStats() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<TableDescriptorProto> getAllTables() throws CatalogException { - throw new UnsupportedOperationException(); - } - - @Override - public List<TablespaceProto> getTablespaces() throws CatalogException { - throw new UnsupportedOperationException(); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java deleted file mode 100644 index 8ccb100..0000000 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2012 Cloudera Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package org.apache.tajo.catalog.store; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.*; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.log4j.Logger; - -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Manages a pool of HiveMetaStoreClient connections. If the connection pool is empty - * a new client is created and added to the pool. There is no size limit. - */ -public class HCatalogStoreClientPool { - private static final Logger LOG = Logger.getLogger(HCatalogStoreClientPool.class); - private final ConcurrentLinkedQueue<HCatalogStoreClient> clientPool = - new ConcurrentLinkedQueue<HCatalogStoreClient>(); - private AtomicBoolean poolClosed = new AtomicBoolean(false); - private HiveConf hiveConf; - - /** - * A wrapper around the HiveMetaStoreClient that manages interactions with the - * connection pool. - */ - public class HCatalogStoreClient { - private final IMetaStoreClient hiveClient; - public AtomicBoolean isInUse = new AtomicBoolean(false); - - private HCatalogStoreClient(HiveConf hiveConf) { - try { - HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { - @Override - public HiveMetaHook getHook(Table table) throws MetaException { - /* metadata hook implementation, or null if this - * storage handler does not need any metadata notifications - */ - return null; - } - }; - - this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); - clientPool.add(this); - LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")"); - } catch (Exception e) { - // Turn in to an unchecked exception - throw new IllegalStateException(e); - } - } - - /** - * Returns the internal HiveMetaStoreClient object. - */ - public IMetaStoreClient getHiveClient() { - return hiveClient; - } - - /** - * Returns this client back to the connection pool. If the connection pool has been - * closed, just close the Hive client connection. - */ - public synchronized void release() { - if(!this.isInUse.getAndSet(false)){ - return; - } - // Ensure the connection isn't returned to the pool if the pool has been closed. - // This lock is needed to ensure proper behavior when a thread reads poolClosed - // is false, but a call to pool.close() comes in immediately afterward. - if (poolClosed.get()) { - this.getHiveClient().close(); - } else { - clientPool.add(this); - } - } - - // Marks this client as in use - private void markInUse() { - isInUse.set(true); - } - } - - public HCatalogStoreClientPool(int initialSize) { - this(initialSize, new HiveConf(HCatalogStoreClientPool.class)); - } - - public HCatalogStoreClientPool(int initialSize, HiveConf hiveConf) { - this.hiveConf = hiveConf; - addClients(initialSize); - } - - public HCatalogStoreClientPool(int initialSize, Configuration conf) { - this.hiveConf = new HiveConf(); - setParameters(conf); - addClients(initialSize); - } - - public void setParameters(Configuration conf) { - for( Iterator<Entry<String, String>> iter = conf.iterator(); iter.hasNext();) { - Map.Entry<String, String> entry = iter.next(); - this.hiveConf.set(entry.getKey(), entry.getValue()); - } - } - - /** - * Add numClients to the client pool. - */ - public void addClients(int numClients) { - for (int i = 0; i < numClients; ++i) { - clientPool.add(new HCatalogStoreClient(hiveConf)); - } - } - - /** - * Gets a client from the pool. If the pool is empty a new client is created. - */ - public synchronized HCatalogStoreClient getClient() { - // The MetaStoreClient c'tor relies on knowing the Hadoop version by asking - // org.apache.hadoop.util.VersionInfo. The VersionInfo class relies on opening - // the 'common-version-info.properties' file as a resource from hadoop-common*.jar - // using the Thread's context classloader. If necessary, set the Thread's context - // classloader, otherwise VersionInfo will fail in it's c'tor. - if (Thread.currentThread().getContextClassLoader() == null) { - Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); - } - - HCatalogStoreClient client = clientPool.poll(); - // The pool was empty so create a new client and return that. - if (client == null) { - client = new HCatalogStoreClient(hiveConf); - } - client.markInUse(); - - return client; - } - - /** - * Removes all items from the connection pool and closes all Hive Meta Store client - * connections. Can be called multiple times. - */ - public void close() { - // Ensure no more items get added to the pool once close is called. - if (poolClosed.getAndSet(true)) { - return; - } - - HCatalogStoreClient client = null; - while ((client = clientPool.poll()) != null) { - client.getHiveClient().close(); - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/facbf432/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java deleted file mode 100644 index 8e8e58c..0000000 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.catalog.store; - -import com.google.common.base.Preconditions; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; -import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hcatalog.common.HCatException; -import org.apache.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.tajo.catalog.exception.CatalogException; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.thrift.TException; -import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; - -public class HCatalogUtil { - protected final Log LOG = LogFactory.getLog(getClass()); - - public static void validateHCatTableAndTajoSchema(HCatSchema tblSchema) throws CatalogException { - for (HCatFieldSchema hcatField : tblSchema.getFields()) { - validateHCatFieldAndTajoSchema(hcatField); - } - } - - private static void validateHCatFieldAndTajoSchema(HCatFieldSchema fieldSchema) throws CatalogException { - try { - HCatFieldSchema.Type fieldType = fieldSchema.getType(); - switch (fieldType) { - case ARRAY: - throw new HCatException("Tajo cannot support array field type."); - case STRUCT: - throw new HCatException("Tajo cannot support struct field type."); - case MAP: - throw new HCatException("Tajo cannot support map field type."); - } - } catch (HCatException e) { - throw new CatalogException("incompatible hcatalog types when assigning to tajo type. - " + - "HCatFieldSchema:" + fieldSchema); - } - } - - public static TajoDataTypes.Type getTajoFieldType(String fieldType) { - Preconditions.checkNotNull(fieldType); - - String typeStr = null; - - if(fieldType.equalsIgnoreCase(serdeConstants.INT_TYPE_NAME)) - typeStr = "INT4"; - else if(fieldType.equalsIgnoreCase(serdeConstants.TINYINT_TYPE_NAME)) - typeStr = "INT1"; - else if(fieldType.equalsIgnoreCase(serdeConstants.SMALLINT_TYPE_NAME)) - typeStr = "INT2"; - else if(fieldType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) - typeStr = "INT8"; - else if(fieldType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) - typeStr = "BOOLEAN"; - else if(fieldType.equalsIgnoreCase(serdeConstants.FLOAT_TYPE_NAME)) - typeStr = "FLOAT4"; - else if(fieldType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)) - typeStr = "FLOAT8"; - else if(fieldType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) - typeStr = "TEXT"; - else if(fieldType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)) - typeStr = "BLOB"; - - try { - return Enum.valueOf(TajoDataTypes.Type.class, typeStr); - } catch (IllegalArgumentException iae) { - throw new CatalogException("Cannot find a matched type against from '" + typeStr + "'"); - } - } - - public static String getHiveFieldType(TajoDataTypes.DataType dataType) { - Preconditions.checkNotNull(dataType); - - switch (dataType.getType()) { - case CHAR: return serdeConstants.CHAR_TYPE_NAME; - case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME; - case INT1: return serdeConstants.TINYINT_TYPE_NAME; - case INT2: return serdeConstants.SMALLINT_TYPE_NAME; - case INT4: return serdeConstants.INT_TYPE_NAME; - case INT8: return serdeConstants.BIGINT_TYPE_NAME; - case FLOAT4: return serdeConstants.FLOAT_TYPE_NAME; - case FLOAT8: return serdeConstants.DOUBLE_TYPE_NAME; - case TEXT: return serdeConstants.STRING_TYPE_NAME; - case VARCHAR: return serdeConstants.VARCHAR_TYPE_NAME; - case NCHAR: return serdeConstants.VARCHAR_TYPE_NAME; - case NVARCHAR: return serdeConstants.VARCHAR_TYPE_NAME; - case BINARY: return serdeConstants.BINARY_TYPE_NAME; - case VARBINARY: return serdeConstants.BINARY_TYPE_NAME; - case BLOB: return serdeConstants.BINARY_TYPE_NAME; - case DATE: return serdeConstants.DATE_TYPE_NAME; - case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME; - default: - throw new CatalogException(dataType + " is not supported."); - } - } - - public static String getStoreType(String fileFormat) { - Preconditions.checkNotNull(fileFormat); - - String[] fileFormatArrary = fileFormat.split("\\."); - if(fileFormatArrary.length < 1) { - throw new CatalogException("Hive file output format is wrong. - file output format:" + fileFormat); - } - - String outputFormatClass = fileFormatArrary[fileFormatArrary.length-1]; - if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { - return CatalogUtil.TEXTFILE_NAME; - } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.SEQUENCEFILE.name(); - } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.RCFILE.name(); - } else if(outputFormatClass.equals(DeprecatedParquetOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.PARQUET.name(); - } else { - throw new CatalogException("Not supported file output format. - file output format:" + fileFormat); - } - } - - public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException { - return new Table(client.getTable(dbName, tableName)); - } -}
