http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/Type.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Type.java b/fe/src/main/java/com/cloudera/impala/catalog/Type.java deleted file mode 100644 index 0162ec6..0000000 --- a/fe/src/main/java/com/cloudera/impala/catalog/Type.java +++ /dev/null @@ -1,781 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.catalog; - -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -import com.cloudera.impala.analysis.CreateTableStmt; -import com.cloudera.impala.analysis.SqlParser; -import com.cloudera.impala.analysis.SqlScanner; -import com.cloudera.impala.analysis.TypeDef; -import com.cloudera.impala.common.Pair; -import com.cloudera.impala.thrift.TColumnType; -import com.cloudera.impala.thrift.TPrimitiveType; -import com.cloudera.impala.thrift.TScalarType; -import com.cloudera.impala.thrift.TStructField; -import com.cloudera.impala.thrift.TTypeNode; -import com.cloudera.impala.thrift.TTypeNodeType; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Abstract class describing an Impala data type (scalar/complex type). - * Mostly contains static type instances and helper methods for convenience, as well - * as abstract methods that subclasses must implement. - */ -public abstract class Type { - // Maximum nesting depth of a type. This limit was determined experimentally by - // generating and scanning deeply nested Parquet and Avro files. In those experiments, - // we exceeded the stack space in the scanner (which uses recursion for dealing with - // nested types) at a nesting depth between 200 and 300 (200 worked, 300 crashed). - public static int MAX_NESTING_DEPTH = 100; - - // Static constant types for scalar types that don't require additional information. - public static final ScalarType INVALID = new ScalarType(PrimitiveType.INVALID_TYPE); - public static final ScalarType NULL = new ScalarType(PrimitiveType.NULL_TYPE); - public static final ScalarType BOOLEAN = new ScalarType(PrimitiveType.BOOLEAN); - public static final ScalarType TINYINT = new ScalarType(PrimitiveType.TINYINT); - public static final ScalarType SMALLINT = new ScalarType(PrimitiveType.SMALLINT); - public static final ScalarType INT = new ScalarType(PrimitiveType.INT); - public static final ScalarType BIGINT = new ScalarType(PrimitiveType.BIGINT); - public static final ScalarType FLOAT = new ScalarType(PrimitiveType.FLOAT); - public static final ScalarType DOUBLE = new ScalarType(PrimitiveType.DOUBLE); - public static final ScalarType STRING = new ScalarType(PrimitiveType.STRING); - public static final ScalarType BINARY = new ScalarType(PrimitiveType.BINARY); - public static final ScalarType TIMESTAMP = new ScalarType(PrimitiveType.TIMESTAMP); - public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE); - public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME); - public static final ScalarType DEFAULT_DECIMAL = (ScalarType) - ScalarType.createDecimalType(ScalarType.DEFAULT_PRECISION, - ScalarType.DEFAULT_SCALE); - public static final ScalarType DECIMAL = - (ScalarType) ScalarType.createDecimalTypeInternal(-1, -1); - public static final ScalarType DEFAULT_VARCHAR = ScalarType.createVarcharType(-1); - public static final ScalarType VARCHAR = ScalarType.createVarcharType(-1); - public static final ScalarType CHAR = (ScalarType) ScalarType.createCharType(-1); - - private static ArrayList<ScalarType> integerTypes; - private static ArrayList<ScalarType> numericTypes; - private static ArrayList<ScalarType> supportedTypes; - private static ArrayList<ScalarType> unsupportedTypes; - - static { - integerTypes = Lists.newArrayList(); - integerTypes.add(TINYINT); - integerTypes.add(SMALLINT); - integerTypes.add(INT); - integerTypes.add(BIGINT); - - numericTypes = Lists.newArrayList(); - numericTypes.add(TINYINT); - numericTypes.add(SMALLINT); - numericTypes.add(INT); - numericTypes.add(BIGINT); - numericTypes.add(FLOAT); - numericTypes.add(DOUBLE); - numericTypes.add(DECIMAL); - - supportedTypes = Lists.newArrayList(); - supportedTypes.add(NULL); - supportedTypes.add(BOOLEAN); - supportedTypes.add(TINYINT); - supportedTypes.add(SMALLINT); - supportedTypes.add(INT); - supportedTypes.add(BIGINT); - supportedTypes.add(FLOAT); - supportedTypes.add(DOUBLE); - supportedTypes.add(STRING); - supportedTypes.add(VARCHAR); - supportedTypes.add(CHAR); - supportedTypes.add(TIMESTAMP); - supportedTypes.add(DECIMAL); - - unsupportedTypes = Lists.newArrayList(); - unsupportedTypes.add(BINARY); - unsupportedTypes.add(DATE); - unsupportedTypes.add(DATETIME); - } - - public static ArrayList<ScalarType> getIntegerTypes() { - return integerTypes; - } - public static ArrayList<ScalarType> getNumericTypes() { - return numericTypes; - } - public static ArrayList<ScalarType> getSupportedTypes() { - return supportedTypes; - } - public static ArrayList<ScalarType> getUnsupportedTypes() { - return unsupportedTypes; - } - - /** - * The output of this is stored directly in the hive metastore as the column type. - * The string must match exactly. - */ - public final String toSql() { return toSql(0); } - - /** - * Recursive helper for toSql() to be implemented by subclasses. Keeps track of the - * nesting depth and terminates the recursion if MAX_NESTING_DEPTH is reached. - */ - protected abstract String toSql(int depth); - - /** - * Same as toSql() but adds newlines and spaces for better readability of nested types. - */ - public String prettyPrint() { return prettyPrint(0); } - - /** - * Pretty prints this type with lpad number of leading spaces. Used to implement - * prettyPrint() with space-indented nested types. - */ - protected abstract String prettyPrint(int lpad); - - public boolean isInvalid() { return isScalarType(PrimitiveType.INVALID_TYPE); } - public boolean isValid() { return !isInvalid(); } - public boolean isNull() { return isScalarType(PrimitiveType.NULL_TYPE); } - public boolean isBoolean() { return isScalarType(PrimitiveType.BOOLEAN); } - public boolean isTimestamp() { return isScalarType(PrimitiveType.TIMESTAMP); } - public boolean isDecimal() { return isScalarType(PrimitiveType.DECIMAL); } - public boolean isDecimalOrNull() { return isDecimal() || isNull(); } - public boolean isFullySpecifiedDecimal() { return false; } - public boolean isWildcardDecimal() { return false; } - public boolean isWildcardVarchar() { return false; } - public boolean isWildcardChar() { return false; } - - public boolean isStringType() { - return isScalarType(PrimitiveType.STRING) || isScalarType(PrimitiveType.VARCHAR) || - isScalarType(PrimitiveType.CHAR); - } - - public boolean isScalarType() { return this instanceof ScalarType; } - public boolean isScalarType(PrimitiveType t) { - return isScalarType() && ((ScalarType) this).getPrimitiveType() == t; - } - - public boolean isFixedPointType() { - return isScalarType(PrimitiveType.TINYINT) || isScalarType(PrimitiveType.SMALLINT) || - isScalarType(PrimitiveType.INT) || isScalarType(PrimitiveType.BIGINT) || - isScalarType(PrimitiveType.DECIMAL); - } - - public boolean isFloatingPointType() { - return isScalarType(PrimitiveType.FLOAT) || isScalarType(PrimitiveType.DOUBLE); - } - - public boolean isIntegerType() { - return isScalarType(PrimitiveType.TINYINT) || isScalarType(PrimitiveType.SMALLINT) - || isScalarType(PrimitiveType.INT) || isScalarType(PrimitiveType.BIGINT); - } - - // TODO: Handle complex types properly. Some instances may be fixed length. - public boolean isFixedLengthType() { return false; } - - public boolean isNumericType() { - return isFixedPointType() || isFloatingPointType() || isDecimal(); - } - - public boolean isDateType() { - return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME) - || isScalarType(PrimitiveType.TIMESTAMP); - } - - public boolean isComplexType() { return isStructType() || isCollectionType(); } - public boolean isCollectionType() { return isMapType() || isArrayType(); } - public boolean isMapType() { return this instanceof MapType; } - public boolean isArrayType() { return this instanceof ArrayType; } - public boolean isStructType() { return this instanceof StructType; } - - /** - * Returns true if Impala supports this type in the metdata. It does not mean we - * can manipulate data of this type. For tables that contain columns with these - * types, we can safely skip over them. - */ - public boolean isSupported() { return true; } - - /** - * Indicates whether we support partitioning tables on columns of this type. - */ - public boolean supportsTablePartitioning() { return false; } - - public PrimitiveType getPrimitiveType() { return PrimitiveType.INVALID_TYPE; } - - /** - * Returns the size in bytes of the fixed-length portion that a slot of this type - * occupies in a tuple. - */ - public int getSlotSize() { - // 8-byte pointer and 4-byte length indicator (12 bytes total). - // Per struct alignment rules, there is an extra 4 bytes of padding to align to 8 - // bytes so 16 bytes total. - if (isCollectionType()) return 16; - throw new IllegalStateException("getSlotSize() not implemented for type " + toSql()); - } - - public TColumnType toThrift() { - TColumnType container = new TColumnType(); - container.setTypes(new ArrayList<TTypeNode>()); - toThrift(container); - return container; - } - - /** - * Subclasses should override this method to add themselves to the thrift container. - */ - public abstract void toThrift(TColumnType container); - - /** - * Returns true if this type is equal to t, or if t is a wildcard variant of this - * type. Subclasses should override this as appropriate. The default implementation - * here is to avoid special-casing logic in callers for concrete types. - */ - public boolean matchesType(Type t) { return false; } - - /** - * Gets the ColumnType from the given FieldSchema by using Impala's SqlParser. - * Returns null if the FieldSchema could not be parsed. - * The type can either be: - * - Supported by Impala, in which case the type is returned. - * - A type Impala understands but is not yet implemented (e.g. date), the type is - * returned but type.IsSupported() returns false. - * - A type Impala can't understand at all in which case null is returned. - */ - public static Type parseColumnType(String typeStr) { - // Wrap the type string in a CREATE TABLE stmt and use Impala's Parser - // to get the ColumnType. - // Pick a table name that can't be used. - String stmt = String.format("CREATE TABLE $DUMMY ($DUMMY %s)", typeStr); - SqlScanner input = new SqlScanner(new StringReader(stmt)); - SqlParser parser = new SqlParser(input); - CreateTableStmt createTableStmt; - try { - Object o = parser.parse().value; - if (!(o instanceof CreateTableStmt)) { - // Should never get here. - throw new IllegalStateException("Couldn't parse create table stmt."); - } - createTableStmt = (CreateTableStmt) o; - if (createTableStmt.getColumnDefs().isEmpty()) { - // Should never get here. - throw new IllegalStateException("Invalid create table stmt."); - } - } catch (Exception e) { - return null; - } - TypeDef typeDef = createTableStmt.getColumnDefs().get(0).getTypeDef(); - return typeDef.getType(); - } - - /** - * Returns true if t1 can be implicitly cast to t2 according to Impala's casting rules. - * Implicit casts are always allowed when no loss of precision would result (i.e. every - * value of t1 can be represented exactly by a value of t2). Implicit casts are allowed - * in certain other cases such as casting numeric types to floating point types and - * converting strings to timestamps. - * If strict is true, only consider casts that result in no loss of precision. - * TODO: Support casting of non-scalar types. - */ - public static boolean isImplicitlyCastable(Type t1, Type t2, boolean strict) { - if (t1.isScalarType() && t2.isScalarType()) { - return ScalarType.isImplicitlyCastable( - (ScalarType) t1, (ScalarType) t2, strict); - } - return false; - } - - /** - * Return type t such that values from both t1 and t2 can be assigned to t without an - * explicit cast. If strict, does not consider conversions that would result in loss - * of precision (e.g. converting decimal to float). Returns INVALID_TYPE if there is - * no such type or if any of t1 and t2 is INVALID_TYPE. - * TODO: Support non-scalar types. - */ - public static Type getAssignmentCompatibleType(Type t1, Type t2, boolean strict) { - if (t1.isScalarType() && t2.isScalarType()) { - return ScalarType.getAssignmentCompatibleType( - (ScalarType) t1, (ScalarType) t2, strict); - } - return ScalarType.INVALID; - } - - /** - * Returns true if this type exceeds the MAX_NESTING_DEPTH, false otherwise. - */ - public boolean exceedsMaxNestingDepth() { return exceedsMaxNestingDepth(0); } - - /** - * Helper for exceedsMaxNestingDepth(). Recursively computes the max nesting depth, - * terminating early if MAX_NESTING_DEPTH is reached. Returns true if this type - * exceeds the MAX_NESTING_DEPTH, false otherwise. - * - * Examples of types and their nesting depth: - * INT --> 1 - * STRUCT<f1:INT> --> 2 - * STRUCT<f1:STRUCT<f2:INT>> --> 3 - * ARRAY<INT> --> 2 - * ARRAY<STRUCT<f1:INT>> --> 3 - * MAP<STRING,INT> --> 2 - * MAP<STRING,STRUCT<f1:INT>> --> 3 - */ - private boolean exceedsMaxNestingDepth(int d) { - if (d >= MAX_NESTING_DEPTH) return true; - if (isStructType()) { - StructType structType = (StructType) this; - for (StructField f: structType.getFields()) { - if (f.getType().exceedsMaxNestingDepth(d + 1)) return true; - } - } else if (isArrayType()) { - ArrayType arrayType = (ArrayType) this; - if (arrayType.getItemType().exceedsMaxNestingDepth(d + 1)) return true; - } else if (isMapType()) { - MapType mapType = (MapType) this; - if (mapType.getValueType().exceedsMaxNestingDepth(d + 1)) return true; - } else { - Preconditions.checkState(isScalarType()); - } - return false; - } - - public static List<TColumnType> toThrift(Type[] types) { - return toThrift(Lists.newArrayList(types)); - } - - public static List<TColumnType> toThrift(ArrayList<Type> types) { - ArrayList<TColumnType> result = Lists.newArrayList(); - for (Type t: types) { - result.add(t.toThrift()); - } - return result; - } - - public static Type fromThrift(TColumnType thrift) { - Preconditions.checkState(thrift.types.size() > 0); - Pair<Type, Integer> t = fromThrift(thrift, 0); - Preconditions.checkState(t.second.equals(thrift.getTypesSize())); - return t.first; - } - - /** - * Constructs a ColumnType rooted at the TTypeNode at nodeIdx in TColumnType. - * Returned pair: The resulting ColumnType and the next nodeIdx that is not a child - * type of the result. - */ - protected static Pair<Type, Integer> fromThrift(TColumnType col, int nodeIdx) { - TTypeNode node = col.getTypes().get(nodeIdx); - Type type = null; - switch (node.getType()) { - case SCALAR: { - Preconditions.checkState(node.isSetScalar_type()); - TScalarType scalarType = node.getScalar_type(); - if (scalarType.getType() == TPrimitiveType.CHAR) { - Preconditions.checkState(scalarType.isSetLen()); - type = ScalarType.createCharType(scalarType.getLen()); - } else if (scalarType.getType() == TPrimitiveType.VARCHAR) { - Preconditions.checkState(scalarType.isSetLen()); - type = ScalarType.createVarcharType(scalarType.getLen()); - } else if (scalarType.getType() == TPrimitiveType.DECIMAL) { - Preconditions.checkState(scalarType.isSetPrecision() - && scalarType.isSetScale()); - type = ScalarType.createDecimalType(scalarType.getPrecision(), - scalarType.getScale()); - } else { - type = ScalarType.createType( - PrimitiveType.fromThrift(scalarType.getType())); - } - ++nodeIdx; - break; - } - case ARRAY: { - Preconditions.checkState(nodeIdx + 1 < col.getTypesSize()); - Pair<Type, Integer> childType = fromThrift(col, nodeIdx + 1); - type = new ArrayType(childType.first); - nodeIdx = childType.second; - break; - } - case MAP: { - Preconditions.checkState(nodeIdx + 2 < col.getTypesSize()); - Pair<Type, Integer> keyType = fromThrift(col, nodeIdx + 1); - Pair<Type, Integer> valueType = fromThrift(col, keyType.second); - type = new MapType(keyType.first, valueType.first); - nodeIdx = valueType.second; - break; - } - case STRUCT: { - Preconditions.checkState(nodeIdx + node.getStruct_fieldsSize() < col.getTypesSize()); - ArrayList<StructField> structFields = Lists.newArrayList(); - ++nodeIdx; - for (int i = 0; i < node.getStruct_fieldsSize(); ++i) { - TStructField thriftField = node.getStruct_fields().get(i); - String name = thriftField.getName(); - String comment = null; - if (thriftField.isSetComment()) comment = thriftField.getComment(); - Pair<Type, Integer> res = fromThrift(col, nodeIdx); - nodeIdx = res.second.intValue(); - structFields.add(new StructField(name, res.first, comment)); - } - type = new StructType(structFields); - break; - } - } - return new Pair<Type, Integer>(type, nodeIdx); - } - - /** - * Utility function to get the primitive type of a thrift type that is known - * to be scalar. - */ - public TPrimitiveType getTPrimitiveType(TColumnType ttype) { - Preconditions.checkState(ttype.getTypesSize() == 1); - Preconditions.checkState(ttype.types.get(0).getType() == TTypeNodeType.SCALAR); - return ttype.types.get(0).scalar_type.getType(); - } - - /** - * JDBC data type description - * Returns the column size for this type. - * For numeric data this is the maximum precision. - * For character data this is the length in characters. - * For datetime types this is the length in characters of the String representation - * (assuming the maximum allowed precision of the fractional seconds component). - * For binary data this is the length in bytes. - * Null is returned for for data types where the column size is not applicable. - */ - public Integer getColumnSize() { - if (!isScalarType()) return null; - if (isNumericType()) return getPrecision(); - ScalarType t = (ScalarType) this; - switch (t.getPrimitiveType()) { - case STRING: - return Integer.MAX_VALUE; - case TIMESTAMP: - return 29; - case CHAR: - case VARCHAR: - return t.getLength(); - default: - return null; - } - } - - /** - * JDBC data type description - * For numeric types, returns the maximum precision for this type. - * For non-numeric types, returns null. - */ - public Integer getPrecision() { - if (!isScalarType()) return null; - ScalarType t = (ScalarType) this; - switch (t.getPrimitiveType()) { - case TINYINT: - return 3; - case SMALLINT: - return 5; - case INT: - return 10; - case BIGINT: - return 19; - case FLOAT: - return 7; - case DOUBLE: - return 15; - case DECIMAL: - return t.decimalPrecision(); - default: - return null; - } - } - - /** - * JDBC data type description - * Returns the number of fractional digits for this type, or null if not applicable. - * For timestamp/time types, returns the number of digits in the fractional seconds - * component. - */ - public Integer getDecimalDigits() { - if (!isScalarType()) return null; - ScalarType t = (ScalarType) this; - switch (t.getPrimitiveType()) { - case BOOLEAN: - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: - return 0; - case FLOAT: - return 7; - case DOUBLE: - return 15; - case TIMESTAMP: - return 9; - case DECIMAL: - return t.decimalScale(); - default: - return null; - } - } - - /** - * JDBC data type description - * For numeric data types, either 10 or 2. If it is 10, the values in COLUMN_SIZE - * and DECIMAL_DIGITS give the number of decimal digits allowed for the column. - * For example, a DECIMAL(12,5) column would return a NUM_PREC_RADIX of 10, - * a COLUMN_SIZE of 12, and a DECIMAL_DIGITS of 5; a FLOAT column could return - * a NUM_PREC_RADIX of 10, a COLUMN_SIZE of 15, and a DECIMAL_DIGITS of NULL. - * If it is 2, the values in COLUMN_SIZE and DECIMAL_DIGITS give the number of bits - * allowed in the column. For example, a FLOAT column could return a RADIX of 2, - * a COLUMN_SIZE of 53, and a DECIMAL_DIGITS of NULL. NULL is returned for data - * types where NUM_PREC_RADIX is not applicable. - */ - public Integer getNumPrecRadix() { - if (!isScalarType()) return null; - ScalarType t = (ScalarType) this; - switch (t.getPrimitiveType()) { - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: - case FLOAT: - case DOUBLE: - case DECIMAL: - return 10; - default: - // everything else (including boolean and string) is null - return null; - } - } - - /** - * JDBC data type description - * Returns the java SQL type enum - */ - public int getJavaSqlType() { - if (isStructType()) return java.sql.Types.STRUCT; - // Both MAP and ARRAY are reported as ARRAY, since there is no better matching - // Java SQL type. This behavior is consistent with Hive. - if (isCollectionType()) return java.sql.Types.ARRAY; - - Preconditions.checkState(isScalarType(), "Invalid non-scalar type: " + toSql()); - ScalarType t = (ScalarType) this; - switch (t.getPrimitiveType()) { - case NULL_TYPE: return java.sql.Types.NULL; - case BOOLEAN: return java.sql.Types.BOOLEAN; - case TINYINT: return java.sql.Types.TINYINT; - case SMALLINT: return java.sql.Types.SMALLINT; - case INT: return java.sql.Types.INTEGER; - case BIGINT: return java.sql.Types.BIGINT; - case FLOAT: return java.sql.Types.FLOAT; - case DOUBLE: return java.sql.Types.DOUBLE; - case TIMESTAMP: return java.sql.Types.TIMESTAMP; - case STRING: return java.sql.Types.VARCHAR; - case CHAR: return java.sql.Types.CHAR; - case VARCHAR: return java.sql.Types.VARCHAR; - case BINARY: return java.sql.Types.BINARY; - case DECIMAL: return java.sql.Types.DECIMAL; - default: - Preconditions.checkArgument(false, "Invalid primitive type " + - t.getPrimitiveType().name()); - return 0; - } - } - - /** - * Matrix that records "smallest" assignment-compatible type of two types - * (INVALID_TYPE if no such type exists, ie, if the input types are fundamentally - * incompatible). A value of any of the two types could be assigned to a slot - * of the assignment-compatible type. For strict compatibility, this can be done - * without any loss of precision. For non-strict compatibility, there may be loss of - * precision, e.g. if converting from BIGINT to FLOAT. - * - * We chose not to follow MySQL's type casting behavior as described here: - * http://dev.mysql.com/doc/refman/5.0/en/type-conversion.html - * for the following reasons: - * conservative casting in arithmetic exprs: TINYINT + TINYINT -> BIGINT - * comparison of many types as double: INT < FLOAT -> comparison as DOUBLE - * special cases when dealing with dates and timestamps. - */ - protected static PrimitiveType[][] compatibilityMatrix; - - /** - * If we are checking in strict mode, any non-null entry in this matrix overrides - * compatibilityMatrix. If the entry is null, the entry in compatibility matrix - * is valid. - */ - protected static PrimitiveType[][] strictCompatibilityMatrix; - - static { - compatibilityMatrix = new - PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length]; - strictCompatibilityMatrix = new - PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length]; - - for (int i = 0; i < PrimitiveType.values().length; ++i) { - // Each type is compatible with itself. - compatibilityMatrix[i][i] = PrimitiveType.values()[i]; - // BINARY is not supported. - compatibilityMatrix[BINARY.ordinal()][i] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[i][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE; - } - - compatibilityMatrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; - compatibilityMatrix[BOOLEAN.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; - compatibilityMatrix[BOOLEAN.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[BOOLEAN.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - compatibilityMatrix[BOOLEAN.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; - compatibilityMatrix[TINYINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[TINYINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 8 bit integer fits in mantissa of both float and double. - compatibilityMatrix[TINYINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[SMALLINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 16 bit integer fits in mantissa of both float and double. - compatibilityMatrix[SMALLINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 32 bit integer fits only mantissa of double. - // TODO: arguably we should promote INT + FLOAT to DOUBLE to avoid loss of precision, - // but we depend on it remaining FLOAT for some use cases, e.g. - // "insert into tbl (float_col) select int_col + float_col from ..." - compatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - strictCompatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - // 64 bit integer does not fit in mantissa of double or float. - // TODO: arguably we should always promote BIGINT + FLOAT to double here to keep as - // much precision as possible, but we depend on this implicit cast for some use - // cases, similarly to INT + FLOAT. - compatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - strictCompatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; - // TODO: we're breaking the definition of strict compatibility for BIGINT + DOUBLE, - // but this forces function overloading to consider the DOUBLE overload first. - compatibilityMatrix[BIGINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[BIGINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[FLOAT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIME; - compatibilityMatrix[DATE.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.TIMESTAMP; - compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[DATETIME.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.TIMESTAMP; - compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - - // We can convert some but not all string values to timestamps. - compatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] = - PrimitiveType.TIMESTAMP; - strictCompatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TIMESTAMP.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TIMESTAMP.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[STRING.ordinal()][VARCHAR.ordinal()] = PrimitiveType.STRING; - compatibilityMatrix[STRING.ordinal()][CHAR.ordinal()] = PrimitiveType.STRING; - - compatibilityMatrix[VARCHAR.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - // Check all of the necessary entries that should be filled. - for (int i = 0; i < PrimitiveType.values().length; ++i) { - for (int j = i; j < PrimitiveType.values().length; ++j) { - PrimitiveType t1 = PrimitiveType.values()[i]; - PrimitiveType t2 = PrimitiveType.values()[j]; - // DECIMAL, NULL, and INVALID_TYPE are handled separately. - if (t1 == PrimitiveType.INVALID_TYPE || - t2 == PrimitiveType.INVALID_TYPE) continue; - if (t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE) continue; - if (t1 == PrimitiveType.DECIMAL || t2 == PrimitiveType.DECIMAL) continue; - Preconditions.checkNotNull(compatibilityMatrix[i][j]); - } - } - } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/View.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/View.java b/fe/src/main/java/com/cloudera/impala/catalog/View.java deleted file mode 100644 index cc82f95..0000000 --- a/fe/src/main/java/com/cloudera/impala/catalog/View.java +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.catalog; - -import java.io.StringReader; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import com.cloudera.impala.analysis.ParseNode; -import com.cloudera.impala.analysis.QueryStmt; -import com.cloudera.impala.analysis.SqlParser; -import com.cloudera.impala.analysis.SqlScanner; -import com.cloudera.impala.thrift.TCatalogObjectType; -import com.cloudera.impala.thrift.TTable; -import com.cloudera.impala.thrift.TTableDescriptor; -import com.cloudera.impala.thrift.TTableType; -import com.google.common.collect.Lists; - -/** - * Table metadata representing a catalog view or a local view from a WITH clause. - * Most methods inherited from Table are not supposed to be called on this class because - * views are substituted with their underlying definition during analysis of a statement. - * - * Refreshing or invalidating a view will reload the view's definition but will not - * affect the metadata of the underlying tables (if any). - */ -public class View extends Table { - - // The original SQL-string given as view definition. Set during analysis. - // Corresponds to Hive's viewOriginalText. - private String originalViewDef_; - - // Query statement (as SQL string) that defines the View for view substitution. - // It is a transformation of the original view definition, e.g., to enforce the - // explicit column definitions even if the original view definition has explicit - // column aliases. - // If column definitions were given, then this "expanded" view definition - // wraps the original view definition in a select stmt as follows. - // - // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ... - // FROM (originalViewDef) AS viewName - // - // Corresponds to Hive's viewExpandedText, but is not identical to the SQL - // Hive would produce in view creation. - private String inlineViewDef_; - - // View definition created by parsing inlineViewDef_ into a QueryStmt. - private QueryStmt queryStmt_; - - // Set if this View is from a WITH clause and not persisted in the catalog. - private final boolean isLocalView_; - - // Set if this View is from a WITH clause with column labels. - private List<String> colLabels_; - - public View(TableId id, org.apache.hadoop.hive.metastore.api.Table msTable, - Db db, String name, String owner) { - super(id, msTable, db, name, owner); - isLocalView_ = false; - } - - /** - * C'tor for WITH-clause views that already have a parsed QueryStmt and an optional - * list of column labels. - */ - public View(String alias, QueryStmt queryStmt, List<String> colLabels) { - super(null, null, null, alias, null); - isLocalView_ = true; - queryStmt_ = queryStmt; - colLabels_ = colLabels; - } - - /** - * Creates a view for testig purposes. - */ - private View(Db db, String name, QueryStmt queryStmt) { - super(null, null, db, name, null); - isLocalView_ = false; - queryStmt_ = queryStmt; - colLabels_ = null; - } - - @Override - public void load(boolean reuseMetadata, IMetaStoreClient client, - org.apache.hadoop.hive.metastore.api.Table msTbl) throws TableLoadingException { - try { - clearColumns(); - msTable_ = msTbl; - // Load columns. - List<FieldSchema> fieldSchemas = client.getFields(db_.getName(), name_); - for (int i = 0; i < fieldSchemas.size(); ++i) { - FieldSchema s = fieldSchemas.get(i); - Type type = parseColumnType(s); - Column col = new Column(s.getName(), type, s.getComment(), i); - addColumn(col); - } - // These fields are irrelevant for views. - numClusteringCols_ = 0; - numRows_ = -1; - init(); - } catch (TableLoadingException e) { - throw e; - } catch (Exception e) { - throw new TableLoadingException("Failed to load metadata for view: " + name_, e); - } - } - - @Override - protected void loadFromThrift(TTable t) throws TableLoadingException { - super.loadFromThrift(t); - init(); - } - - /** - * Initializes the originalViewDef_, inlineViewDef_, and queryStmt_ members - * by parsing the expanded view definition SQL-string. - * Throws a TableLoadingException if there was any error parsing the - * the SQL or if the view definition did not parse into a QueryStmt. - */ - private void init() throws TableLoadingException { - // Set view-definition SQL strings. - originalViewDef_ = getMetaStoreTable().getViewOriginalText(); - inlineViewDef_ = getMetaStoreTable().getViewExpandedText(); - // Parse the expanded view definition SQL-string into a QueryStmt and - // populate a view definition. - SqlScanner input = new SqlScanner(new StringReader(inlineViewDef_)); - SqlParser parser = new SqlParser(input); - ParseNode node = null; - try { - node = (ParseNode) parser.parse().value; - } catch (Exception e) { - // Do not pass e as the exception cause because it might reveal the existence - // of tables that the user triggering this load may not have privileges on. - throw new TableLoadingException( - String.format("Failed to parse view-definition statement of view: " + - "%s.%s", db_.getName(), name_)); - } - // Make sure the view definition parses to a query statement. - if (!(node instanceof QueryStmt)) { - throw new TableLoadingException(String.format("View definition of %s.%s " + - "is not a query statement", db_.getName(), name_)); - } - queryStmt_ = (QueryStmt) node; - } - - @Override - public TCatalogObjectType getCatalogObjectType() { return TCatalogObjectType.VIEW; } - public QueryStmt getQueryStmt() { return queryStmt_; } - public String getOriginalViewDef() { return originalViewDef_; } - public String getInlineViewDef() { return inlineViewDef_; } - public boolean isLocalView() { return isLocalView_; } - - /** - * Returns the column labels the user specified in the WITH-clause. - */ - public List<String> getOriginalColLabels() { return colLabels_; } - - /** - * Returns the explicit column labels for this view, or null if they need to be derived - * entirely from the underlying query statement. The returned list has at least as many - * elements as the number of column labels in the query stmt. - */ - public List<String> getColLabels() { - if (colLabels_ == null) return null; - if (colLabels_.size() >= queryStmt_.getColLabels().size()) return colLabels_; - List<String> explicitColLabels = Lists.newArrayList(colLabels_); - explicitColLabels.addAll(queryStmt_.getColLabels().subList( - colLabels_.size(), queryStmt_.getColLabels().size())); - return explicitColLabels; - } - - public boolean hasColLabels() { return colLabels_ != null; } - - @Override - public TTableDescriptor toThriftDescriptor(Set<Long> referencedPartitions) { - throw new IllegalStateException("Cannot call toThriftDescriptor() on a view."); - } - - @Override - public TTable toThrift() { - TTable view = super.toThrift(); - view.setTable_type(TTableType.VIEW); - return view; - } - - public static View createTestView(Db db, String name, QueryStmt viewDefStmt) { - return new View(db, name, viewDefStmt); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java deleted file mode 100644 index a21bd90..0000000 --- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.catalog.delegates; - -import java.util.List; - -import com.cloudera.impala.thrift.TDistributeParam; -import org.apache.hadoop.hive.metastore.api.Table; - -import com.cloudera.impala.common.ImpalaRuntimeException; -import com.cloudera.impala.thrift.TAlterTableParams; - -/** - * Abstract class to implement the storage specific portion of DDL requests. - * - * During catalog DDL operations the CatalogOpExecutor will instantiate the correct - * subclass of this class to handle the DDL operation to the storage backend. See, - * CatalogOpExecutor::createDDLDelegate() for details. - * - */ -public abstract class DdlDelegate { - - protected Table msTbl_; - protected TAlterTableParams tAlterTableParams_; - protected List<TDistributeParam> distributeParams_; - - /** - * Creates a new delegate to modify Table 'msTbl'. - */ - public DdlDelegate setMsTbl(Table msTbl) { - msTbl_ = msTbl; - return this; - } - - public DdlDelegate setAlterTableParams(TAlterTableParams p) { - tAlterTableParams_ = p; - return this; - } - - public DdlDelegate setDistributeParams(List<TDistributeParam> p) { - distributeParams_ = p; - return this; - } - - /** - * Creates the table. - */ - public abstract void createTable() throws ImpalaRuntimeException; - - /** - * Drops the table. - */ - public abstract void dropTable() throws ImpalaRuntimeException; - - /** - * Performs an alter table with the parameters set with setAlterTableParams(). - */ - public abstract boolean alterTable() throws ImpalaRuntimeException; - -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java deleted file mode 100644 index ecfeb1a..0000000 --- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.catalog.delegates; - -import static com.cloudera.impala.util.KuduUtil.compareSchema; -import static com.cloudera.impala.util.KuduUtil.fromImpalaType; -import static com.cloudera.impala.util.KuduUtil.parseKeyColumns; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder; -import org.apache.kudu.Schema; -import org.apache.kudu.Type; -import org.apache.kudu.client.CreateTableOptions; -import org.apache.kudu.client.KuduClient; -import org.apache.kudu.client.PartialRow; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.catalog.KuduTable; -import com.cloudera.impala.common.ImpalaRuntimeException; -import com.cloudera.impala.thrift.TDistributeParam; -import com.cloudera.impala.util.KuduUtil; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - - -/** - * Implementation of the Kudu DDL Delegate. Propagates create and drop table statements to - * Kudu. - */ -public class KuduDdlDelegate extends DdlDelegate { - - private static final Logger LOG = LoggerFactory.getLogger(KuduDdlDelegate.class); - - public KuduDdlDelegate(Table msTbl) { - setMsTbl(msTbl); - } - - /** - * Creates the Kudu table if it does not exist and returns true. If the table exists and - * the table is not a managed table ignore and return false, otherwise throw an - * exception. - */ - @Override - public void createTable() - throws ImpalaRuntimeException { - - String kuduTableName = msTbl_.getParameters().get(KuduTable.KEY_TABLE_NAME); - String kuduMasters = msTbl_.getParameters().get(KuduTable.KEY_MASTER_ADDRESSES); - - // Can be optional for un-managed tables - String kuduKeyCols = msTbl_.getParameters().get(KuduTable.KEY_KEY_COLUMNS); - - String replication = msTbl_.getParameters().get(KuduTable.KEY_TABLET_REPLICAS); - - try (KuduClient client = new KuduClient.KuduClientBuilder(kuduMasters).build()) { - // TODO should we throw if the table does not exist when its an external table? - if (client.tableExists(kuduTableName)) { - if (msTbl_.getTableType().equals(TableType.MANAGED_TABLE.toString())) { - throw new ImpalaRuntimeException(String.format( - "Table %s already exists in Kudu master %s.", kuduTableName, kuduMasters)); - } - - // Check if the external table matches the schema - org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName); - if (!compareSchema(msTbl_, kuduTable)) { - throw new ImpalaRuntimeException(String.format( - "Table %s (%s) has a different schema in Kudu than in Hive.", - msTbl_.getTableName(), kuduTableName)); - } - return; - } - - HashSet<String> keyColNames = parseKeyColumns(kuduKeyCols); - List<ColumnSchema> keyColSchemas = new ArrayList<>(); - - // Create a new Schema and map the types accordingly - ArrayList<ColumnSchema> columns = Lists.newArrayList(); - for (FieldSchema fieldSchema: msTbl_.getSd().getCols()) { - com.cloudera.impala.catalog.Type catalogType = com.cloudera.impala.catalog.Type - .parseColumnType(fieldSchema.getType()); - if (catalogType == null) { - throw new ImpalaRuntimeException(String.format( - "Could not parse column type %s.", fieldSchema.getType())); - } - Type t = fromImpalaType(catalogType); - // Create the actual column and check if the column is a key column - ColumnSchemaBuilder csb = new ColumnSchemaBuilder( - fieldSchema.getName(), t); - boolean isKeyColumn = keyColNames.contains(fieldSchema.getName()); - csb.key(isKeyColumn); - csb.nullable(!isKeyColumn); - ColumnSchema cs = csb.build(); - columns.add(cs); - if (isKeyColumn) keyColSchemas.add(cs); - } - - Schema schema = new Schema(columns); - CreateTableOptions cto = new CreateTableOptions(); - - // Handle auto-partitioning of the Kudu table - if (distributeParams_ != null) { - for (TDistributeParam param : distributeParams_) { - if (param.isSetBy_hash_param()) { - Preconditions.checkState(!param.isSetBy_range_param()); - cto.addHashPartitions(param.getBy_hash_param().getColumns(), - param.getBy_hash_param().getNum_buckets()); - } else { - Preconditions.checkState(param.isSetBy_range_param()); - cto.setRangePartitionColumns(param.getBy_range_param().getColumns()); - for (PartialRow p : KuduUtil.parseSplits(schema, param.getBy_range_param())) { - cto.addSplitRow(p); - } - } - } - } - - if (!Strings.isNullOrEmpty(replication)) { - int r = Integer.parseInt(replication); - if (r <= 0) { - throw new ImpalaRuntimeException( - "Number of tablet replicas must be greater than zero. " + - "Given number of replicas is: " + Integer.toString(r)); - } - cto.setNumReplicas(r); - } - - client.createTable(kuduTableName, schema, cto); - } catch (ImpalaRuntimeException e) { - throw e; - } catch (Exception e) { - throw new ImpalaRuntimeException("Error creating Kudu table", e); - } - } - - @Override - public void dropTable() throws ImpalaRuntimeException { - // If table is an external table, do not delete the data - if (msTbl_.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) return; - - String kuduTableName = msTbl_.getParameters().get(KuduTable.KEY_TABLE_NAME); - String kuduMasters = msTbl_.getParameters().get(KuduTable.KEY_MASTER_ADDRESSES); - - try (KuduClient client = new KuduClient.KuduClientBuilder(kuduMasters).build()) { - if (!client.tableExists(kuduTableName)) { - LOG.warn("Table: %s is in inconsistent state. It does not exist in Kudu master(s)" - + " %s, but it exists in Hive metastore. Deleting from metastore only.", - kuduTableName, kuduMasters); - return; - } - client.deleteTable(kuduTableName); - return; - } catch (Exception e) { - throw new ImpalaRuntimeException("Error dropping Kudu table", e); - } - } - - public static boolean canHandle(org.apache.hadoop.hive.metastore.api.Table msTbl) { - return KuduTable.isKuduTable(msTbl); - } - - @Override - public boolean alterTable() throws ImpalaRuntimeException { - throw new ImpalaRuntimeException( - "Alter table operations are not supported for Kudu tables."); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java deleted file mode 100644 index 52267be..0000000 --- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.catalog.delegates; - -import com.cloudera.impala.common.ImpalaRuntimeException; - -/** - * Empty implementation for the DdlDelegate interface that does nothing. - */ -public class UnsupportedOpDelegate extends DdlDelegate { - - @Override - public void createTable() throws ImpalaRuntimeException { } - - @Override - public void dropTable() throws ImpalaRuntimeException { } - - @Override - public boolean alterTable() throws ImpalaRuntimeException { return true; } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java b/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java deleted file mode 100644 index 2d75d18..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.util.Set; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; - -/** - * Abstract class representing an alias generator. It uses a prefix and a - * monotonically increasing counter to generate new aliases. Classes extending - * this class are responsible for initializing the prefix. - */ -public abstract class AliasGenerator { - private int numGeneratedAliases_ = 1; - protected String aliasPrefix_ = null; - protected Set<String> usedAliases_ = Sets.newHashSet(); - - /** - * Return the next available alias. - */ - public String getNextAlias() { - Preconditions.checkNotNull(aliasPrefix_); - while (true) { - String candidateAlias = aliasPrefix_ + Integer.toString(numGeneratedAliases_++); - if (usedAliases_.add(candidateAlias)) return candidateAlias; - if (numGeneratedAliases_ < 0) { - throw new IllegalStateException("Overflow occured during alias generation."); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java b/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java deleted file mode 100644 index 464cfa0..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Thrown for errors encountered during analysis of a SQL statement. - * - */ -public class AnalysisException extends ImpalaException { - public AnalysisException(String msg, Throwable cause) { - super(msg, cause); - } - - public AnalysisException(String msg) { - super(msg); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java b/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java deleted file mode 100644 index f8b43ab..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Byte unit constants. - */ -public class ByteUnits { - - /** - * One kilobyte in bytes. - */ - public final static long KILOBYTE = 1024; - - /** - * One megabyte in bytes. - */ - public final static long MEGABYTE = KILOBYTE * 1024; - - /** - * One gigabyte in bytes. - */ - public final static long GIGABYTE = MEGABYTE * 1024; - - /** - * One terabyte in bytes. - */ - public final static long TERABYTE = GIGABYTE * 1024; - - /** - * One petabyte in bytes. - */ - public final static long PETABYTE = TERABYTE * 1024; -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java b/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java deleted file mode 100644 index 51c8ece..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.util.List; - -import com.google.common.base.Preconditions; - -public class ColumnAliasGenerator extends AliasGenerator { - private static final String DEFAULT_COL_ALIAS_PREFIX = "$c$"; - - public ColumnAliasGenerator(List<String> existingLabels, String prefix) { - Preconditions.checkNotNull(existingLabels); - aliasPrefix_ = prefix != null ? prefix : DEFAULT_COL_ALIAS_PREFIX; - usedAliases_.addAll(existingLabels); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java deleted file mode 100644 index 2239853..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java +++ /dev/null @@ -1,409 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.UUID; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3.S3FileSystem; -import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3native.NativeS3FileSystem; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.client.HdfsAdmin; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; -import org.apache.log4j.Logger; - -import com.google.common.base.Preconditions; - -/** - * Common utility functions for operating on FileSystem objects. - */ -public class FileSystemUtil { - private static final Configuration CONF = new Configuration(); - private static final Logger LOG = Logger.getLogger(FileSystemUtil.class); - - /** - * Performs a non-recursive delete of all visible (non-hidden) files in a given - * directory. Returns the number of files deleted as part of this operation. - */ - public static int deleteAllVisibleFiles(Path directory) - throws IOException { - FileSystem fs = directory.getFileSystem(CONF); - Preconditions.checkState(fs.getFileStatus(directory).isDirectory()); - int numFilesDeleted = 0; - for (FileStatus fStatus: fs.listStatus(directory)) { - // Only delete files that are not hidden. - if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) { - LOG.debug("Removing: " + fStatus.getPath()); - fs.delete(fStatus.getPath(), false); - ++numFilesDeleted; - } - } - return numFilesDeleted; - } - - /** - * Returns the total number of visible (non-hidden) files in a directory. - */ - public static int getTotalNumVisibleFiles(Path directory) throws IOException { - FileSystem fs = directory.getFileSystem(CONF); - Preconditions.checkState(fs.getFileStatus(directory).isDirectory()); - int numFiles = 0; - for (FileStatus fStatus: fs.listStatus(directory)) { - // Only delete files that are not hidden. - if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) { - ++numFiles; - } - } - return numFiles; - } - - /** - * Returns true if path p1 and path p2 are in the same encryption zone in HDFS. - * Returns false if they are in different encryption zones or if either of the paths - * are not on HDFS. - */ - private static boolean arePathsInSameHdfsEncryptionZone(FileSystem fs, Path p1, - Path p2) throws IOException { - // Only distributed file systems have encryption zones. - if (!isDistributedFileSystem(p1) || !isDistributedFileSystem(p2)) return false; - HdfsAdmin hdfsAdmin = new HdfsAdmin(fs.getUri(), CONF); - EncryptionZone z1 = hdfsAdmin.getEncryptionZoneForPath(p1); - EncryptionZone z2 = hdfsAdmin.getEncryptionZoneForPath(p2); - if (z1 == null && z2 == null) return true; - if (z1 == null || z2 == null) return false; - return z1.equals(z2); - } - - /** - * Relocates all visible (non-hidden) files from a source directory to a destination - * directory. Files are moved (renamed) to the new location unless the source and - * destination directories are in different encryption zones, in which case the files - * are copied so that they are decrypted and/or encrypted. Naming conflicts are - * resolved by appending a UUID to the base file name. Any sub-directories within the - * source directory are skipped. Returns the number of files relocated as part of this - * operation. - */ - public static int relocateAllVisibleFiles(Path sourceDir, Path destDir) - throws IOException { - FileSystem destFs = destDir.getFileSystem(CONF); - FileSystem sourceFs = sourceDir.getFileSystem(CONF); - Preconditions.checkState(destFs.isDirectory(destDir)); - Preconditions.checkState(sourceFs.isDirectory(sourceDir)); - - // Use the same UUID to resolve all file name conflicts. This helps mitigate problems - // that might happen if there is a conflict moving a set of files that have - // dependent file names. For example, foo.lzo and foo.lzo_index. - UUID uuid = UUID.randomUUID(); - - // Enumerate all the files in the source - int numFilesMoved = 0; - for (FileStatus fStatus: sourceFs.listStatus(sourceDir)) { - if (fStatus.isDirectory()) { - LOG.debug("Skipping copy of directory: " + fStatus.getPath()); - continue; - } else if (isHiddenFile(fStatus.getPath().getName())) { - continue; - } - - Path destFile = new Path(destDir, fStatus.getPath().getName()); - if (destFs.exists(destFile)) { - destFile = new Path(destDir, - appendToBaseFileName(destFile.getName(), uuid.toString())); - } - FileSystemUtil.relocateFile(fStatus.getPath(), destFile, false); - ++numFilesMoved; - } - return numFilesMoved; - } - - /** - * Relocates the given file to a new location (either another directory or a - * file in the same or different filesystem). The file is generally moved (renamed) to - * the new location. However, the file is copied if the source and destination are in - * different encryption zones so that the file can be decrypted and/or encrypted, or if - * the source and destination are in different filesystems. If renameIfAlreadyExists is - * true, no error will be thrown if a file with the same name already exists in the - * destination location. Instead, a UUID will be appended to the base file name, - * preserving the existing file extension. If renameIfAlreadyExists is false, an - * IOException will be thrown if there is a file name conflict. - */ - public static void relocateFile(Path sourceFile, Path dest, - boolean renameIfAlreadyExists) throws IOException { - FileSystem destFs = dest.getFileSystem(CONF); - FileSystem sourceFs = sourceFile.getFileSystem(CONF); - - Path destFile = - destFs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest; - // If a file with the same name does not already exist in the destination location - // then use the same file name. Otherwise, generate a unique file name. - if (renameIfAlreadyExists && destFs.exists(destFile)) { - Path destDir = destFs.isDirectory(dest) ? dest : dest.getParent(); - destFile = new Path(destDir, - appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString())); - } - boolean sameFileSystem = isPathOnFileSystem(sourceFile, destFs); - boolean destIsDfs = isDistributedFileSystem(destFs); - - // If the source and the destination are on different file systems, or in different - // encryption zones, files can't be moved from one location to the other and must be - // copied instead. - boolean sameEncryptionZone = - arePathsInSameHdfsEncryptionZone(destFs, sourceFile, destFile); - // We can do a rename if the src and dst are in the same encryption zone in the same - // distributed filesystem. - boolean doRename = destIsDfs && sameFileSystem && sameEncryptionZone; - // Alternatively, we can do a rename if the src and dst are on the same - // non-distributed filesystem. - if (!doRename) doRename = !destIsDfs && sameFileSystem; - if (doRename) { - LOG.debug(String.format( - "Moving '%s' to '%s'", sourceFile.toString(), destFile.toString())); - // Move (rename) the file. - destFs.rename(sourceFile, destFile); - return; - } - if (destIsDfs && sameFileSystem) { - Preconditions.checkState(!doRename); - // We must copy rather than move if the source and dest are in different - // encryption zones. A move would return an error from the NN because a move is a - // metadata-only operation and the files would not be encrypted/decrypted properly - // on the DNs. - LOG.info(String.format( - "Copying source '%s' to '%s' because HDFS encryption zones are different.", - sourceFile, destFile)); - } else { - Preconditions.checkState(!sameFileSystem); - LOG.info(String.format("Copying '%s' to '%s' between filesystems.", - sourceFile, destFile)); - } - FileUtil.copy(sourceFs, sourceFile, destFs, destFile, true, true, CONF); - } - - /** - * Reads the file at path and returns the contents. - */ - public static String readFile(Path file) throws IOException { - FileSystem fs = file.getFileSystem(CONF); - InputStream fileStream = fs.open(file); - try { - return IOUtils.toString(fileStream); - } finally { - IOUtils.closeQuietly(fileStream); - } - } - - /** - * Builds a new file name based on a base file name. This is done by inserting - * the given appendStr into the base file name, preserving the file extension (if - * one exists). - * For example, this could be passed a UUID string to uniquify files: - * file1.snap -> file1_<uuid>.snap - * file1 -> file1_<uuid> - */ - private static String appendToBaseFileName(String baseFileName, String appendStr) { - StringBuilder sb = new StringBuilder(baseFileName); - // Insert the string to append, preserving the file extension. - int extensionIdx = baseFileName.lastIndexOf('.'); - if (extensionIdx != -1) { - sb.replace(extensionIdx, extensionIdx + 1, "_" + appendStr + "."); - } else { - sb.append("_" + appendStr); - } - return sb.toString(); - } - - /** - * Returns true if the given Path contains any visible sub directories, otherwise false. - */ - public static boolean containsVisibleSubdirectory(Path directory) - throws FileNotFoundException, IOException { - FileSystem fs = directory.getFileSystem(CONF); - // Enumerate all the files in the source - for (FileStatus fStatus: fs.listStatus(directory)) { - String pathName = fStatus.getPath().getName(); - if (fStatus.isDirectory() && !isHiddenFile(pathName)) { - return true; - } - } - return false; - } - - /** - * Makes a temporary unique directory within the given directory. - */ - public static Path makeTmpSubdirectory(Path directory) throws IOException { - FileSystem fs = directory.getFileSystem(CONF); - Path tmpDir = new Path(directory, ".tmp_" + UUID.randomUUID().toString()); - fs.mkdirs(tmpDir); - return tmpDir; - } - - public static boolean isHiddenFile(String fileName) { - // Hidden files start with '.' or '_'. The '.copying' suffix is used by some - // filesystem utilities (e.g. hdfs put) as a temporary destination when copying - // files. The '.tmp' suffix is Flume's default for temporary files. - String lcFileName = fileName.toLowerCase(); - return lcFileName.startsWith(".") || lcFileName.startsWith("_") || - lcFileName.endsWith(".copying") || lcFileName.endsWith(".tmp"); - } - - /** - * Returns true if the filesystem might override getFileBlockLocations(). - */ - public static boolean hasGetFileBlockLocations(FileSystem fs) { - // Common case. - if (isDistributedFileSystem(fs)) return true; - // Blacklist FileSystems that are known to not implement getFileBlockLocations(). - return !(fs instanceof S3AFileSystem || fs instanceof NativeS3FileSystem || - fs instanceof S3FileSystem || fs instanceof LocalFileSystem); - } - - /** - * Returns true iff the filesystem is a S3AFileSystem. - */ - public static boolean isS3AFileSystem(FileSystem fs) { - return fs instanceof S3AFileSystem; - } - - /** - * Returns true iff the path is on a S3AFileSystem. - */ - public static boolean isS3AFileSystem(Path path) throws IOException { - return isS3AFileSystem(path.getFileSystem(CONF)); - } - - /** - * Returns true iff the filesystem is an instance of LocalFileSystem. - */ - public static boolean isLocalFileSystem(FileSystem fs) { - return fs instanceof LocalFileSystem; - } - - /** - * Return true iff path is on a local filesystem. - */ - public static boolean isLocalFileSystem(Path path) throws IOException { - return isLocalFileSystem(path.getFileSystem(CONF)); - } - - /** - * Returns true iff the filesystem is a DistributedFileSystem. - */ - public static boolean isDistributedFileSystem(FileSystem fs) { - return fs instanceof DistributedFileSystem; - } - - /** - * Return true iff path is on a DFS filesystem. - */ - public static boolean isDistributedFileSystem(Path path) throws IOException { - return isDistributedFileSystem(path.getFileSystem(CONF)); - } - - public static FileSystem getDefaultFileSystem() throws IOException { - Path path = new Path(FileSystem.getDefaultUri(CONF)); - FileSystem fs = path.getFileSystem(CONF); - return fs; - } - - public static DistributedFileSystem getDistributedFileSystem() throws IOException { - FileSystem fs = getDefaultFileSystem(); - Preconditions.checkState(fs instanceof DistributedFileSystem); - return (DistributedFileSystem) fs; - } - - /** - * Fully-qualifies the given path based on the FileSystem configuration. - */ - public static Path createFullyQualifiedPath(Path location) { - URI defaultUri = FileSystem.getDefaultUri(CONF); - URI locationUri = location.toUri(); - // Use the default URI only if location has no scheme or it has the same scheme as - // the default URI. Otherwise, Path.makeQualified() will incorrectly use the - // authority from the default URI even though the schemes don't match. See HDFS-7031. - if (locationUri.getScheme() == null || - locationUri.getScheme().equalsIgnoreCase(defaultUri.getScheme())) { - return location.makeQualified(defaultUri, location); - } - // Already qualified (has scheme). - return location; - } - - /** - * Return true iff the path is on the given filesystem. - */ - public static boolean isPathOnFileSystem(Path path, FileSystem fs) { - try { - // Call makeQualified() for the side-effect of FileSystem.checkPath() which will - // throw an exception if path is not on fs. - fs.makeQualified(path); - return true; - } catch (IllegalArgumentException e) { - // Path is not on fs. - return false; - } - } - - /** - * Copies the source file to a destination path on the local filesystem. - * Throws IOException on failure. - */ - public static void copyToLocal(Path source, Path dest) throws IOException { - FileSystem fs = source.getFileSystem(CONF); - fs.copyToLocalFile(source, dest); - } - - /** - * Returns true if the given path is a location which supports caching (e.g. HDFS). - */ - public static boolean isPathCacheable(Path path) { - try { - return isDistributedFileSystem(path); - } catch (IOException e) { - return false; - } - } - - /** - * Returns the configuration. - */ - public static Configuration getConfiguration() { - return CONF; - } - - /** - * Returns true iff the given location is on a filesystem that Impala can write to. - */ - public static boolean isImpalaWritableFilesystem(String location) - throws IOException { - Path path = new Path(location); - return (FileSystemUtil.isDistributedFileSystem(path) || - FileSystemUtil.isLocalFileSystem(path) || FileSystemUtil.isS3AFileSystem(path)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/Id.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/Id.java b/fe/src/main/java/com/cloudera/impala/common/Id.java deleted file mode 100644 index 88f626f..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/Id.java +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.util.ArrayList; -import java.util.List; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; - -/** - * Integer ids that cannot accidentally be compared with ints. - */ -public class Id<IdType extends Id<IdType>> implements Comparable<Id<IdType>> { - static protected int INVALID_ID = -1; - protected final int id_; - - public Id(int id) { - this.id_ = id; - } - - public boolean isValid() { return id_ != INVALID_ID; } - public int asInt() { return id_; } - - @Override - public int hashCode() { - return Integer.valueOf(id_).hashCode(); - } - - @Override - public String toString() { - return Integer.toString(id_); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) return false; - // only ids of the same subclass are comparable - if (obj.getClass() != this.getClass()) return false; - return ((Id)obj).id_ == id_; - } - - @Override - public int compareTo(Id<IdType> cmp) { - return id_ - cmp.id_; - } - - public ArrayList<IdType> asList() { - ArrayList<IdType> list = new ArrayList<IdType>(); - list.add((IdType) this); - return list; - } - - public static <C extends Id> String printIds(List<C> ids) { - ArrayList<String> l = Lists.newArrayList(); - for (C id: ids) { - l.add(id.toString()); - } - return "(" + Joiner.on(" ").join(l) + ")"; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java b/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java deleted file mode 100644 index cdece7d..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Generator of consecutively numbered integers to be used as ids by subclasses of Id. - * Subclasses of Id should be able to create a generator for their Id type. - */ -public abstract class IdGenerator<IdType extends Id<IdType>> { - protected int nextId_ = 0; - public abstract IdType getNextId(); - public abstract IdType getMaxId(); -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java b/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java deleted file mode 100644 index 6a50bf8..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - - -/** - * The parent class of all custom Impala exceptions. - * - */ -abstract public class ImpalaException extends java.lang.Exception { - public ImpalaException(String msg, Throwable cause) { - super(msg, cause); - } - - protected ImpalaException(String msg) { - super(msg); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java b/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java deleted file mode 100644 index 27a8925..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Thrown for errors encountered during the execution of a SQL statement. - * - */ -public class ImpalaRuntimeException extends ImpalaException { - public ImpalaRuntimeException(String msg, Throwable cause) { - super(msg, cause); - } - - public ImpalaRuntimeException(String msg) { - super(msg); - } -}
