http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/InternalException.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/InternalException.java b/fe/src/main/java/com/cloudera/impala/common/InternalException.java deleted file mode 100644 index 64a8d5e..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/InternalException.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Thrown for internal server errors. - * - */ -public class InternalException extends ImpalaException { - public InternalException(String msg, Throwable cause) { - super(msg, cause); - } - - public InternalException(String msg) { - super(msg); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/JniUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/JniUtil.java b/fe/src/main/java/com/cloudera/impala/common/JniUtil.java deleted file mode 100644 index 2f11c52..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/JniUtil.java +++ /dev/null @@ -1,165 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.Writer; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryUsage; -import java.util.ArrayList; - -import org.apache.thrift.TBase; -import org.apache.thrift.TSerializer; -import org.apache.thrift.TDeserializer; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocolFactory; - -import com.cloudera.impala.thrift.TGetJvmMetricsRequest; -import com.cloudera.impala.thrift.TGetJvmMetricsResponse; -import com.cloudera.impala.thrift.TJvmMemoryPool; - -/** - * Utility class with methods intended for JNI clients - */ -public class JniUtil { - private final static TBinaryProtocol.Factory protocolFactory_ = - new TBinaryProtocol.Factory(); - - /** - * Returns a formatted string containing the simple exception name and the - * exception message without the full stack trace. Includes the - * the chain of causes each in a separate line. - */ - public static String throwableToString(Throwable t) { - Writer output = new StringWriter(); - try { - output.write(String.format("%s: %s", t.getClass().getSimpleName(), - t.getMessage())); - // Follow the chain of exception causes and print them as well. - Throwable cause = t; - while ((cause = cause.getCause()) != null) { - output.write(String.format("\nCAUSED BY: %s: %s", - cause.getClass().getSimpleName(), cause.getMessage())); - } - } catch (IOException e) { - throw new Error(e); - } - return output.toString(); - } - - /** - * Returns the stack trace of the Throwable object. - */ - public static String throwableToStackTrace(Throwable t) { - Writer output = new StringWriter(); - t.printStackTrace(new PrintWriter(output)); - return output.toString(); - } - - /** - * Deserialize a serialized form of a Thrift data structure to its object form. - */ - public static <T extends TBase<?, ?>, F extends TProtocolFactory> - void deserializeThrift(F protocolFactory, T result, byte[] thriftData) - throws ImpalaException { - // TODO: avoid creating deserializer for each query? - TDeserializer deserializer = new TDeserializer(protocolFactory); - try { - deserializer.deserialize(result, thriftData); - } catch (TException e) { - throw new InternalException(e.getMessage()); - } - } - - /** - * Collect the JVM's memory statistics into a thrift structure for translation into - * Impala metrics by the backend. A synthetic 'total' memory pool is included with - * aggregate statistics for all real pools. - */ - public static byte[] getJvmMetrics(byte[] argument) throws ImpalaException { - TGetJvmMetricsRequest request = new TGetJvmMetricsRequest(); - JniUtil.deserializeThrift(protocolFactory_, request, argument); - - TGetJvmMetricsResponse jvmMetrics = new TGetJvmMetricsResponse(); - jvmMetrics.setMemory_pools(new ArrayList<TJvmMemoryPool>()); - TJvmMemoryPool totalUsage = new TJvmMemoryPool(); - boolean is_total = - request.getMemory_pool() != null && request.getMemory_pool().equals("total"); - - if (request.get_all || is_total) { - totalUsage.setName("total"); - jvmMetrics.getMemory_pools().add(totalUsage); - } - for (MemoryPoolMXBean memBean: ManagementFactory.getMemoryPoolMXBeans()) { - if (request.get_all || is_total || - memBean.getName().equals(request.getMemory_pool())) { - TJvmMemoryPool usage = new TJvmMemoryPool(); - MemoryUsage beanUsage = memBean.getUsage(); - usage.setCommitted(beanUsage.getCommitted()); - usage.setInit(beanUsage.getInit()); - usage.setMax(beanUsage.getMax()); - usage.setUsed(beanUsage.getUsed()); - usage.setName(memBean.getName()); - - totalUsage.committed += beanUsage.getCommitted(); - totalUsage.init += beanUsage.getInit(); - totalUsage.max += beanUsage.getMax(); - totalUsage.used += beanUsage.getUsed(); - - MemoryUsage peakUsage = memBean.getPeakUsage(); - usage.setPeak_committed(peakUsage.getCommitted()); - usage.setPeak_init(peakUsage.getInit()); - usage.setPeak_max(peakUsage.getMax()); - usage.setPeak_used(peakUsage.getUsed()); - - totalUsage.peak_committed += peakUsage.getCommitted(); - totalUsage.peak_init += peakUsage.getInit(); - totalUsage.peak_max += peakUsage.getMax(); - totalUsage.peak_used += peakUsage.getUsed(); - - if (!is_total) { - jvmMetrics.getMemory_pools().add(usage); - if (!request.get_all) break; - } - } - } - TSerializer serializer = new TSerializer(protocolFactory_); - try { - return serializer.serialize(jvmMetrics); - } catch (TException e) { - throw new InternalException(e.getMessage()); - } - } - - /** - * Get Java version and vendor information - */ - public static String getJavaVersion() { - StringBuilder sb = new StringBuilder(); - sb.append("Java Version Info: "); - sb.append(System.getProperty("java.runtime.name")); - sb.append(" ("); - sb.append(System.getProperty("java.runtime.version")); - sb.append(")"); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/NotImplementedException.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/NotImplementedException.java b/fe/src/main/java/com/cloudera/impala/common/NotImplementedException.java deleted file mode 100644 index b06a130..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/NotImplementedException.java +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * Thrown for SQL statements that require as yet unimplemented functionality. - * - */ -public class NotImplementedException extends ImpalaException { - public NotImplementedException(String msg) { - super(msg); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/Pair.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/Pair.java b/fe/src/main/java/com/cloudera/impala/common/Pair.java deleted file mode 100644 index c17c535..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/Pair.java +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * The equivalent of C++'s std::pair<>. - */ -public class Pair<F, S> { - public F first; - public S second; - - public Pair(F first, S second) { - this.first = first; - this.second = second; - } - - @Override - /** - * A pair is equal if both parts are equal(). - */ - public boolean equals(Object o) { - if (o instanceof Pair) { - Pair<F,S> other = (Pair<F,S>) o; - return this.first.equals(other.first) && this.second.equals(other.second); - } - return false; - } - - @Override - public int hashCode() { - int hashFirst = first != null ? first.hashCode() : 0; - int hashSecond = second != null ? second.hashCode() : 0; - - return (hashFirst + hashSecond) * hashSecond + hashFirst; - } - - static public <F, S> Pair<F, S> create(F first, S second) { - return new Pair<F, S>(first, second); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/PrintUtils.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/PrintUtils.java b/fe/src/main/java/com/cloudera/impala/common/PrintUtils.java deleted file mode 100644 index 78e0d6d..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/PrintUtils.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import static com.cloudera.impala.common.ByteUnits.GIGABYTE; -import static com.cloudera.impala.common.ByteUnits.KILOBYTE; -import static com.cloudera.impala.common.ByteUnits.MEGABYTE; -import static com.cloudera.impala.common.ByteUnits.PETABYTE; -import static com.cloudera.impala.common.ByteUnits.TERABYTE; - -import java.text.DecimalFormat; - -import org.apache.commons.lang3.StringUtils; - -/** - * Utility functions for pretty printing. - */ -public class PrintUtils { - /** - * Prints the given number of bytes in PB, TB, GB, MB, KB with 2 decimal points. - * For example 5000 will be returned as 4.88KB. - */ - public static String printBytes(long bytes) { - double result = bytes; - // Avoid String.format() due to IMPALA-1572 which happens on JDK7 but not JDK6. - if (bytes >= PETABYTE) return new DecimalFormat(".00PB").format(result / PETABYTE); - if (bytes >= TERABYTE) return new DecimalFormat(".00TB").format(result / TERABYTE); - if (bytes >= GIGABYTE) return new DecimalFormat(".00GB").format(result / GIGABYTE); - if (bytes >= MEGABYTE) return new DecimalFormat(".00MB").format(result / MEGABYTE); - if (bytes >= KILOBYTE) return new DecimalFormat(".00KB").format(result / KILOBYTE); - return bytes + "B"; - } - - public static String printCardinality(String prefix, long cardinality) { - return prefix + "cardinality=" + - ((cardinality != -1) ? String.valueOf(cardinality) : "unavailable"); - } - - public static String printHosts(String prefix, long numHosts) { - return prefix + "hosts=" + ((numHosts != -1) ? numHosts : "unavailable"); - } - - public static String printMemCost(String prefix, long perHostMemCost) { - return prefix + "per-host-mem=" + - ((perHostMemCost != -1) ? printBytes(perHostMemCost) : "unavailable"); - } - - /** - * Prints the given square matrix into matrixStr. Separates cells by cellSpacing. - */ - public static void printMatrix(boolean[][] matrix, int cellSpacing, - StringBuilder matrixStr) { - // Print labels. - matrixStr.append(StringUtils.repeat(' ', cellSpacing)); - String formatStr = "%Xd".replace("X", String.valueOf(cellSpacing)); - for (int i = 0; i < matrix.length; ++i) { - matrixStr.append(String.format(formatStr, i)); - } - matrixStr.append("\n"); - - // Print matrix. - for (int i = 0; i < matrix.length; ++i) { - matrixStr.append(String.format(formatStr, i)); - for (int j = 0; j < matrix.length; ++j) { - int cell = (matrix[i][j]) ? 1 : 0; - matrixStr.append(String.format(formatStr, cell)); - } - matrixStr.append("\n"); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/Reference.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/Reference.java b/fe/src/main/java/com/cloudera/impala/common/Reference.java deleted file mode 100644 index aff1dae..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/Reference.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -/** - * "Indirection layer" that allows returning an object via an output - * parameter of a function call, similar to a pointer or reference parameter - * in C/C++. - * Example: - * Reference<T> ref = new Reference<T>(); - * createT(ref); // calls ref.setRef() - * <do something with ref.getRef()>; - */ -public class Reference<RefType> { - protected RefType ref_; - - public Reference(RefType ref) { - this.ref_ = ref; - } - - public Reference() { - this.ref_ = null; - } - - public RefType getRef() { return ref_; } - public void setRef(RefType ref) { this.ref_ = ref; } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/RuntimeEnv.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/RuntimeEnv.java b/fe/src/main/java/com/cloudera/impala/common/RuntimeEnv.java deleted file mode 100644 index febd657..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/RuntimeEnv.java +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.service.FeSupport; -import com.cloudera.impala.thrift.TStartupOptions; - -/** - * Contains runtime-specific parameters such as the number of CPU cores. Currently only - * used in Plan cost estimation. The static RuntimeEnv members can be set so that tests - * can rely on a machine-independent RuntimeEnv. - */ -public class RuntimeEnv { - private final static Logger LOG = LoggerFactory.getLogger(RuntimeEnv.class); - - public static RuntimeEnv INSTANCE = new RuntimeEnv(); - - private int numCores_; - - // Indicates if column lineage information should be computed for each query. - private boolean computeLineage_; - - // Indicates whether this is an environment for testing. - private boolean isTestEnv_; - - public RuntimeEnv() { - reset(); - try { - TStartupOptions opts = FeSupport.GetStartupOptions(); - computeLineage_ = opts.compute_lineage; - } catch (InternalException e) { - LOG.error("Error retrieving BE startup options. Shutting down JVM"); - System.exit(1); - } - } - - /** - * Resets this RuntimeEnv back to its machine-dependent state. - */ - public void reset() { - numCores_ = Runtime.getRuntime().availableProcessors(); - } - - public int getNumCores() { return numCores_; } - public void setNumCores(int numCores) { this.numCores_ = numCores; } - public void setTestEnv(boolean v) { isTestEnv_ = v; } - public boolean isTestEnv() { return isTestEnv_; } - public boolean computeLineage() { return computeLineage_; } - public boolean isKuduSupported() { - return "true".equals(System.getenv("KUDU_IS_SUPPORTED")); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/TableAliasGenerator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/TableAliasGenerator.java b/fe/src/main/java/com/cloudera/impala/common/TableAliasGenerator.java deleted file mode 100644 index 6ded288..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/TableAliasGenerator.java +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import com.cloudera.impala.analysis.Analyzer; - -import com.google.common.base.Preconditions; - -public class TableAliasGenerator extends AliasGenerator { - private static final String DEFAULT_TBL_ALIAS_PREFIX = "$a$"; - - public TableAliasGenerator(Analyzer analyzer, String prefix) { - Preconditions.checkNotNull(analyzer); - aliasPrefix_ = prefix != null ? prefix : DEFAULT_TBL_ALIAS_PREFIX; - Analyzer currentAnalyzer = analyzer; - do { - usedAliases_.addAll(currentAnalyzer.getAliases()); - usedAliases_.addAll(currentAnalyzer.getLocalViews().keySet()); - currentAnalyzer = currentAnalyzer.getParentAnalyzer(); - } while (currentAnalyzer != null); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/TreeNode.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/TreeNode.java b/fe/src/main/java/com/cloudera/impala/common/TreeNode.java deleted file mode 100644 index ac8f297..0000000 --- a/fe/src/main/java/com/cloudera/impala/common/TreeNode.java +++ /dev/null @@ -1,182 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.common; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import com.cloudera.impala.util.Visitor; -import com.google.common.base.Predicate; - -/** - * Generic tree structure. Only concrete subclasses of this can be instantiated. - */ -public abstract class TreeNode<NodeType extends TreeNode<NodeType>> { - protected ArrayList<NodeType> children_ = new ArrayList<NodeType>(); - - public NodeType getChild(int i) { - return hasChild(i) ? children_.get(i) : null; - } - - public void addChild(NodeType n) { - children_.add(n); - } - - public void removeChild(NodeType n) { children_.remove(n); } - - public void clearChildren() { children_.clear(); } - - public void addChildren(List<? extends NodeType> l) { - children_.addAll(l); - } - - public boolean hasChild(int i) { return children_.size() > i; } - public void setChild(int index, NodeType n) { children_.set(index, n); } - public ArrayList<NodeType> getChildren() { return children_; } - - /** - * Count the total number of nodes in this tree. Leaf node will return 1. - * Non-leaf node will include all its children. - */ - public int numNodes() { - int numNodes = 1; - for (NodeType child: children_) numNodes += child.numNodes(); - return numNodes; - } - - /** - * Add all nodes in the tree that satisfy 'predicate' to the list 'matches' - * This node is checked first, followed by its children in order. If the node - * itself matches, the children are skipped. - */ - public <C extends TreeNode<NodeType>, D extends C> void collect( - Predicate<? super C> predicate, Collection<D> matches) { - // TODO: the semantics of this function are very strange. contains() - // checks using .equals() on the nodes. In the case of literals, slotrefs - // and maybe others, two different tree node objects can be equal and - // this function would only return one of them. This is not intuitive. - // We rely on these semantics to not have duplicate nodes. Investigate this. - if (predicate.apply((C) this) && !matches.contains(this)) { - matches.add((D) this); - return; - } - for (NodeType child: children_) child.collect(predicate, matches); - } - - /** - * Add all nodes in the tree that are of class 'cl' to the list 'matches'. - * This node is checked first, followed by its children in order. If the node - * itself is of class 'cl', the children are skipped. - */ - public <C extends TreeNode<NodeType>, D extends C> void collect( - Class cl, Collection<D> matches) { - if (cl.equals(getClass())) { - matches.add((D) this); - return; - } - for (NodeType child: children_) child.collect(cl, matches); - } - - /** - * Add all nodes in the tree that satisfy 'predicate' to the list 'matches' - * This node is checked first, followed by its children in order. All nodes - * that match in the subtree are added. - */ - public <C extends TreeNode<NodeType>, D extends C> void collectAll( - Predicate<? super C> predicate, List<D> matches) { - if (predicate.apply((C) this)) matches.add((D) this); - for (NodeType child: children_) child.collectAll(predicate, matches); - } - - /** - * For each expression in 'nodeList', collect all subexpressions satisfying 'predicate' - * into 'matches' - */ - public static <C extends TreeNode<C>, D extends C> void collect( - Collection<C> nodeList, Predicate<? super C> predicate, Collection<D> matches) { - for (C node: nodeList) node.collect(predicate, matches); - } - - /** - * For each expression in 'nodeList', collect all subexpressions of class 'cl' - * into 'matches' - */ - public static <C extends TreeNode<C>, D extends C> void collect( - Collection<C> nodeList, Class cl, Collection<D> matches) { - for (C node: nodeList) node.collect(cl, matches); - } - - /** - * Return true if this node or any of its children satisfy 'predicate'. - */ - public <C extends TreeNode<NodeType>> boolean contains( - Predicate<? super C> predicate) { - if (predicate.apply((C) this)) return true; - for (NodeType child: children_) if (child.contains(predicate)) return true; - return false; - } - - /** - * Return true if this node or any of its children is an instance of class 'cl'. - */ - public boolean contains(Class cl) { - if (cl.equals(getClass())) return true; - for (NodeType child: children_) if (child.contains(cl)) return true; - return false; - } - - /** - * For each node in nodeList, return true if any subexpression satisfies - * contains('predicate'). - */ - public static <C extends TreeNode<C>, D extends C> boolean contains( - Collection<C> nodeList, Predicate<? super C> predicate) { - for (C node: nodeList) if (node.contains(predicate)) return true; - return false; - } - - /** - * Return true if any node in nodeList contains children of class cl. - */ - public static <C extends TreeNode<C>> boolean contains( - List<C> nodeList, Class cl) { - for (C node: nodeList) if (node.contains(cl)) return true; - return false; - } - - /** - * Returns the first node/child of class cl (depth-first traversal). - */ - public <C extends NodeType> C findFirstOf(Class<C> cl) { - if (this.getClass().equals(cl)) return (C) this; - for (NodeType child: children_) { - NodeType result = child.findFirstOf(cl); - if (result != null) return (C) result; - } - return null; - } - - /** - * Visitor pattern accept method - */ - public <C extends TreeNode<NodeType>> void accept(Visitor<C> visitor) { - visitor.visit((C) this); - for (NodeType p: children_) p.accept(visitor); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/extdatasource/ApiVersion.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/extdatasource/ApiVersion.java b/fe/src/main/java/com/cloudera/impala/extdatasource/ApiVersion.java deleted file mode 100644 index d2285a4..0000000 --- a/fe/src/main/java/com/cloudera/impala/extdatasource/ApiVersion.java +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.extdatasource; - -import com.cloudera.impala.extdatasource.v1.ExternalDataSource; -import com.google.common.base.Strings; - -/** - * Enumerates the valid versions of the {@link ExternalDataSource} API. - */ -public enum ApiVersion { - V1(com.cloudera.impala.extdatasource.v1.ExternalDataSource.class); - - private final Class<?> apiInterface_; - - ApiVersion(Class<?> interfaceClass) { - apiInterface_ = interfaceClass; - } - - /** - * Gets the {@link Class} for the interface this API version represents. - */ - public Class<?> getApiInterface() { return apiInterface_; } - - /** - * Parses the API version from the string. Is case-insensitive. - * @return The value of the ApiVersion enum represented by the string or null - * if the string is not a valid ApiVersion. - */ - public static ApiVersion parseApiVersion(String apiVersionString) { - if (Strings.isNullOrEmpty(apiVersionString)) return null; - try { - return valueOf(apiVersionString.toUpperCase()); - } catch (IllegalArgumentException ex) { - return null; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/extdatasource/ExternalDataSourceExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/extdatasource/ExternalDataSourceExecutor.java b/fe/src/main/java/com/cloudera/impala/extdatasource/ExternalDataSourceExecutor.java deleted file mode 100644 index 3af470f..0000000 --- a/fe/src/main/java/com/cloudera/impala/extdatasource/ExternalDataSourceExecutor.java +++ /dev/null @@ -1,267 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.extdatasource; - -import java.io.File; -import java.lang.reflect.Constructor; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Map; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.common.ImpalaException; -import com.cloudera.impala.common.ImpalaRuntimeException; -import com.cloudera.impala.common.InternalException; -import com.cloudera.impala.common.JniUtil; -import com.cloudera.impala.extdatasource.thrift.TCloseParams; -import com.cloudera.impala.extdatasource.thrift.TCloseResult; -import com.cloudera.impala.extdatasource.thrift.TGetNextParams; -import com.cloudera.impala.extdatasource.thrift.TGetNextResult; -import com.cloudera.impala.extdatasource.thrift.TOpenParams; -import com.cloudera.impala.extdatasource.thrift.TOpenResult; -import com.cloudera.impala.extdatasource.thrift.TPrepareParams; -import com.cloudera.impala.extdatasource.thrift.TPrepareResult; -import com.cloudera.impala.extdatasource.v1.ExternalDataSource; -import com.cloudera.impala.thrift.TErrorCode; -import com.cloudera.impala.thrift.TStatus; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -/** - * Wraps and executes an ExternalDataSource specified in an external jar. Used - * in planning to call prepare() and in the backend to fetch results. The - * executor takes the API version and abstracts the versioning from the caller, - * e.g. calling the correct API interface and massaging any parameters that can - * be handled here. There are thrift structures for all param and return types - * representing the necessary structures. If future versions of the API are - * added, the executor should be updated to call the appropriate API and handle - * any differences. It is assumed that the API is updated in a way that - * backwards compatibility is possible. - */ -public class ExternalDataSourceExecutor { - private final static Logger LOG = LoggerFactory.getLogger( - ExternalDataSourceExecutor.class); - private final static TBinaryProtocol.Factory protocolFactory_ = - new TBinaryProtocol.Factory(); - - // Init string prefix used to indicate if the class should be cached. When this - // is specified, the Class is loaded and initialized at most once. Instances of - // the cached Class are still created for every query. - private final static String CACHE_CLASS_PREFIX = "CACHE_CLASS::"; - - // Map of class name to cached ExternalDataSource classes. - // Protected by cachedClassesLock_. - private final static Map<String, Class<?>> cachedClasses_ = - Maps.newHashMap(); - - // Number of cache hits/misses in cachedClasses_. Protected by cachedClassesLock_. - private static long numClassCacheHits_ = 0; - private static long numClassCacheMisses_ = 0; - - // Protects cachedClasses_, numClassCacheHits_, and numClassCacheMisses_. - private final static Object cachedClassesLock_ = new Object(); - - private final ApiVersion apiVersion_; - private final ExternalDataSource dataSource_; - private final String jarPath_; - private final String className_; - private final String initString_; - - public static long getNumClassCacheHits() { - synchronized (cachedClassesLock_) { - return numClassCacheHits_; - } - } - - public static long getNumClassCacheMisses() { - synchronized (cachedClassesLock_) { - return numClassCacheMisses_; - } - } - - /** - * @param jarPath The local path to the jar containing the ExternalDataSource. - * @param className The name of the class implementing the ExternalDataSource. - * @param apiVersionStr The API version the ExternalDataSource implements. - * Must be a valid value of {@link ApiVersion}. - * @param initString The init string registered with this data source. - */ - public ExternalDataSourceExecutor(String jarPath, String className, - String apiVersionStr, String initString) throws ImpalaException { - Preconditions.checkNotNull(jarPath); - - apiVersion_ = ApiVersion.valueOf(apiVersionStr); - if (apiVersion_ == null) { - throw new ImpalaRuntimeException("Invalid API version: " + apiVersionStr); - } - jarPath_ = jarPath; - className_ = className; - initString_ = initString; - - try { - Class<?> c = getDataSourceClass(); - Constructor<?> ctor = c.getConstructor(); - dataSource_ = (ExternalDataSource) ctor.newInstance(); - } catch (Exception ex) { - throw new ImpalaRuntimeException(String.format("Unable to load external data " + - "source library from path=%s className=%s apiVersion=%s", jarPath, - className, apiVersionStr), ex); - } - } - - /** - * Returns the ExternalDataSource class, loading the jar if necessary. The - * class is cached if initString_ starts with CACHE_CLASS_PREFIX. - */ - private Class<?> getDataSourceClass() throws Exception { - Class<?> c = null; - // Cache map key needs to contain both the class name and init string in case - // the same class is used for multiple tables where some are cached and others - // are not. - String cacheMapKey = String.format("%s.%s", className_, initString_); - synchronized (cachedClassesLock_) { - c = cachedClasses_.get(cacheMapKey); - if (c == null) { - URL url = new File(jarPath_).toURI().toURL(); - URLClassLoader loader = URLClassLoader.newInstance( - new URL[] { url }, getClass().getClassLoader()); - c = Class.forName(className_, true, loader); - if (!ArrayUtils.contains(c.getInterfaces(), apiVersion_.getApiInterface())) { - throw new ImpalaRuntimeException(String.format( - "Class '%s' does not implement interface '%s' required for API version %s", - className_, apiVersion_.getApiInterface().getName(), apiVersion_.name())); - } - // Only cache the class if the init string starts with CACHE_CLASS_PREFIX - if (initString_ != null && initString_.startsWith(CACHE_CLASS_PREFIX)) { - cachedClasses_.put(cacheMapKey, c); - } - LOG.info("Loaded jar for class {} at path {}", className_, jarPath_); - numClassCacheMisses_++; - } else { - numClassCacheHits_++; - } - } - return c; - } - - public byte[] prepare(byte[] thriftParams) throws ImpalaException { - TPrepareParams params = new TPrepareParams(); - JniUtil.deserializeThrift(protocolFactory_, params, thriftParams); - TPrepareResult result = prepare(params); - try { - return new TSerializer(protocolFactory_).serialize(result); - } catch (TException e) { - throw new InternalException(e.getMessage(), e); - } - } - - public byte[] open(byte[] thriftParams) throws ImpalaException { - TOpenParams params = new TOpenParams(); - JniUtil.deserializeThrift(protocolFactory_, params, thriftParams); - TOpenResult result = open(params); - try { - return new TSerializer(protocolFactory_).serialize(result); - } catch (TException e) { - throw new InternalException(e.getMessage(), e); - } - } - - public byte[] getNext(byte[] thriftParams) throws ImpalaException { - TGetNextParams params = new TGetNextParams(); - JniUtil.deserializeThrift(protocolFactory_, params, thriftParams); - TGetNextResult result = getNext(params); - try { - return new TSerializer(protocolFactory_).serialize(result); - } catch (TException e) { - throw new InternalException(e.getMessage(), e); - } - } - - public byte[] close(byte[] thriftParams) throws ImpalaException { - TCloseParams params = new TCloseParams(); - JniUtil.deserializeThrift(protocolFactory_, params, thriftParams); - TCloseResult result = close(params); - try { - return new TSerializer(protocolFactory_).serialize(result); - } catch (TException e) { - throw new InternalException(e.getMessage(), e); - } - } - - // Helper method to log the exception to capture the stack and return an error TStatus - private TStatus logAndMakeErrorStatus(String opName, Exception e) { - String exceptionMessage = e.getMessage(); - if (exceptionMessage == null) { - exceptionMessage = "No error message returned by data source. Check the " + - "impalad log for more information."; - } - String errorMessage = String.format( - "Error in data source (path=%s, class=%s, version=%s) %s: %s", - jarPath_, className_, apiVersion_.name(), opName, - exceptionMessage); - LOG.error(errorMessage, e); // Logs the stack - return new TStatus(TErrorCode.RUNTIME_ERROR, Lists.newArrayList(errorMessage)); - } - - public TPrepareResult prepare(TPrepareParams params) { - try { - TPrepareResult result = dataSource_.prepare(params); - result.validate(); - return result; - } catch (Exception e) { - return new TPrepareResult(logAndMakeErrorStatus("prepare()", e)); - } - } - - public TOpenResult open(TOpenParams params) { - try { - TOpenResult result = dataSource_.open(params); - result.validate(); - return result; - } catch (Exception e) { - return new TOpenResult(logAndMakeErrorStatus("open()", e)); - } - } - - public TGetNextResult getNext(TGetNextParams params) { - try { - TGetNextResult result = dataSource_.getNext(params); - result.validate(); - return result; - } catch (Exception e) { - return new TGetNextResult(logAndMakeErrorStatus("getNext()", e)); - } - } - - public TCloseResult close(TCloseParams params) { - try { - TCloseResult result = dataSource_.close(params); - result.validate(); - return result; - } catch (Exception e) { - return new TCloseResult(logAndMakeErrorStatus("close()", e)); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBigIntWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBigIntWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBigIntWritable.java deleted file mode 100644 index f410ddf..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBigIntWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.LongWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaBigIntWritable extends LongWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaBigIntWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public long get() { return UnsafeUtil.UNSAFE.getLong(ptr_); } - - @Override - public void set(long v) { UnsafeUtil.UNSAFE.putLong(ptr_, v); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBooleanWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBooleanWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBooleanWritable.java deleted file mode 100644 index f20f49e..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBooleanWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.BooleanWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaBooleanWritable extends BooleanWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaBooleanWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public boolean get() { return UnsafeUtil.UNSAFE.getByte(ptr_) != 0; } - - @Override - public void set(boolean v) { UnsafeUtil.UNSAFE.putByte(ptr_, v ? (byte)1 : 0); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBytesWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBytesWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBytesWritable.java deleted file mode 100644 index ce6fc0c..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaBytesWritable.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.BytesWritable; - -/** - * Impala writable type that implements the BytesWritable interface. The data - * marshalling is handled by the underlying {@link ImpalaStringWritable} object. - */ -public class ImpalaBytesWritable extends BytesWritable { - private final ImpalaStringWritable string_; - - public ImpalaBytesWritable(long ptr) { - string_ = new ImpalaStringWritable(ptr); - } - - @Override - public byte[] copyBytes() { - byte[] src = getBytes(); - return src.clone(); - } - - @Override - public byte[] get() { return getBytes(); } - @Override - public byte[] getBytes() { return string_.getBytes(); } - @Override - public int getCapacity() { return string_.getCapacity(); } - @Override - public int getLength() { return string_.getLength(); } - - public ImpalaStringWritable getStringWritable() { return string_; } - - @Override - public void set(byte[] v, int offset, int len) { string_.set(v, offset, len); } - @Override - public void setCapacity(int newCap) { string_.setCapacity(newCap); } - @Override - public void setSize(int size) { string_.setSize(size); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaDoubleWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaDoubleWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaDoubleWritable.java deleted file mode 100644 index cc96895..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaDoubleWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.hive.serde2.io.DoubleWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaDoubleWritable extends DoubleWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaDoubleWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public double get() { return UnsafeUtil.UNSAFE.getDouble(ptr_); } - - @Override - public void set(double v) { UnsafeUtil.UNSAFE.putDouble(ptr_, v); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaFloatWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaFloatWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaFloatWritable.java deleted file mode 100644 index b68ee69..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaFloatWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.FloatWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaFloatWritable extends FloatWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaFloatWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public float get() { return UnsafeUtil.UNSAFE.getFloat(ptr_); } - - @Override - public void set(float v) { UnsafeUtil.UNSAFE.putFloat(ptr_, v); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaIntWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaIntWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaIntWritable.java deleted file mode 100644 index afe59c4..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaIntWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.IntWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaIntWritable extends IntWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaIntWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public int get() { return UnsafeUtil.UNSAFE.getInt(ptr_); } - - @Override - public void set(int v) { UnsafeUtil.UNSAFE.putInt(ptr_, v); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaSmallIntWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaSmallIntWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaSmallIntWritable.java deleted file mode 100644 index 5a8e346..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaSmallIntWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.hive.serde2.io.ShortWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaSmallIntWritable extends ShortWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaSmallIntWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public short get() { return UnsafeUtil.UNSAFE.getShort(ptr_); } - - @Override - public void set(short v) { UnsafeUtil.UNSAFE.putShort(ptr_, v); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaStringWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaStringWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaStringWritable.java deleted file mode 100644 index a9dca74..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaStringWritable.java +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import java.nio.ByteBuffer; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -/** - * Underlying class for Text and Bytes writable. This class understands marshalling - * values that map to StringValue in the BE. - * StringValue is replicated here: - * struct StringValue { - * char* ptr; - * int len; - * }; - */ -public class ImpalaStringWritable { - // The length is 8 bytes into the struct. - static public final int STRING_VALUE_LEN_OFFSET = 8; - - // Ptr (to native heap) where the value should be read from and written to. - // This needs to be ABI compatible with the BE StringValue class - private final long stringValPtr_; - - // Array object to convert between native and java heap (i.e. byte[]). - private ByteBuffer array_; - - // Set if this object had to allocate from the native heap on the java side. If this - // is set, it will always be stringValPtr_->ptr - // We only need to allocate from the java side if we are trying to set the - // StringValue to a bigger size than what the native side allocated. - // If this object is used as a read-only input argument, this value will stay - // 0. - private long bufferPtr_; - - // Allocation size of stringValPtr_'s ptr. - private int bufferCapacity_; - - // Creates a string writable backed by a StringValue object. Ptr must be a valid - // StringValue (in the native heap). - public ImpalaStringWritable(long ptr) { - stringValPtr_ = ptr; - bufferPtr_= 0; - bufferCapacity_ = getLength(); - array_ = ByteBuffer.allocate(0); - } - - /* - * Implement finalize() to clean up any allocations from the native heap. - */ - @Override - protected void finalize() throws Throwable { - UnsafeUtil.UNSAFE.freeMemory(bufferPtr_); - super.finalize(); - } - - // Returns the underlying bytes as a byte[] - public byte[] getBytes() { - int len = getLength(); - // TODO: reuse this array. - array_ = ByteBuffer.allocate(len); - byte[] buffer = array_.array(); - - long srcPtr = UnsafeUtil.UNSAFE.getLong(stringValPtr_); - UnsafeUtil.Copy(buffer, 0, srcPtr, len); - return buffer; - } - - // Returns the capacity of the underlying array - public int getCapacity() { - return bufferCapacity_; - } - - // Updates the new capacity. No-op if the new capacity is smaller. - public void setCapacity(int newCap) { - if (newCap <= bufferCapacity_) return; - bufferPtr_ = UnsafeUtil.UNSAFE.reallocateMemory(bufferPtr_, newCap); - UnsafeUtil.UNSAFE.putLong(stringValPtr_, bufferPtr_); - bufferCapacity_ = newCap; - } - - // Returns the length of the string - public int getLength() { - return UnsafeUtil.UNSAFE.getInt(stringValPtr_ + STRING_VALUE_LEN_OFFSET); - } - - // Updates the length of the string. If the new length is bigger, - // the additional bytes are undefined. - public void setSize(int s) { - setCapacity(s); - UnsafeUtil.UNSAFE.putInt(stringValPtr_ + 8, s); - } - - // Sets (v[offset], len) to the underlying buffer, growing it as necessary. - public void set(byte[] v, int offset, int len) { - setSize(len); - long strPtr = UnsafeUtil.UNSAFE.getLong(stringValPtr_); - UnsafeUtil.Copy(strPtr, v, offset, len); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTextWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTextWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTextWritable.java deleted file mode 100644 index e17a4df..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTextWritable.java +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.io.Text; - -/** - * Impala writable type that implements the Text interface. The data marshalling is - * handled by the underlying {@link ImpalaStringWritable} object. - */ -public class ImpalaTextWritable extends Text { - private final ImpalaStringWritable string_; - - public ImpalaTextWritable(long ptr) { - string_ = new ImpalaStringWritable(ptr); - } - - @Override - public String toString() { return new String(getBytes()); } - @Override - public byte[] getBytes() { return string_.getBytes(); } - @Override - public int getLength() { return string_.getLength(); } - - public ImpalaStringWritable getStringWritable() { return string_; } - - @Override - public void set(byte[] v, int offset, int len) { - string_.set(v, offset, len); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTinyIntWritable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTinyIntWritable.java b/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTinyIntWritable.java deleted file mode 100644 index 17a7b14..0000000 --- a/fe/src/main/java/com/cloudera/impala/hive/executor/ImpalaTinyIntWritable.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.hive.executor; - -import org.apache.hadoop.hive.serde2.io.ByteWritable; - -import com.cloudera.impala.util.UnsafeUtil; - -@SuppressWarnings("restriction") -public class ImpalaTinyIntWritable extends ByteWritable { - // Ptr (to native heap) where the value should be read from and written to. - private final long ptr_; - - public ImpalaTinyIntWritable(long ptr) { - ptr_ = ptr; - } - - @Override - public byte get() { return UnsafeUtil.UNSAFE.getByte(ptr_); } - - @Override - public void set(byte v) { UnsafeUtil.UNSAFE.putByte(ptr_, v); } -}
