http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/common/src/test/org/apache/hadoop/hive/common/ndv/hll/TestSparseEncodeHash.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/ndv/hll/TestSparseEncodeHash.java b/common/src/test/org/apache/hadoop/hive/common/ndv/hll/TestSparseEncodeHash.java deleted file mode 100644 index 2c7e89b..0000000 --- a/common/src/test/org/apache/hadoop/hive/common/ndv/hll/TestSparseEncodeHash.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.ndv.hll; - -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; -import java.util.Collection; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@RunWith(value = Parameterized.class) -public class TestSparseEncodeHash { - - private long input; - private int expected; - - public TestSparseEncodeHash(long i, int e) { - this.input = i; - this.expected = e; - } - - @Parameters - public static Collection<Object[]> data() { - Object[][] data = new Object[][] { { 11111111111L, 373692871 }, - { 4314495982023L, -1711269433 }, { 4314529536455L, -1744823865 }, - { 4314563074503L, 268425671 }, { 17257983908295L, -1644160569 }, { 536861127L, 536861127 }, - { 536844743L, 536844743 }, { 144115188075862471L, -671082041 } }; - return Arrays.asList(data); - } - - @Test - public void testEncodeHash() { - HLLSparseRegister reg = new HLLSparseRegister(14, 25, 6); - int got = reg.encodeHash(input); - assertEquals(expected, got); - } -}
http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/pom.xml ---------------------------------------------------------------------- diff --git a/metastore/pom.xml b/metastore/pom.xml index 5430580..04c6f47 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -268,20 +268,6 @@ <plugins> <!-- plugins are always listed in sorted order by groupId, artifectId --> <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr3-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>antlr</goal> - </goals> - </execution> - </executions> - <configuration> - <sourceDirectory>${basedir}/src/java</sourceDirectory> - </configuration> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <executions> http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java deleted file mode 100644 index 99bd7b0..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.metastore.api.MetaException; - -/** - * This class is used to monitor long running methods in a thread. - * It is recommended to use it as a ThreadLocal variable. - */ -public class Deadline { - private static final Logger LOG = LoggerFactory.getLogger(Deadline.class.getName()); - - /** - * its value is init from conf, and could be reset from client. - */ - private long timeoutNanos; - - /** - * it is reset before executing a method - */ - private long startTime = NO_DEADLINE; - - /** - * The name of public methods in HMSHandler - */ - private String method; - - private Deadline(long timeoutMs) { - this.timeoutNanos = timeoutMs * 1000000L; - } - - /** - * Deadline object per thread. - */ - private static final ThreadLocal<Deadline> DEADLINE_THREAD_LOCAL = new ThreadLocal<Deadline>() { - @Override - protected Deadline initialValue() { - return null; - } - }; - - private static void setCurrentDeadline(Deadline deadline) { - DEADLINE_THREAD_LOCAL.set(deadline); - } - - static Deadline getCurrentDeadline() { - return DEADLINE_THREAD_LOCAL.get(); - } - - private static void removeCurrentDeadline() { - DEADLINE_THREAD_LOCAL.remove(); - } - - /** - * register a Deadline threadlocal object to current thread. - * @param timeout - */ - public static void registerIfNot(long timeout) { - if (getCurrentDeadline() == null) { - setCurrentDeadline(new Deadline(timeout)); - } - } - - /** - * reset the timeout value of this timer. - * @param timeoutMs - */ - public static void resetTimeout(long timeoutMs) throws MetaException { - if (timeoutMs <= 0) { - throw MetaStoreUtils.newMetaException(new DeadlineException("The reset timeout value should be " + - "larger than 0: " + timeoutMs)); - } - Deadline deadline = getCurrentDeadline(); - if (deadline != null) { - deadline.timeoutNanos = timeoutMs * 1000000L; - } else { - throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it first.")); - } - } - - /** - * start the timer before a method is invoked. - * @param method - */ - public static boolean startTimer(String method) throws MetaException { - Deadline deadline = getCurrentDeadline(); - if (deadline == null) { - throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it first.")); - } - if (deadline.startTime != NO_DEADLINE) return false; - deadline.method = method; - do { - deadline.startTime = System.nanoTime(); - } while (deadline.startTime == NO_DEADLINE); - return true; - } - - /** - * end the time after a method is done. - */ - public static void stopTimer() throws MetaException { - Deadline deadline = getCurrentDeadline(); - if (deadline != null) { - deadline.startTime = NO_DEADLINE; - deadline.method = null; - } else { - throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it first.")); - } - } - - /** - * remove the registered Deadline threadlocal object from current thread. - */ - public static void clear() { - removeCurrentDeadline(); - } - - /** - * Check whether the long running method timeout. - * @throws MetaException when the method timeout - */ - public static void checkTimeout() throws MetaException { - Deadline deadline = getCurrentDeadline(); - if (deadline != null) { - deadline.check(); - } else { - throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it first.")); - } - } - - private static final long NO_DEADLINE = Long.MIN_VALUE; - - private void check() throws MetaException{ - try { - if (startTime == NO_DEADLINE) { - throw new DeadlineException("Should execute startTimer() method before " + - "checkTimeout. Error happens in method: " + method); - } - long elapsedTime = System.nanoTime() - startTime; - if (elapsedTime > timeoutNanos) { - throw new DeadlineException("Timeout when executing method: " + method + "; " - + (elapsedTime / 1000000L) + "ms exceeds " + (timeoutNanos / 1000000L) + "ms"); - } - } catch (DeadlineException e) { - throw MetaStoreUtils.newMetaException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/DeadlineException.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DeadlineException.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DeadlineException.java deleted file mode 100644 index bfff89d..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/DeadlineException.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -/** - * Thrown when a long running method timeout is checked. - */ -public class DeadlineException extends Exception { - - public DeadlineException(String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java deleted file mode 100644 index 832daec..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; -import org.apache.hadoop.hive.metastore.hbase.MetadataStore; - -/** - * The base implementation of a file metadata handler for a specific file type. - * There are currently two classes for each file type (of 1), this one, which is very simple due - * to the fact that it just calls the proxy class for most calls; and the proxy class, that - * contains the actual implementation that depends on some stuff in QL (for ORC). - */ -public abstract class FileMetadataHandler { - protected static final Log LOG = LogFactory.getLog(FileMetadataHandler.class); - - private Configuration conf; - private PartitionExpressionProxy expressionProxy; - private FileFormatProxy fileFormatProxy; - private MetadataStore store; - - /** - * Same as RawStore.getFileMetadataByExpr. - */ - public abstract void getFileMetadataByExpr(List<Long> fileIds, byte[] expr, - ByteBuffer[] metadatas, ByteBuffer[] results, boolean[] eliminated) throws IOException; - - protected abstract FileMetadataExprType getType(); - - protected PartitionExpressionProxy getExpressionProxy() { - return expressionProxy; - } - - protected FileFormatProxy getFileFormatProxy() { - return fileFormatProxy; - } - - protected MetadataStore getStore() { - return store; - } - - /** - * Configures the handler. Called once before use. - * @param conf Config. - * @param expressionProxy Expression proxy to access ql stuff. - * @param store Storage interface to manipulate the metadata. - */ - public void configure( - Configuration conf, PartitionExpressionProxy expressionProxy, MetadataStore store) { - this.conf = conf; - this.expressionProxy = expressionProxy; - this.store = store; - this.fileFormatProxy = expressionProxy.getFileFormatProxy(getType()); - } - - /** - * Caches the file metadata for a particular file. - * @param fileId File id. - * @param fs The filesystem of the file. - * @param path Path to the file. - */ - public void cacheFileMetadata(long fileId, FileSystem fs, Path path) - throws IOException, InterruptedException { - // ORC is in ql, so we cannot do anything here. For now, all the logic is in the proxy. - ByteBuffer[] cols = fileFormatProxy.getAddedColumnsToCache(); - ByteBuffer[] vals = (cols == null) ? null : new ByteBuffer[cols.length]; - ByteBuffer metadata = fileFormatProxy.getMetadataToCache(fs, path, vals); - LOG.info("Caching file metadata for " + path + ", size " + metadata.remaining()); - store.storeFileMetadata(fileId, metadata, cols, vals); - } - - /** - * @return the added column names to be cached in metastore with the metadata for this type. - */ - public ByteBuffer[] createAddedCols() { - return fileFormatProxy.getAddedColumnsToCache(); - } - - /** - * @return the values for the added columns returned by createAddedCols for respective metadatas. - */ - public ByteBuffer[][] createAddedColVals(List<ByteBuffer> metadata) { - return fileFormatProxy.getAddedValuesToCache(metadata); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index d7c33c3..12faf82 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -190,17 +190,6 @@ public class HiveMetaStore extends ThriftHiveMetastore { @VisibleForTesting static long TEST_TIMEOUT_VALUE = -1; - /** A fixed date format to be used for hive partition column values. */ - public static final ThreadLocal<DateFormat> PARTITION_DATE_FORMAT = - new ThreadLocal<DateFormat>() { - @Override - protected DateFormat initialValue() { - DateFormat val = new SimpleDateFormat("yyyy-MM-dd"); - val.setLenient(false); // Without this, 2020-20-20 becomes 2021-08-20. - return val; - }; - }; - public static final String ADMIN = "admin"; public static final String PUBLIC = "public"; /** MM write states. */
