hive git commit: HIVE-13174: Remove Vectorizer noise in logs (Wei Zheng, reviewed by Matt McCline)
Repository: hive Updated Branches: refs/heads/branch-1 82c2acdbe -> 9b36bbb04 HIVE-13174: Remove Vectorizer noise in logs (Wei Zheng, reviewed by Matt McCline) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b36bbb0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b36bbb0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b36bbb0 Branch: refs/heads/branch-1 Commit: 9b36bbb042fae311434545f6c4e69b7370d7fd2f Parents: 82c2acd Author: Daniel DaiAuthored: Tue Mar 1 14:43:16 2016 -0800 Committer: Daniel Dai Committed: Tue Mar 1 14:44:46 2016 -0800 -- .../hadoop/hive/ql/optimizer/physical/Vectorizer.java | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/9b36bbb0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 5d010cc..84c1b35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -1539,7 +1539,10 @@ public class Vectorizer implements PhysicalPlanResolver { return false; } } catch (Exception e) { - LOG.info("Failed to vectorize", e); + if (LOG.isDebugEnabled()) { +LOG.debug("Failed to vectorize", e); + } + return false; } return true; @@ -1600,7 +1603,9 @@ public class Vectorizer implements PhysicalPlanResolver { vectorAggrExpr = vc.getAggregatorExpression(aggDesc, isReduce); } catch (Exception e) { // We should have already attempted to vectorize in validateAggregationDesc. - LOG.info("Vectorization of aggreation should have succeeded ", e); + if (LOG.isDebugEnabled()) { +LOG.debug("Vectorization of aggreation should have succeeded ", e); + } return false; }
hive git commit: HIVE-13174: Remove Vectorizer noise in logs (Wei Zheng, reviewed by Matt McCline)
Repository: hive Updated Branches: refs/heads/master 14e927036 -> 86386473c HIVE-13174: Remove Vectorizer noise in logs (Wei Zheng, reviewed by Matt McCline) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86386473 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86386473 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86386473 Branch: refs/heads/master Commit: 86386473c7de6075d124d4e094ffe6cd7fb2e88a Parents: 14e9270 Author: Daniel DaiAuthored: Tue Mar 1 14:43:16 2016 -0800 Committer: Daniel Dai Committed: Tue Mar 1 14:43:16 2016 -0800 -- .../hadoop/hive/ql/optimizer/physical/Vectorizer.java | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/86386473/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index ee080aa..607eb4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -1623,7 +1623,10 @@ public class Vectorizer implements PhysicalPlanResolver { return false; } } catch (Exception e) { - LOG.info("Failed to vectorize", e); + if (LOG.isDebugEnabled()) { +LOG.debug("Failed to vectorize", e); + } + return false; } return true; @@ -1667,7 +1670,9 @@ public class Vectorizer implements PhysicalPlanResolver { vectorAggrExpr = vc.getAggregatorExpression(aggDesc, isReduceMergePartial); } catch (Exception e) { // We should have already attempted to vectorize in validateAggregationDesc. - LOG.info("Vectorization of aggreation should have succeeded ", e); + if (LOG.isDebugEnabled()) { +LOG.debug("Vectorization of aggreation should have succeeded ", e); + } return false; }
hive git commit: HIVE-12757 : Fix TestCodahaleMetrics#testFileReporting (Szehon, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master db4216d6c -> 14e927036 HIVE-12757 : Fix TestCodahaleMetrics#testFileReporting (Szehon, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/14e92703 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/14e92703 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/14e92703 Branch: refs/heads/master Commit: 14e927036e4f7122f694bad10e1957e076a6ec25 Parents: db4216d Author: Szehon HoAuthored: Tue Mar 1 12:07:48 2016 -0800 Committer: Szehon Ho Committed: Tue Mar 1 12:07:48 2016 -0800 -- .../hive/common/metrics/MetricsTestUtils.java | 9 +++ .../metrics/metrics2/TestCodahaleMetrics.java | 26 +--- 2 files changed, 15 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/14e92703/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java -- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java index c90a614..4aaa808 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java @@ -57,4 +57,13 @@ public class MetricsTestUtils { JsonNode metricsNode = categoryNode.path(metricsName); return metricsNode.path(category.metricsHandle); } + + public static byte[] getFileData(String path, int timeoutInterval, int tries) throws Exception { +File file = new File(path); +do { + Thread.sleep(timeoutInterval); + tries--; +} while (tries > 0 && !file.exists()); +return Files.readAllBytes(Paths.get(path)); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/14e92703/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java -- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index 55bb2c2..6ee6245 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -23,6 +23,7 @@ import com.codahale.metrics.Timer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hive.common.metrics.MetricsTestUtils; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; @@ -122,11 +123,9 @@ public class TestCodahaleMetrics { int runs = 5; for (int i = 0; i < runs; i++) { MetricsFactory.getInstance().incrementCounter("count2"); - Thread.sleep(100); } -Thread.sleep(2000); -byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); +byte[] jsonData = MetricsTestUtils.getFileData(jsonReportFile.getAbsolutePath(), 2000, 3); ObjectMapper objectMapper = new ObjectMapper(); JsonNode rootNode = objectMapper.readTree(jsonData); @@ -154,25 +153,12 @@ public class TestCodahaleMetrics { testVar.setValue(20); MetricsFactory.getInstance().addGauge("gauge1", testVar); -Thread.sleep(2000); -byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); -ObjectMapper objectMapper = new ObjectMapper(); +String json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); +MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, "gauge1", testVar.getValue()); -JsonNode rootNode = objectMapper.readTree(jsonData); -JsonNode gaugesNode = rootNode.path("gauges"); -JsonNode methodGaugeNode = gaugesNode.path("gauge1"); -JsonNode countNode = methodGaugeNode.path("value"); -Assert.assertEquals(countNode.asInt(), testVar.getValue()); testVar.setValue(40); -Thread.sleep(2000); - -jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); - -rootNode = objectMapper.readTree(jsonData); -gaugesNode = rootNode.path("gauges"); -methodGaugeNode = gaugesNode.path("gauge1"); -countNode = methodGaugeNode.path("value"); -Assert.assertEquals(countNode.asInt(), testVar.getValue()); +json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); +MetricsTestUtils.verifyMetricsJson(json,
hive git commit: HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-1 f1c75b5f0 -> 82c2acdbe HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82c2acdb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82c2acdb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82c2acdb Branch: refs/heads/branch-1 Commit: 82c2acdbe10cf7a27d58cf0f1fb291b7cdb18e54 Parents: f1c75b5 Author: Yongzhi ChenAuthored: Thu Feb 25 12:54:03 2016 -0500 Committer: Yongzhi Chen Committed: Tue Mar 1 09:45:07 2016 -0500 -- .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 2 +- .../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/ReaderImpl.java | 2 +- ql/src/test/queries/clientpositive/orc_create.q | 12 + .../results/clientpositive/orc_create.q.out | 52 5 files changed, 67 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java index 08f2f5e..b13dad2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java @@ -307,7 +307,7 @@ public final class OrcFile { propValue = tableProperties == null ? null : tableProperties.getProperty(OrcTableProperties.COMPRESSION.propName); compressValue = propValue == null ? CompressionKind.valueOf(HiveConf.getVar(conf, HIVE_ORC_DEFAULT_COMPRESS)) - : CompressionKind.valueOf(propValue); + : CompressionKind.valueOf(propValue.toUpperCase()); propValue = tableProperties == null ? null : tableProperties.getProperty(OrcTableProperties.BLOOM_FILTER_COLUMNS.propName); http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java index 8625ff1..bc55677 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java @@ -140,7 +140,7 @@ public class OrcOutputFormat extends FileOutputFormat if ((propVal = getSettingFromPropsFallingBackToConf( OrcFile.OrcTableProperties.COMPRESSION.getPropName(),props,conf)) != null){ - options.compress(CompressionKind.valueOf(propVal)); + options.compress(CompressionKind.valueOf(propVal.toUpperCase())); } if ((propVal = getSettingFromPropsFallingBackToConf( http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index fc1a8fa..a7c564a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -456,7 +456,7 @@ public class ReaderImpl implements Reader { MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize, ByteBuffer footerBuffer) throws IOException { - this.compressionKind = CompressionKind.valueOf(codecStr); + this.compressionKind = CompressionKind.valueOf(codecStr.toUpperCase()); this.bufferSize = bufferSize; this.codec = WriterImpl.createCodec(compressionKind); this.metadataSize = metadataSize; http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/test/queries/clientpositive/orc_create.q -- diff --git a/ql/src/test/queries/clientpositive/orc_create.q b/ql/src/test/queries/clientpositive/orc_create.q index e42c5ae..dc28105 100644 --- a/ql/src/test/queries/clientpositive/orc_create.q +++ b/ql/src/test/queries/clientpositive/orc_create.q @@ -5,6 +5,7 @@ DROP TABLE orc_create_complex; DROP TABLE orc_create_staging; DROP TABLE orc_create_people_staging; DROP TABLE orc_create_people; +DROP TABLE if exists orc_create_cprl; CREATE TABLE orc_create_staging ( str STRING, @@ -118,8 +119,19 @@ SELECT COUNT(*) FROM orc_create_people where salary = 200.00 and state = 'Ca'; SELECT id, first_name, last_name, address FROM
hive git commit: HIVE-13160: HS2 unable to load UDFs on startup when HMS is not ready (Reviewed by Yongzhi Chen, Sergey Shelukhin)
Repository: hive Updated Branches: refs/heads/master db5fbe913 -> db4216d6c HIVE-13160: HS2 unable to load UDFs on startup when HMS is not ready (Reviewed by Yongzhi Chen, Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db4216d6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db4216d6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db4216d6 Branch: refs/heads/master Commit: db4216d6c1e019d43c94a46fba8f75ce2c78d097 Parents: db5fbe9 Author: Aihua XuAuthored: Fri Feb 26 14:34:46 2016 -0500 Committer: Aihua Xu Committed: Tue Mar 1 09:28:42 2016 -0500 -- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/db4216d6/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index aa1c925..fdc7956 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -175,7 +175,7 @@ public class Hive { private final static int REG_FUNCS_NO = 0, REG_FUNCS_DONE = 2, REG_FUNCS_PENDING = 1; // register all permanent functions. need improvement - private void registerAllFunctionsOnce() { + private void registerAllFunctionsOnce() throws HiveException { boolean breakLoop = false; while (!breakLoop) { int val = didRegisterAllFuncs.get(); @@ -204,11 +204,12 @@ public class Hive { } try { reloadFunctions(); + didRegisterAllFuncs.compareAndSet(REG_FUNCS_PENDING, REG_FUNCS_DONE); } catch (Exception e) { LOG.warn("Failed to register all functions.", e); + didRegisterAllFuncs.compareAndSet(REG_FUNCS_PENDING, REG_FUNCS_NO); + throw new HiveException(e); } finally { - boolean result = didRegisterAllFuncs.compareAndSet(REG_FUNCS_PENDING, REG_FUNCS_DONE); - assert result; synchronized (didRegisterAllFuncs) { didRegisterAllFuncs.notifyAll(); }
hive git commit: HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 90a9a90ed -> db5fbe913 HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db5fbe91 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db5fbe91 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db5fbe91 Branch: refs/heads/master Commit: db5fbe913eeff85e5b7ed238857b668f6449eb55 Parents: 90a9a90 Author: Yongzhi ChenAuthored: Thu Feb 25 12:54:03 2016 -0500 Committer: Yongzhi Chen Committed: Tue Mar 1 09:03:42 2016 -0500 -- orc/src/java/org/apache/orc/OrcFile.java| 2 +- .../hadoop/hive/ql/io/orc/ReaderImpl.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 1 + ql/src/test/queries/clientpositive/orc_create.q | 12 + .../results/clientpositive/orc_create.q.out | 52 5 files changed, 67 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/orc/src/java/org/apache/orc/OrcFile.java -- diff --git a/orc/src/java/org/apache/orc/OrcFile.java b/orc/src/java/org/apache/orc/OrcFile.java index 98226f9..8061089 100644 --- a/orc/src/java/org/apache/orc/OrcFile.java +++ b/orc/src/java/org/apache/orc/OrcFile.java @@ -255,7 +255,7 @@ public class OrcFile { OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties, - conf)); + conf).toUpperCase()); String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 3e031ed..1299c9c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -585,7 +585,7 @@ public class ReaderImpl implements Reader { MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize, ByteBuffer footerBuffer) throws IOException { - this.compressionKind = org.apache.orc.CompressionKind.valueOf(codecStr); + this.compressionKind = org.apache.orc.CompressionKind.valueOf(codecStr.toUpperCase()); this.bufferSize = bufferSize; this.codec = WriterImpl.createCodec(compressionKind); this.metadataSize = metadataSize; http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0db1dab..6373746 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12569,4 +12569,5 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { SessionState.getConsole().printInfo( String.format("Warning: %s", msg)); } + } http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/test/queries/clientpositive/orc_create.q -- diff --git a/ql/src/test/queries/clientpositive/orc_create.q b/ql/src/test/queries/clientpositive/orc_create.q index 57912ef..0647200 100644 --- a/ql/src/test/queries/clientpositive/orc_create.q +++ b/ql/src/test/queries/clientpositive/orc_create.q @@ -6,6 +6,7 @@ DROP TABLE orc_create_complex; DROP TABLE orc_create_staging; DROP TABLE orc_create_people_staging; DROP TABLE orc_create_people; +DROP TABLE if exists orc_create_cprl; CREATE TABLE orc_create_staging ( str STRING, @@ -119,8 +120,19 @@ SELECT COUNT(*) FROM orc_create_people where salary = 200.00 and state = 'Ca'; SELECT id, first_name, last_name, address FROM orc_create_people WHERE id > 90; +-- test create with lower case compression method. +CREATE TABLE orc_create_cprl (id int) +PARTITIONED BY (cdate date) +STORED AS ORC +TBLPROPERTIES ( +'orc.compress'='snappy'); +INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03') +SELECT 1 from src limit 1; +SELECT * from orc_create_cprl; + DROP TABLE orc_create; DROP TABLE orc_create_complex; DROP TABLE orc_create_staging; DROP TABLE