Repository: incubator-gobblin Updated Branches: refs/heads/master 0412b7ee7 -> 568f0f4d8
[GOBBLIN-645] Fix some typos as reading thru code Log Optimization on query Statement printing Closes #2515 from autumnust/avro2ORCUnblock Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/568f0f4d Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/568f0f4d Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/568f0f4d Branch: refs/heads/master Commit: 568f0f4d8e486e379f1bff0e2c1f3eb1e3db145a Parents: 0412b7e Author: Lei Sun <[email protected]> Authored: Thu Dec 6 09:23:49 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Thu Dec 6 09:23:49 2018 -0800 ---------------------------------------------------------------------- .../hive/dataset/ConvertibleHiveDataset.java | 3 +- .../adaptors/Hive-Avro-To-ORC-Converter.md | 2 +- .../org/apache/gobblin/runtime/fork/Fork.java | 2 +- .../apache/gobblin/util/HiveJdbcConnector.java | 13 +++--- .../gobblin/util/HiveJdbcConnectorTest.java | 47 ++++++++++++++++++++ 5 files changed, 59 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/568f0f4d/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/dataset/ConvertibleHiveDataset.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/dataset/ConvertibleHiveDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/dataset/ConvertibleHiveDataset.java index 63f1bee..3dcbef8 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/dataset/ConvertibleHiveDataset.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/dataset/ConvertibleHiveDataset.java @@ -122,7 +122,7 @@ public class ConvertibleHiveDataset extends HiveDataset { for (String format : this.destFormats) { if (this.datasetConfig.hasPath(format)) { - log.debug("Found desination format: " + format); + log.debug("Found destination format: " + format); this.destConversionConfigs.put(format, new ConversionConfig(this.datasetConfig.getConfig(format), table, format)); } @@ -154,6 +154,7 @@ public class ConvertibleHiveDataset extends HiveDataset { String sourceTable = getTable().getDbName() + "." + getTable().getTableName(); DatasetDescriptor source = new DatasetDescriptor(DatasetConstants.PLATFORM_HIVE, sourceTable); Path sourcePath = getTable().getDataLocation(); + log.info(String.format("[%s]Source path %s being used in conversion", this.getClass().getName(), sourcePath)); String sourceLocation = Path.getPathWithoutSchemeAndAuthority(sourcePath).toString(); FileSystem sourceFs = sourcePath.getFileSystem(new Configuration()); source.addMetadata(DatasetConstants.FS_SCHEME, sourceFs.getScheme()); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/568f0f4d/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md ---------------------------------------------------------------------- diff --git a/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md b/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md index e82b867..c03c6e4 100644 --- a/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md +++ b/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md @@ -74,7 +74,7 @@ By default publishing happens per dataset (dataset = table in this context). If # Job Config Properties -These are some of the job config properties used by `KafkaSource` and `KafkaExtractor`. +These are some of the job config properties used by `HiveAvroToOrcSource` and `HiveConvertExtractor`. <table style="table-layout: fixed; width: 100%"> <col width="20%"> http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/568f0f4d/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/fork/Fork.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/fork/Fork.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/fork/Fork.java index d31f027..68f4536 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/fork/Fork.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/fork/Fork.java @@ -357,7 +357,7 @@ public class Fork<S, D> implements Closeable, FinalState, RecordStreamConsumer<S if (checkDataQuality(this.convertedSchema)) { // Commit data if all quality checkers pass. Again, not to catch the exception // it may throw so the exception gets propagated to the caller of this method. - this.logger.info(String.format("Committing data for fork %d of task %s", this.index, this.taskId)); + this.logger.debug(String.format("Committing data for fork %d of task %s", this.index, this.taskId)); commitData(); verifyAndSetForkState(ForkState.SUCCEEDED, ForkState.COMMITTED); return true; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/568f0f4d/gobblin-utility/src/main/java/org/apache/gobblin/util/HiveJdbcConnector.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/HiveJdbcConnector.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/HiveJdbcConnector.java index 1dd448b..518c75b 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/HiveJdbcConnector.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/HiveJdbcConnector.java @@ -241,14 +241,14 @@ public class HiveJdbcConnector implements Closeable { for (String statement : statements) { if (isSimulate) { - LOG.info("[SIMULATE MODE] STATEMENT NOT RUN: " + choppedStatement(statement)); + LOG.info("[SIMULATE MODE] STATEMENT NOT RUN: " + choppedStatementNoLineChange(statement)); } else { - LOG.info("RUNNING STATEMENT: " + choppedStatement(statement)); + LOG.info("RUNNING STATEMENT: " + choppedStatementNoLineChange(statement)); try (Statement stmt = this.conn.createStatement()) { try { stmt.execute(statement); } catch (SQLException sqe) { - LOG.error("Failed statement: " + statement); + LOG.error("Failed statement: " + choppedStatementNoLineChange(statement)); throw sqe; } } @@ -256,12 +256,15 @@ public class HiveJdbcConnector implements Closeable { } } - private static String choppedStatement(String statement) { + // Chopped statements with all line-changing character being removed for saving space of log. + static String choppedStatementNoLineChange(String statement) { + // \r\n needs to be the first element in the pipe. + statement = statement.replaceAll("\\r\\n|\\r|\\n", " "); if (statement.length() <= MAX_OUTPUT_STMT_LENGTH) { return statement; } return statement.substring(0, MAX_OUTPUT_STMT_LENGTH) + "...... (" + (statement.length() - MAX_OUTPUT_STMT_LENGTH) - + " characters ommitted)"; + + " characters omitted)"; } public Connection getConnection() { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/568f0f4d/gobblin-utility/src/test/java/org/apache/gobblin/util/HiveJdbcConnectorTest.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/java/org/apache/gobblin/util/HiveJdbcConnectorTest.java b/gobblin-utility/src/test/java/org/apache/gobblin/util/HiveJdbcConnectorTest.java new file mode 100644 index 0000000..4b85c1f --- /dev/null +++ b/gobblin-utility/src/test/java/org/apache/gobblin/util/HiveJdbcConnectorTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.util; + +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class HiveJdbcConnectorTest { + + @Test + public void testChoppedStatementNoLineChange() { + String example1 = "This is\na test"; + String example2 = "This is\r\na test\nstring"; + String expected1 = "This is a test"; + String expected2 = "This is a test string"; + Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(example1), expected1); + Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(example2), expected2); + + // Generate a random string longer than 1000 charaters + int iter = 501; + StringBuilder exampleExpected = new StringBuilder(); + StringBuilder exampleResult = new StringBuilder(); + while (iter > 0) { + exampleExpected.append("a "); + exampleResult.append("a\n"); + iter -- ; + } + String expected = exampleExpected.toString().substring(0, 1000) + "...... (2 characters omitted)"; + Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(exampleResult.toString()), expected); + } +} \ No newline at end of file
