This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c5ff6a2113f [MINOR] Removed useless checks from SqlBasedTransformers
(#11499)
c5ff6a2113f is described below
commit c5ff6a2113fae46a0c3d29b0f5e574eb8e0a9c62
Author: Vova Kolmakov <[email protected]>
AuthorDate: Tue Jun 25 21:32:10 2024 +0700
[MINOR] Removed useless checks from SqlBasedTransformers (#11499)
Co-authored-by: Vova Kolmakov <[email protected]>
---
.../apache/hudi/utilities/transform/SqlFileBasedTransformer.java | 9 ++-------
.../hudi/utilities/transform/SqlQueryBasedTransformer.java | 8 ++------
.../hudi/utilities/transform/TestSqlQueryBasedTransformer.java | 5 +++++
3 files changed, 9 insertions(+), 13 deletions(-)
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
index 6c3b10bd264..cdef1677e58 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
@@ -21,7 +21,6 @@ package org.apache.hudi.utilities.transform;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.utilities.config.SqlTransformerConfig;
-import org.apache.hudi.utilities.exception.HoodieTransformException;
import org.apache.hudi.utilities.exception.HoodieTransformExecutionException;
import org.apache.hadoop.fs.FileSystem;
@@ -72,22 +71,18 @@ public class SqlFileBasedTransformer implements Transformer
{
final TypedProperties props) {
final String sqlFile = getStringWithAltKeys(props,
SqlTransformerConfig.TRANSFORMER_SQL_FILE);
- if (null == sqlFile) {
- throw new HoodieTransformException(
- "Missing required configuration : (" +
SqlTransformerConfig.TRANSFORMER_SQL_FILE.key() + ")");
- }
final FileSystem fs = HadoopFSUtils.getFs(sqlFile,
jsc.hadoopConfiguration(), true);
// tmp table name doesn't like dashes
final String tmpTable =
TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
- LOG.info("Registering tmp table : " + tmpTable);
+ LOG.info("Registering tmp table: {}", tmpTable);
rowDataset.createOrReplaceTempView(tmpTable);
try (final Scanner scanner = new Scanner(fs.open(new Path(sqlFile)),
"UTF-8")) {
Dataset<Row> rows = null;
// each sql statement is separated with semicolon hence set that as
delimiter.
scanner.useDelimiter(";");
- LOG.info("SQL Query for transformation : ");
+ LOG.info("SQL Query for transformation:");
while (scanner.hasNext()) {
String sqlStr = scanner.next();
sqlStr = sqlStr.replaceAll(SRC_PATTERN, tmpTable).trim();
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
index 636e4784950..290e3a69432 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
@@ -20,7 +20,6 @@ package org.apache.hudi.utilities.transform;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.utilities.config.SqlTransformerConfig;
-import org.apache.hudi.utilities.exception.HoodieTransformException;
import org.apache.hudi.utilities.exception.HoodieTransformExecutionException;
import org.apache.spark.api.java.JavaSparkContext;
@@ -50,17 +49,14 @@ public class SqlQueryBasedTransformer implements
Transformer {
public Dataset<Row> apply(JavaSparkContext jsc, SparkSession sparkSession,
Dataset<Row> rowDataset,
TypedProperties properties) {
String transformerSQL = getStringWithAltKeys(properties,
SqlTransformerConfig.TRANSFORMER_SQL);
- if (null == transformerSQL) {
- throw new HoodieTransformException("Missing configuration : (" +
SqlTransformerConfig.TRANSFORMER_SQL.key() + ")");
- }
try {
// tmp table name doesn't like dashes
String tmpTable =
TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
- LOG.info("Registering tmp table : " + tmpTable);
+ LOG.info("Registering tmp table: {}", tmpTable);
rowDataset.createOrReplaceTempView(tmpTable);
String sqlStr = transformerSQL.replaceAll(SRC_PATTERN, tmpTable);
- LOG.debug("SQL Query for transformation : (" + sqlStr + ")");
+ LOG.debug("SQL Query for transformation: {}", sqlStr);
Dataset<Row> transformed = sparkSession.sql(sqlStr);
sparkSession.catalog().dropTempView(tmpTable);
return transformed;
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java
index e9f6f9e4fd3..18360819265 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java
@@ -32,6 +32,7 @@ import java.util.Collections;
import static
org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
public class TestSqlQueryBasedTransformer {
@@ -82,6 +83,10 @@ public class TestSqlQueryBasedTransformer {
// transform
SqlQueryBasedTransformer transformer = new SqlQueryBasedTransformer();
+
+ // test if the class throws illegal argument exception when sql-config is
missing
+ assertThrows(IllegalArgumentException.class, () -> transformer.apply(jsc,
spark, ds, new TypedProperties()));
+
Dataset<Row> result = transformer.apply(jsc, spark, ds, props);
// check result
