This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 58113a70a [hive] Drop a Hive external table only deletes metadata but
not data files. (#2979)
58113a70a is described below
commit 58113a70a766170de1e2d632fdb055fc22d0ecbf
Author: Kerwin <[email protected]>
AuthorDate: Tue Mar 12 13:24:29 2024 +0800
[hive] Drop a Hive external table only deletes metadata but not data files.
(#2979)
---
.../generated/jdbc_catalog_configuration.html | 2 +-
.../java/org/apache/paimon/jdbc/JdbcCatalog.java | 3 +--
.../org/apache/paimon/jdbc/JdbcCatalogOptions.java | 2 +-
.../paimon/operation/AppendOnlyFileStoreRead.java | 8 +++---
.../org/apache/paimon/utils/BulkFormatMapping.java | 9 ++++---
.../java/org/apache/paimon/hive/HiveCatalog.java | 29 ++++++++--------------
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 2 +-
7 files changed, 24 insertions(+), 31 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html
b/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html
index 617018ce9..87939512b 100644
--- a/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html
@@ -28,7 +28,7 @@ under the License.
<tbody>
<tr>
<td><h5>catalog-key</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
+ <td style="word-wrap: break-word;">"jdbc"</td>
<td>String</td>
<td>Custom jdbc catalog store key.</td>
</tr>
diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
index 952b03607..61dc5959c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
@@ -31,7 +31,6 @@ import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.utils.Preconditions;
-import org.apache.paimon.utils.StringUtils;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
@@ -78,7 +77,7 @@ public class JdbcCatalog extends AbstractCatalog {
protected JdbcCatalog(
FileIO fileIO, String catalogKey, Map<String, String> config,
String warehouse) {
super(fileIO);
- this.catalogKey = StringUtils.isBlank(catalogKey) ? "jdbc" :
catalogKey;
+ this.catalogKey = catalogKey;
this.options = config;
this.warehouse = warehouse;
Preconditions.checkNotNull(options, "Invalid catalog properties:
null");
diff --git
a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java
b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java
index dd4afd473..97828f7c8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java
@@ -27,7 +27,7 @@ public final class JdbcCatalogOptions {
public static final ConfigOption<String> CATALOG_KEY =
ConfigOptions.key("catalog-key")
.stringType()
- .defaultValue(null)
+ .defaultValue("jdbc")
.withDescription("Custom jdbc catalog store key.");
private JdbcCatalogOptions() {}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreRead.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreRead.java
index 93cecaa43..c06cce458 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreRead.java
@@ -140,15 +140,15 @@ public class AppendOnlyFileStoreRead implements
FileStoreRead<InternalRow> {
Pair<int[], RowType> partitionPair = null;
if (!dataSchema.partitionKeys().isEmpty()) {
- Pair<int[], int[][]> partitionMappping =
+ Pair<int[], int[][]> partitionMapping =
PartitionUtils.constructPartitionMapping(
dataSchema,
dataProjection);
// if partition fields are not selected,
we just do nothing
- if (partitionMappping != null) {
- dataProjection =
partitionMappping.getRight();
+ if (partitionMapping != null) {
+ dataProjection =
partitionMapping.getRight();
partitionPair =
Pair.of(
-
partitionMappping.getLeft(),
+
partitionMapping.getLeft(),
dataSchema.projectedLogicalRowType(
dataSchema.partitionKeys()));
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
b/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
index 84476dac7..97c18d769 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
@@ -64,6 +64,7 @@ public class BulkFormatMapping {
return castMapping;
}
+ @Nullable
public Pair<int[], RowType> getPartitionPair() {
return partitionPair;
}
@@ -166,15 +167,15 @@ public class BulkFormatMapping {
Pair<int[], RowType> partitionPair = null;
if (!dataSchema.partitionKeys().isEmpty()) {
- Pair<int[], int[][]> partitionMappping =
+ Pair<int[], int[][]> partitionMapping =
PartitionUtils.constructPartitionMapping(
dataRecordType, dataSchema.partitionKeys(),
dataProjection);
// is partition fields are not selected, we just do nothing.
- if (partitionMappping != null) {
- dataProjection = partitionMappping.getRight();
+ if (partitionMapping != null) {
+ dataProjection = partitionMapping.getRight();
partitionPair =
Pair.of(
- partitionMappping.getLeft(),
+ partitionMapping.getLeft(),
dataSchema.projectedLogicalRowType(dataSchema.partitionKeys()));
}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index a2b1b82f0..024857976 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -87,7 +87,6 @@ import static
org.apache.paimon.options.CatalogOptions.LOCK_ENABLED;
import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE;
import static
org.apache.paimon.options.OptionsUtils.convertToPropertiesPrefixKey;
import static org.apache.paimon.utils.Preconditions.checkArgument;
-import static org.apache.paimon.utils.Preconditions.checkState;
import static org.apache.paimon.utils.StringUtils.isNullOrWhitespaceOnly;
/** A catalog implementation for Hive. */
@@ -349,6 +348,17 @@ public class HiveCatalog extends AbstractCatalog {
try {
client.dropTable(
identifier.getDatabaseName(), identifier.getObjectName(),
true, false, true);
+
+ // When drop a Hive external table, only the hive metadata is
deleted and the data files
+ // are not deleted.
+ TableType tableType =
+ OptionsUtils.convertToEnum(
+ hiveConf.get(TABLE_TYPE.key(),
TableType.MANAGED.toString()),
+ TableType.class);
+ if (TableType.EXTERNAL.equals(tableType)) {
+ return;
+ }
+
// Deletes table directory to avoid schema in filesystem exists
after dropping hive
// table successfully to keep the table consistency between which
in filesystem and
// which in Hive metastore.
@@ -468,19 +478,6 @@ public class HiveCatalog extends AbstractCatalog {
return warehouse;
}
- private void checkIdentifierUpperCase(Identifier identifier) {
- checkState(
-
identifier.getDatabaseName().equals(identifier.getDatabaseName().toLowerCase()),
- String.format(
- "Database name[%s] cannot contain upper case in hive
catalog",
- identifier.getDatabaseName()));
- checkState(
-
identifier.getObjectName().equals(identifier.getObjectName().toLowerCase()),
- String.format(
- "Table name[%s] cannot contain upper case in hive
catalog",
- identifier.getObjectName()));
- }
-
private Table newHmsTable(Identifier identifier, Map<String, String>
tableParameters) {
long currentTimeMillis = System.currentTimeMillis();
TableType tableType =
@@ -587,10 +584,6 @@ public class HiveCatalog extends AbstractCatalog {
dataField.description());
}
- private boolean schemaFileExists(Identifier identifier) {
- return new SchemaManager(fileIO,
getDataTableLocation(identifier)).latest().isPresent();
- }
-
private SchemaManager schemaManager(Identifier identifier) {
return new SchemaManager(fileIO, getDataTableLocation(identifier))
.withLock(lock(identifier));
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index e830ed653..80d6c5460 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -277,7 +277,7 @@ public abstract class HiveCatalogITCaseBase {
.isTrue();
tEnv.executeSql("DROP TABLE t").await();
Path tablePath = new Path(path, "test_db.db/t");
- assertThat(tablePath.getFileSystem().exists(tablePath)).isFalse();
+ assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue();
}
@Test
