zhuangchong commented on code in PR #970:
URL: https://github.com/apache/incubator-paimon/pull/970#discussion_r1185854094
##########
paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java:
##########
@@ -86,33 +95,67 @@ public static HiveSchema extract(@Nullable Configuration
configuration, Properti
throw new UnsupportedOperationException(
"Location property is missing for table "
+ tableName
- + ". Currently Paimon only supports external table
for Hive "
- + "so location property must be set.");
+ + ". Currently Paimon only supports hive table
location property must be set.");
}
Path path = new Path(location);
Options options = PaimonJobConf.extractCatalogConfig(configuration);
options.set(CoreOptions.PATH, path.toUri().toString());
CatalogContext catalogContext = CatalogContext.create(options,
configuration);
- TableSchema tableSchema =
FileStoreTableFactory.create(catalogContext).schema();
-
- if (properties.containsKey(serdeConstants.LIST_COLUMNS)
- && properties.containsKey(serdeConstants.LIST_COLUMN_TYPES)) {
- String columnNames =
properties.getProperty(serdeConstants.LIST_COLUMNS);
- String columnNameDelimiter =
- properties.getProperty(
- // serdeConstants.COLUMN_NAME_DELIMITER is not
defined in earlier Hive
- // versions, so we use a constant string instead
- "column.name.delimite",
String.valueOf(SerDeUtils.COMMA));
- List<String> names =
Arrays.asList(columnNames.split(columnNameDelimiter));
-
- String columnTypes =
properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
- List<TypeInfo> typeInfos =
TypeInfoUtils.getTypeInfosFromTypeString(columnTypes);
-
- if (names.size() > 0 && typeInfos.size() > 0) {
- checkSchemaMatched(names, typeInfos, tableSchema);
+ Optional<TableSchema> tableSchemaOptional =
TableUtils.schema(catalogContext);
Review Comment:
done.
##########
paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java:
##########
@@ -86,33 +95,67 @@ public static HiveSchema extract(@Nullable Configuration
configuration, Properti
throw new UnsupportedOperationException(
"Location property is missing for table "
+ tableName
- + ". Currently Paimon only supports external table
for Hive "
- + "so location property must be set.");
+ + ". Currently Paimon only supports hive table
location property must be set.");
}
Path path = new Path(location);
Options options = PaimonJobConf.extractCatalogConfig(configuration);
options.set(CoreOptions.PATH, path.toUri().toString());
CatalogContext catalogContext = CatalogContext.create(options,
configuration);
- TableSchema tableSchema =
FileStoreTableFactory.create(catalogContext).schema();
-
- if (properties.containsKey(serdeConstants.LIST_COLUMNS)
- && properties.containsKey(serdeConstants.LIST_COLUMN_TYPES)) {
- String columnNames =
properties.getProperty(serdeConstants.LIST_COLUMNS);
- String columnNameDelimiter =
- properties.getProperty(
- // serdeConstants.COLUMN_NAME_DELIMITER is not
defined in earlier Hive
- // versions, so we use a constant string instead
- "column.name.delimite",
String.valueOf(SerDeUtils.COMMA));
- List<String> names =
Arrays.asList(columnNames.split(columnNameDelimiter));
-
- String columnTypes =
properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
- List<TypeInfo> typeInfos =
TypeInfoUtils.getTypeInfosFromTypeString(columnTypes);
-
- if (names.size() > 0 && typeInfos.size() > 0) {
- checkSchemaMatched(names, typeInfos, tableSchema);
+ Optional<TableSchema> tableSchemaOptional =
TableUtils.schema(catalogContext);
+
+ String columnProperty =
properties.getProperty(serdeConstants.LIST_COLUMNS);
+ // Create hive external table with empty ddl
+ if (StringUtils.isEmpty(columnProperty)) {
+ if (!tableSchemaOptional.isPresent()) {
+ throw new IllegalArgumentException(
+ "Schema file not found in location "
+ + location
+ + ". Please create table first.");
}
+ // Paimon external table can read schema from the specified
location
+ return new HiveSchema(tableSchemaOptional.get());
}
+ // Create hive external table with ddl
+ String columnNameDelimiter =
+ properties.getProperty(
+ // serdeConstants.COLUMN_NAME_DELIMITER is not defined
in earlier Hive
+ // versions, so we use a constant string instead
+ "column.name.delimite",
String.valueOf(SerDeUtils.COMMA));
+ List<String> columnNames =
Arrays.asList(columnProperty.split(columnNameDelimiter));
+ String columnTypes =
properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ List<TypeInfo> typeInfos =
TypeInfoUtils.getTypeInfosFromTypeString(columnTypes);
+ List<String> comments =
+ Lists.newArrayList(
+
Splitter.on('\0').split(properties.getProperty("columns.comments")));
+ // Both Paimon table schema and Hive table schema exist
+ if (tableSchemaOptional.isPresent() && columnNames.size() > 0 &&
typeInfos.size() > 0) {
+ LOG.debug(
+ "Extract schema with exists DDL and exists paimon table,
table location:[{}].",
+ location);
+ checkSchemaMatched(columnNames, typeInfos,
tableSchemaOptional.get());
+ comments = schemaComments(tableSchemaOptional.get());
+ }
+
+ int highestFieldId = -1;
+ List<DataField> columns = new ArrayList<>();
+ for (int i = 0; i < columnNames.size(); i++) {
+ columns.add(
+ new DataField(
+ ++highestFieldId,
+ columnNames.get(i),
+ typeInfoToLogicalType(typeInfos.get(i)),
+ comments.get(i)));
+ }
+ TableSchema tableSchema =
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@paimon.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
