leaves12138 commented on code in PR #9251:
URL: https://github.com/apache/paimon/pull/9251#discussion_r3792042580
##########
paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java:
##########
@@ -449,6 +455,75 @@ private static void validateOnlyContainPrimitiveType(
}
}
+ private static void validateGeospatialTypes(
+ TableSchema schema, CoreOptions options, RowType rowType) {
+ boolean hasGeospatial =
+ containsType(
+ rowType,
+ type -> type.isAnyOf(DataTypeRoot.GEOMETRY,
DataTypeRoot.GEOGRAPHY));
+ if (!hasGeospatial) {
+ return;
+ }
+
+ checkArgument(
+ CoreOptions.FILE_FORMAT_PARQUET.equals(options.formatType()),
+ "Geometry and geography columns require '%s'='parquet', but
was '%s'.",
+ CoreOptions.FILE_FORMAT.key(),
+ options.formatType());
+ options.fileFormatPerLevel()
+ .forEach(
+ (level, format) ->
+ checkArgument(
+
CoreOptions.FILE_FORMAT_PARQUET.equals(format),
+ "Geometry and geography columns
require parquet at every level, but '%s' contains '%s:%s'.",
+
CoreOptions.FILE_FORMAT_PER_LEVEL.key(),
+ level,
+ format));
+ checkArgument(
+ options.changelogFileFormat() == null
+ ||
CoreOptions.FILE_FORMAT_PARQUET.equals(options.changelogFileFormat()),
+ "Geometry and geography columns require '%s' to be parquet,
but was '%s'.",
+ CoreOptions.CHANGELOG_FILE_FORMAT.key(),
+ options.changelogFileFormat());
+ IcebergOptions.StorageType icebergStorage =
+
options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE);
+ if (icebergStorage != IcebergOptions.StorageType.DISABLED) {
+ checkArgument(
+
options.toConfiguration().get(IcebergOptions.FORMAT_VERSION) == 3,
+ "Geometry and geography columns require '%s'='3' when
Iceberg metadata is enabled.",
+ IcebergOptions.FORMAT_VERSION.key());
+ checkArgument(
+ icebergStorage != IcebergOptions.StorageType.REST_CATALOG,
+ "Geometry and geography columns do not support '%s'='%s'
because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial
types.",
+ IcebergOptions.METADATA_ICEBERG_STORAGE.key(),
+ IcebergOptions.StorageType.REST_CATALOG);
+ validateIcebergGeographyCrs(rowType);
+ }
+
+ Set<String> geospatialFields =
+ schema.fields().stream()
+ .filter(
+ field ->
+ field.type()
+ .isAnyOf(
+ DataTypeRoot.GEOMETRY,
+
DataTypeRoot.GEOGRAPHY))
+ .map(DataField::name)
+ .collect(Collectors.toSet());
+ Set<String> geospatialBucketKeys = new HashSet<>(schema.bucketKeys());
+ geospatialBucketKeys.retainAll(geospatialFields);
+ checkArgument(
+ geospatialBucketKeys.isEmpty(),
+ "Geometry and geography columns cannot be bucket keys: %s.",
+ geospatialBucketKeys);
+ Set<String> geospatialSequenceFields = new
HashSet<>(options.sequenceField());
+ geospatialSequenceFields.retainAll(geospatialFields);
+ checkArgument(
+ geospatialSequenceFields.isEmpty(),
+ "Geometry and geography columns cannot be sequence fields:
%s.",
+ geospatialSequenceFields);
Review Comment:
Could we also reject geospatial columns in `clustering.columns`? At the
moment schema validation accepts, for example, an append table with
`clustering.incremental=true`, `clustering.columns=geom`, and
`clustering.strategy=order`, but the clustering path later builds a
`RecordComparator` for this field. `GenerateUtils.generateCompare` has no
`GEOMETRY`/`GEOGRAPHY` branch (and `TypeCheckUtils` marks both types as
non-comparable), so comparator generation fails with `IllegalArgumentException:
Illegal type: GEOMETRY` only when clustering runs. This leaves a valid table
configuration that fails at runtime. Please either define an ordering for these
types or reject them for both incremental clustering and
`pk-clustering-override` during schema validation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
