LadyForest commented on code in PR #23678:
URL: https://github.com/apache/flink/pull/23678#discussion_r1423376727
##########
flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/factories/DataGenTableSourceFactoryTest.java:
##########
@@ -297,10 +295,104 @@ void testVariableLengthDataGeneration() throws Exception
{
anyCauseMatches(
ValidationException.class,
String.format(
- "Only supports specifying '%s' option
for variable-length types (varchar, string, varbinary, bytes). The type of
field %s is not within this range.",
+ "Only supports specifying '%s' option
for variable-length types (VARCHAR/STRING/VARBINARY/BYTES). The type of field
'%s' is not within this range.",
DataGenConnectorOptions.FIELD_VAR_LEN.key(), "f4")));
}
+ @Test
+ void testVariableLengthDataType() throws Exception {
+ DescriptorProperties descriptor = new DescriptorProperties();
+ final int rowsNumber = 200;
+ descriptor.putString(FactoryUtil.CONNECTOR.key(), "datagen");
+ descriptor.putLong(DataGenConnectorOptions.NUMBER_OF_ROWS.key(),
rowsNumber);
+
+ List<RowData> results = runGenerator(LENGTH_CONSTRAINED_SCHEMA,
descriptor);
+ assertThat(results).hasSize(rowsNumber);
+
+ for (RowData row : results) {
+ assertThat(row.getString(2).toString()).hasSize(30);
+ assertThat(row.getBinary(3)).hasSize(20);
+ assertThat(row.getString(4).toString())
+
.hasSize(RandomGeneratorVisitor.RANDOM_STRING_LENGTH_DEFAULT);
+ }
+
+ descriptor.putString(
+ DataGenConnectorOptionsUtil.FIELDS + ".f2." +
DataGenConnectorOptionsUtil.KIND,
+ DataGenConnectorOptionsUtil.RANDOM);
+ descriptor.putLong(
+ DataGenConnectorOptionsUtil.FIELDS + ".f2." +
DataGenConnectorOptionsUtil.LENGTH,
+ 25);
+ descriptor.putString(
+ DataGenConnectorOptionsUtil.FIELDS + ".f4." +
DataGenConnectorOptionsUtil.KIND,
+ DataGenConnectorOptionsUtil.RANDOM);
+ descriptor.putLong(
+ DataGenConnectorOptionsUtil.FIELDS + ".f4." +
DataGenConnectorOptionsUtil.LENGTH,
+ 9999);
+
+ results = runGenerator(LENGTH_CONSTRAINED_SCHEMA, descriptor);
+
+ for (RowData row : results) {
+ assertThat(row.getString(2).toString()).hasSize(25);
+ assertThat(row.getString(4).toString()).hasSize(9999);
+ }
+
+ assertThatThrownBy(
+ () -> {
+ descriptor.putString(
+ DataGenConnectorOptionsUtil.FIELDS
+ + ".f3."
+ + DataGenConnectorOptionsUtil.KIND,
+ DataGenConnectorOptionsUtil.RANDOM);
+ descriptor.putLong(
+ DataGenConnectorOptionsUtil.FIELDS
+ + ".f3."
+ +
DataGenConnectorOptionsUtil.LENGTH,
+ 21);
+
+ runGenerator(LENGTH_CONSTRAINED_SCHEMA,
descriptor);
+ })
+ .satisfies(
+ anyCauseMatches(
+ ValidationException.class,
+ "Custom length '21' for variable-length type
(VARCHAR/STRING/VARBINARY/BYTES) field 'f3' should be shorter than '20' defined
in the schema."));
+ }
+
+ @Test
+ void testFixedLengthDataType() throws Exception {
+ DescriptorProperties descriptor = new DescriptorProperties();
+ final int rowsNumber = 200;
+ descriptor.putString(FactoryUtil.CONNECTOR.key(), "datagen");
+ descriptor.putLong(DataGenConnectorOptions.NUMBER_OF_ROWS.key(),
rowsNumber);
+
+ List<RowData> results = runGenerator(LENGTH_CONSTRAINED_SCHEMA,
descriptor);
+ assertThat(results).hasSize(rowsNumber);
+
+ for (RowData row : results) {
+ assertThat(row.getString(0).toString()).hasSize(50);
+ assertThat(row.getBinary(1)).hasSize(40);
+ }
+
+ assertThatThrownBy(
+ () -> {
+ descriptor.putString(
+ DataGenConnectorOptionsUtil.FIELDS
+ + ".f0."
+ + DataGenConnectorOptionsUtil.KIND,
+ DataGenConnectorOptionsUtil.RANDOM);
+ descriptor.putLong(
+ DataGenConnectorOptionsUtil.FIELDS
+ + ".f0."
+ +
DataGenConnectorOptionsUtil.LENGTH,
+ 20);
+
+ runGenerator(LENGTH_CONSTRAINED_SCHEMA,
descriptor);
+ })
+ .satisfies(
+ anyCauseMatches(
+ ValidationException.class,
+ "Custom length for fixed-length type
(CHAR/BINARY) field 'f0' is not supported."));
+ }
+
Review Comment:
Nit: I feel like the test is a little bit verbose; maybe we can simplify it
a little bit.
E.g.
extract a util
```java
private void assertException(
ResolvedSchema schema,
DescriptorProperties descriptor,
String fieldName,
@Nullable Integer len,
@Nullable Boolean varLen,
String expectedMessage) {
assertThatThrownBy(
() -> {
descriptor.putString(
String.join(
".",
DataGenConnectorOptionsUtil.FIELDS,
fieldName,
DataGenConnectorOptionsUtil.KIND),
DataGenConnectorOptionsUtil.RANDOM);
if (len != null) {
descriptor.putLong(
String.join(
".",
DataGenConnectorOptionsUtil.FIELDS,
fieldName,
DataGenConnectorOptionsUtil.LENGTH),
len);
}
if (varLen != null) {
descriptor.putBoolean(
String.join(
".",
DataGenConnectorOptionsUtil.FIELDS,
fieldName,
DataGenConnectorOptionsUtil.VAR_LEN),
varLen);
}
runGenerator(schema, descriptor);
})
.satisfies(anyCauseMatches(ValidationException.class,
expectedMessage));
```
then
```java
assertException(
LENGTH_CONSTRAINED_SCHEMA,
descriptor,
"f0",
20,
null,
"Custom length for fixed-length type (CHAR/BINARY) field
'f0' is not supported.");
assertException(
LENGTH_CONSTRAINED_SCHEMA,
descriptor,
"f3",
21,
null,
"Custom length '21' for variable-length type
(VARCHAR/STRING/VARBINARY/BYTES) field 'f3' should be shorter than '20' defined
in the schema.");
assertException(
schema,
descriptor,
"f4",
null,
true,
String.format(
"Only supports specifying '%s' option for
variable-length types (VARCHAR/STRING/VARBINARY/BYTES). The type of field '%s'
is not within this range.",
DataGenConnectorOptions.FIELD_VAR_LEN.key(), "f4"));
```
##########
flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/connector/datagen/table/DataGenTableSourceFactory.java:
##########
@@ -141,15 +143,15 @@ private DataGeneratorContainer createContainer(
}
private void validateFieldOptions(String name, DataType type,
ReadableConfig options) {
- ConfigOption<Boolean> lenOption =
+ ConfigOption<Boolean> varLenOption =
key(DataGenConnectorOptionsUtil.FIELDS
+ "."
+ name
+ "."
+ DataGenConnectorOptionsUtil.VAR_LEN)
.booleanType()
.defaultValue(false);
- options.getOptional(lenOption)
+ options.getOptional(varLenOption)
Review Comment:
What is the purpose of L#155 ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
