This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d9b7be518 [core] Parquet becomes default file format (#3663)
d9b7be518 is described below
commit d9b7be5183d2d7806f63e577fc5afb8386977224
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Jul 3 19:44:30 2024 +0800
[core] Parquet becomes default file format (#3663)
---
.../shortcodes/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 2 +-
.../apache/paimon/sort/hilbert/HilbertIndexer.java | 4 +-
.../apache/paimon/format/FormatReadWriteTest.java | 49 +++++++++-------------
.../org/apache/paimon/schema/SchemaSerializer.java | 8 +++-
.../java/org/apache/paimon/schema/TableSchema.java | 3 +-
.../schema/TableSchemaSerializationTest.java | 19 ++++++++-
.../paimon/table/ColumnTypeFileMetaTestBase.java | 5 ++-
.../paimon/table/PrimaryKeyFileDataTableTest.java | 2 +-
.../PrimaryKeyTableColumnTypeFileMetaTest.java | 5 ++-
.../apache/paimon/table/system/FilesTableTest.java | 2 +-
.../java/org/apache/paimon/tests/TypeE2eTest.java | 16 +++----
.../flink/action/cdc/CdcActionITCaseBase.java | 2 +
.../postgres/PostgresSyncTableActionITCase.java | 8 ++--
.../apache/paimon/flink/CatalogTableITCase.java | 20 +++++----
.../apache/paimon/flink/SchemaChangeITCase.java | 6 +--
.../SortCompactActionForUnawareBucketITCase.java | 2 +-
.../paimon/flink/sink/CommitterOperatorTest.java | 2 +-
.../format/parquet/ParquetFormatReadWriteTest.java | 5 ---
19 files changed, 87 insertions(+), 75 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index e57becbab..e66438fcd 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -286,7 +286,7 @@ under the License.
</tr>
<tr>
<td><h5>file.format</h5></td>
- <td style="word-wrap: break-word;">"orc"</td>
+ <td style="word-wrap: break-word;">"parquet"</td>
<td>String</td>
<td>Specify the message format of data files, currently orc,
parquet and avro are supported.</td>
</tr>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index c215a2eaa..0f4a4afd0 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -129,7 +129,7 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<String> FILE_FORMAT =
key("file.format")
.stringType()
- .defaultValue(FILE_FORMAT_ORC)
+ .defaultValue(FILE_FORMAT_PARQUET)
.withDescription(
"Specify the message format of data files,
currently orc, parquet and avro are supported.");
diff --git
a/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java
b/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java
index b75227309..14637a6b1 100644
---
a/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java
+++
b/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java
@@ -231,12 +231,12 @@ public class HilbertIndexer implements Serializable {
@Override
public HProcessFunction visit(DateType dateType) {
- return (row) -> row.isNullAt(fieldIndex) ? PRIMITIVE_EMPTY :
row.getLong(fieldIndex);
+ return (row) -> row.isNullAt(fieldIndex) ? PRIMITIVE_EMPTY :
row.getInt(fieldIndex);
}
@Override
public HProcessFunction visit(TimeType timeType) {
- return (row) -> row.isNullAt(fieldIndex) ? PRIMITIVE_EMPTY :
row.getLong(fieldIndex);
+ return (row) -> row.isNullAt(fieldIndex) ? PRIMITIVE_EMPTY :
row.getInt(fieldIndex);
}
@Override
diff --git
a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
index 8945eb413..78e66b64a 100644
---
a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
@@ -75,10 +75,6 @@ public abstract class FormatReadWriteTest {
protected abstract FileFormat fileFormat();
- protected boolean supportNestedNested() {
- return true;
- }
-
@Test
public void testSimpleTypes() throws IOException {
RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.BIGINT());
@@ -162,24 +158,22 @@ public abstract class FormatReadWriteTest {
.field("date", DataTypes.DATE())
.field("decimal", DataTypes.DECIMAL(2, 2))
.field("decimal2", DataTypes.DECIMAL(38, 2))
- .field("decimal3", DataTypes.DECIMAL(10, 1));
-
- if (supportNestedNested()) {
- builder.field(
- "rowArray",
- DataTypes.ARRAY(
- DataTypes.ROW(
- DataTypes.FIELD(
- 0,
- "int0",
- DataTypes.INT().notNull(),
- "nested row int field 0"),
- DataTypes.FIELD(
- 1,
- "double1",
- DataTypes.DOUBLE().notNull(),
- "nested row double field 1"))));
- }
+ .field("decimal3", DataTypes.DECIMAL(10, 1))
+ .field(
+ "rowArray",
+ DataTypes.ARRAY(
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ 0,
+ "int0",
+
DataTypes.INT().notNull(),
+ "nested row int field
0"),
+ DataTypes.FIELD(
+ 1,
+ "double1",
+
DataTypes.DOUBLE().notNull(),
+ "nested row double
field 1"))));
+
RowType rowType = builder.build();
if (ThreadLocalRandom.current().nextBoolean()) {
@@ -217,14 +211,9 @@ public abstract class FormatReadWriteTest {
2456,
Decimal.fromBigDecimal(new BigDecimal("0.22"), 2, 2),
Decimal.fromBigDecimal(new BigDecimal("12312455.22"),
38, 2),
- Decimal.fromBigDecimal(new BigDecimal("12455.1"), 10,
1));
-
- if (supportNestedNested()) {
- values = new ArrayList<>(values);
- values.add(
- new GenericArray(
- new Object[] {GenericRow.of(1, 0.1D),
GenericRow.of(2, 0.2D)}));
- }
+ Decimal.fromBigDecimal(new BigDecimal("12455.1"), 10,
1),
+ new GenericArray(
+ new Object[] {GenericRow.of(1, 0.1D),
GenericRow.of(2, 0.2D)}));
return GenericRow.of(values.toArray());
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaSerializer.java
index dd77497fe..4fb28359c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaSerializer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaSerializer.java
@@ -34,7 +34,9 @@ import java.util.List;
import java.util.Map;
import static org.apache.paimon.CoreOptions.BUCKET;
+import static org.apache.paimon.CoreOptions.FILE_FORMAT;
import static org.apache.paimon.schema.TableSchema.PAIMON_07_VERSION;
+import static org.apache.paimon.schema.TableSchema.PAIMON_08_VERSION;
/** A {@link JsonSerializer} for {@link TableSchema}. */
public class SchemaSerializer
@@ -119,10 +121,14 @@ public class SchemaSerializer
String key = optionsKeys.next();
options.put(key, optionsJson.get(key).asText());
}
- if (version == PAIMON_07_VERSION &&
!options.containsKey(BUCKET.key())) {
+ if (version <= PAIMON_07_VERSION &&
!options.containsKey(BUCKET.key())) {
// the default value of BUCKET in old version is 1
options.put(BUCKET.key(), "1");
}
+ if (version <= PAIMON_08_VERSION &&
!options.containsKey(FILE_FORMAT.key())) {
+ // the default value of FILE_FORMAT in old version is orc
+ options.put(FILE_FORMAT.key(), "orc");
+ }
JsonNode commentNode = node.get("comment");
String comment = null;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java
b/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java
index 4f94c6470..bcad8e92b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java
@@ -49,7 +49,8 @@ public class TableSchema implements Serializable {
private static final long serialVersionUID = 1L;
public static final int PAIMON_07_VERSION = 1;
- public static final int CURRENT_VERSION = 2;
+ public static final int PAIMON_08_VERSION = 2;
+ public static final int CURRENT_VERSION = 3;
// version of schema for paimon
private final int version;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaSerializationTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaSerializationTest.java
index 4657f0f89..efc96ef4d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaSerializationTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaSerializationTest.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import static org.apache.paimon.schema.TableSchema.PAIMON_07_VERSION;
+import static org.apache.paimon.schema.TableSchema.PAIMON_08_VERSION;
import static org.apache.paimon.schema.TableSchemaTest.newRowType;
import static org.assertj.core.api.Assertions.assertThat;
@@ -61,6 +62,22 @@ public class TableSchemaSerializationTest {
assertSerDeser(
new TableSchema(1, fields, 10, partitionKeys, primaryKeys,
options, "my_comment"),
Collections.emptyMap());
+ Map<String, String> additionalOptions = new HashMap<>();
+ additionalOptions.put("file.format", "orc");
+ assertSerDeser(
+ new TableSchema(
+ PAIMON_08_VERSION,
+ 1,
+ fields,
+ 10,
+ partitionKeys,
+ primaryKeys,
+ options,
+ "my_comment",
+ System.currentTimeMillis()),
+ additionalOptions);
+
+ additionalOptions.put("bucket", "1");
assertSerDeser(
new TableSchema(
PAIMON_07_VERSION,
@@ -72,7 +89,7 @@ public class TableSchemaSerializationTest {
options,
"my_comment",
System.currentTimeMillis()),
- Collections.singletonMap("bucket", "1"));
+ additionalOptions);
}
private void assertSerDeser(TableSchema tableSchema, Map<String, String>
additionalOptions) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
index e54184e73..12544a093 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
@@ -302,8 +302,9 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
assertThat(min.getInt(0)).isEqualTo(2);
assertThat(max.getInt(0)).isEqualTo(2);
- assertThat(min.getString(1)).isEqualTo(BinaryString.fromString("200
"));
- assertThat(max.getString(1)).isEqualTo(BinaryString.fromString("300
"));
+ // parquet does not support padding
+ assertThat(min.getString(1).toString()).startsWith("200");
+ assertThat(max.getString(1).toString()).startsWith("300");
assertThat(min.getString(2)).isEqualTo(BinaryString.fromString("201"));
assertThat(max.getString(2)).isEqualTo(BinaryString.fromString("301"));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java
index 4c75b23dc..1be321975 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java
@@ -209,7 +209,7 @@ public class PrimaryKeyFileDataTableTest extends
FileDataFilterTestBase {
.read()
.dataSplits());
// filter with "kt" = 116 in schema0
- TableRead read =
table.newRead().withFilter(builder.equal(4, 116));
+ TableRead read =
table.newRead().withFilter(builder.equal(4, 116L));
assertThat(getResult(read, splits,
STREAMING_SCHEMA_0_ROW_TO_STRING))
.hasSameElementsAs(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
index d9a2518f0..45b67842b 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
@@ -130,8 +130,9 @@ public class PrimaryKeyTableColumnTypeFileMetaTest extends
ColumnTypeFileMetaTes
InternalRow max = serializer.evolution(stats.maxValues());
assertThat(min.getFieldCount()).isEqualTo(4);
if (filesName.contains(fileMeta.fileName())) {
-
assertThat(min.getString(0)).isEqualTo(BinaryString.fromString("200 "));
-
assertThat(max.getString(0)).isEqualTo(BinaryString.fromString("300 "));
+ // parquet does not support padding
+ assertThat(min.getString(0).toString()).startsWith("200");
+ assertThat(max.getString(0).toString()).startsWith("300");
assertThat(min.getString(1)).isEqualTo(BinaryString.fromString("201"));
assertThat(max.getString(1)).isEqualTo(BinaryString.fromString("301"));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java
index 6a0996325..e26e3ba2a 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java
@@ -193,7 +193,7 @@ public class FilesTableTest extends TableTestBase {
Arrays.toString(new String[] {partition1,
partition2})),
fileEntry.bucket(),
BinaryString.fromString(file.fileName()),
- BinaryString.fromString("orc"),
+ BinaryString.fromString(file.fileFormat()),
file.schemaId(),
file.level(),
file.rowCount(),
diff --git
a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/TypeE2eTest.java
b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/TypeE2eTest.java
index 41e68b350..bbd1b22ac 100644
--- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/TypeE2eTest.java
+++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/TypeE2eTest.java
@@ -41,8 +41,8 @@ public class TypeE2eTest extends E2eTestBase {
"f6 DOUBLE,",
"f7 DECIMAL(5, 3),",
"f8 DECIMAL(26, 8),",
- "f9 CHAR(10),",
- "f10 VARCHAR(10),",
+ "f9 string,",
+ "f10 string,",
"f11 STRING,",
"f12 BYTES,",
"f13 DATE,",
@@ -68,7 +68,7 @@ public class TypeE2eTest extends E2eTestBase {
+ "true, cast(1 as tinyint), cast(10 as smallint), "
+ "100, 1000, cast(1.1 as float), 1.11, 12.456, "
+ "cast('123456789123456789.12345678' as decimal(26,
8)), "
- + "cast('hi' as char(10)), 'hello', 'table桌子store商店', "
+ + "cast('hi' as string), 'hello', 'table桌子store商店', "
+ "ENCODE('table桌子store商店', 'UTF-8'), "
+ "DATE '2022-04-28', TIMESTAMP '2022-04-28
15:35:45.123', "
+ "ARRAY['hi', 'hello', cast(null as string), 'test'],
(1, 10, '测试')"
@@ -76,7 +76,7 @@ public class TypeE2eTest extends E2eTestBase {
+ "cast(null as boolean), cast(null as tinyint),
cast(null as smallint), "
+ "cast(null as int), cast(null as bigint), cast(null
as float), "
+ "cast(null as double), cast(null as decimal(5, 3)),
cast(null as decimal(26, 8)), "
- + "cast(null as char(10)), cast(null as varchar(10)),
cast(null as string), "
+ + "cast(null as string), cast(null as string),
cast(null as string), "
+ "cast(null as bytes), cast(null as date), cast(null
as timestamp(3)), "
+ "cast(null as array<string>), cast(null as row<a
int, b bigint, c string>)"
+ ");",
@@ -117,8 +117,8 @@ public class TypeE2eTest extends E2eTestBase {
"f6 DOUBLE,",
"f7 DECIMAL(5, 3),",
"f8 DECIMAL(26, 8),",
- "f9 CHAR(10),",
- "f10 VARCHAR(10),",
+ "f9 STRING,",
+ "f10 STRING,",
"f11 STRING,",
"f12 BYTES,",
"f13 DATE,",
@@ -153,7 +153,7 @@ public class TypeE2eTest extends E2eTestBase {
+ "true, cast(1 as tinyint), cast(10 as smallint), "
+ "100, 1000, cast(1.1 as float), 1.11, 12.456, "
+ "cast('123456789123456789.12345678' as decimal(26,
8)), "
- + "cast('hi' as char(10)), 'hello', 'table桌子store商店', "
+ + "cast('hi' as string), 'hello', 'table桌子store商店', "
+ "ENCODE('table桌子store商店', 'UTF-8'), "
+ "DATE '2022-04-28', TIMESTAMP '2022-04-28
15:35:45.123', "
+ "ARRAY['hi', 'hello', cast(null as string), 'test'],
(1, 10, '测试'), "
@@ -162,7 +162,7 @@ public class TypeE2eTest extends E2eTestBase {
+ "cast(null as boolean), cast(null as tinyint),
cast(null as smallint), "
+ "cast(null as int), cast(null as bigint), cast(null
as float), "
+ "cast(null as double), cast(null as decimal(5, 3)),
cast(null as decimal(26, 8)), "
- + "cast(null as char(10)), cast(null as varchar(10)),
cast(null as string), "
+ + "cast(null as string), cast(null as string),
cast(null as string), "
+ "cast(null as bytes), cast(null as date), cast(null
as timestamp(3)), "
+ "cast(null as array<string>), cast(null as row<a
int, b bigint, c string>), "
+ "cast(null as map<string, bigint>)"
diff --git
a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java
b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java
index 468a3074d..e8ac05e03 100644
---
a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java
+++
b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java
@@ -151,6 +151,8 @@ public class CdcActionITCaseBase extends ActionITCaseBase {
if (sortedExpected.equals(sortedActual)) {
break;
}
+ LOG.info("actual: " + sortedActual);
+ LOG.info("expected: " + sortedExpected);
Thread.sleep(1000);
}
}
diff --git
a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java
b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java
index 1e1524954..b5b36888e 100644
---
a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java
+++
b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java
@@ -326,14 +326,12 @@ public class PostgresSyncTableActionITCase extends
PostgresActionITCaseBase {
.build();
JobClient client = runActionWithDefaultEnv(action);
- try (Statement statement = getStatement(DATABASE_NAME)) {
- testAllTypesImpl(statement);
- }
+ testAllTypesImpl();
client.cancel().get();
}
- private void testAllTypesImpl(Statement statement) throws Exception {
+ private void testAllTypesImpl() throws Exception {
RowType rowType =
RowType.of(
new DataType[] {
@@ -422,7 +420,7 @@ public class PostgresSyncTableActionITCase extends
PostgresActionITCaseBase {
+ "19439, "
+ "2023-03-23T14:30:05, 2023-03-23T00:00, "
+ "36803000, 36803000, "
- + "Paimon, Apache Paimon, Apache Paimon
PostgreSQL Test Data, "
+ + "Paimon , Apache Paimon, Apache Paimon
PostgreSQL Test Data, "
+ "[98, 121, 116, 101, 115], "
+ "{\"a\": \"b\"}, "
+ "[\"item1\", \"item2\"]"
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
index da62348d1..a25914a3d 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
@@ -602,14 +602,14 @@ public class CatalogTableITCase extends CatalogITCaseBase
{
// Get files with latest snapshot
List<Row> rows1 = sql(String.format("SELECT * FROM %s$files",
tableName));
for (Row row : rows1) {
- assertThat(StringUtils.endsWith((String) row.getField(2), ".orc"))
+ assertThat(StringUtils.endsWith((String) row.getField(2),
".parquet"))
.isTrue(); // check file name
assertThat((long) row.getField(7)).isGreaterThan(0L); // check
file size
}
assertThat(getRowStringList(rows1))
.containsExactlyInAnyOrder(
String.format(
- "[2],0,orc,4,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=23, bb=24, dd=25, f=26, p=2},{a=27, bb=28, dd=29, f=30, p=2}",
+ "[2],0,parquet,4,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=23, bb=24, dd=25, f=26, p=2},{a=27, bb=28, dd=29, f=30, p=2}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
// value count table use all fields as
min/max key
? "[23, 2, 24, 25, 26],[27, 2, 28, 29,
30]"
@@ -619,21 +619,21 @@ public class CatalogTableITCase extends CatalogITCaseBase
{
// with key table use primary
key trimmed partition
: "[23],[27]")),
String.format(
- "[1],0,orc,0,0,2,%s,{a=0, bb=0, dd=2, f=2,
p=0},{a=1, bb=2, dd=null, f=null, p=1},{a=3, bb=4, dd=null, f=null, p=1}",
+ "[1],0,parquet,0,0,2,%s,{a=0, bb=0, dd=2, f=2,
p=0},{a=1, bb=2, dd=null, f=null, p=1},{a=3, bb=4, dd=null, f=null, p=1}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
? "[1, 1, 2, S1],[3, 1, 4, S2]"
: (StringUtils.endsWith(tableName,
"APPEND_ONLY")
? ","
: "[1],[3]")),
String.format(
- "[1],0,orc,1,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=5, bb=6, dd=7, f=9, p=1},{a=10, bb=11, dd=12, f=14, p=1}",
+ "[1],0,parquet,1,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=5, bb=6, dd=7, f=9, p=1},{a=10, bb=11, dd=12, f=14, p=1}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
? "[5, 1, 6, S3, 7, 8, 9],[10, 1, 11,
S4, 12, 13, 14]"
: (StringUtils.endsWith(tableName,
"APPEND_ONLY")
? ","
: "[5],[10]")),
String.format(
- "[1],0,orc,4,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=15, bb=16, dd=17, f=18, p=1},{a=19, bb=20, dd=21, f=22, p=1}",
+ "[1],0,parquet,4,0,2,%s,{a=0, bb=0, dd=0, f=0,
p=0},{a=15, bb=16, dd=17, f=18, p=1},{a=19, bb=20, dd=21, f=22, p=1}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
? "[15, 1, 16, 17, 18],[19, 1, 20, 21,
22]"
: (StringUtils.endsWith(tableName,
"APPEND_ONLY")
@@ -647,21 +647,21 @@ public class CatalogTableITCase extends CatalogITCaseBase
{
"SELECT * FROM %s$files /*+
OPTIONS('scan.snapshot-id'='2') */",
tableName));
for (Row row : rows2) {
- assertThat(StringUtils.endsWith((String) row.getField(2), ".orc"))
+ assertThat(StringUtils.endsWith((String) row.getField(2),
".parquet"))
.isTrue(); // check file name
assertThat((long) row.getField(7)).isGreaterThan(0L); // check
file size
}
assertThat(getRowStringList(rows2))
.containsExactlyInAnyOrder(
String.format(
- "[1],0,orc,0,0,2,%s,{a=0, b=0, c=0, d=2, e=2,
f=2, p=0},{a=1, b=2, c=S1, d=null, e=null, f=null, p=1},{a=3, b=4, c=S2,
d=null, e=null, f=null, p=1}",
+ "[1],0,parquet,0,0,2,%s,{a=0, b=0, c=0, d=2,
e=2, f=2, p=0},{a=1, b=2, c=S1, d=null, e=null, f=null, p=1},{a=3, b=4, c=S2,
d=null, e=null, f=null, p=1}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
? "[1, 1, 2, S1],[3, 1, 4, S2]"
: (StringUtils.endsWith(tableName,
"APPEND_ONLY")
? ","
: "[1],[3]")),
String.format(
- "[1],0,orc,1,0,2,%s,{a=0, b=0, c=0, d=0, e=0,
f=0, p=0},{a=5, b=6, c=S3, d=7, e=8, f=9, p=1},{a=10, b=11, c=S4, d=12, e=13,
f=14, p=1}",
+ "[1],0,parquet,1,0,2,%s,{a=0, b=0, c=0, d=0,
e=0, f=0, p=0},{a=5, b=6, c=S3, d=7, e=8, f=9, p=1},{a=10, b=11, c=S4, d=12,
e=13, f=14, p=1}",
StringUtils.endsWith(tableName, "VALUE_COUNT")
? "[5, 1, 6, S3, 7, 8, 9],[10, 1, 11,
S4, 12, 13, 14]"
: (StringUtils.endsWith(tableName,
"APPEND_ONLY")
@@ -749,7 +749,9 @@ public class CatalogTableITCase extends CatalogITCaseBase {
iterator.close();
List<Row> result = sql("SELECT * FROM T$consumers");
- assertThat(result).containsExactly(Row.of("my1", 3L));
+ assertThat(result).hasSize(1);
+ assertThat(result.get(0).getField(0)).isEqualTo("my1");
+ assertThat((Long) result.get(0).getField(1)).isGreaterThanOrEqualTo(3);
}
@Test
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
index 23776971b..ead4edf16 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
@@ -301,14 +301,14 @@ public class SchemaChangeITCase extends CatalogITCaseBase
{
sql("CREATE TABLE T (a STRING PRIMARY KEY NOT ENFORCED, b BOOLEAN, c
BOOLEAN)");
sql("INSERT INTO T VALUES('paimon', true, false)");
- sql("ALTER TABLE T MODIFY (b CHAR(4), c VARCHAR(6))");
+ sql("ALTER TABLE T MODIFY (b STRING, c STRING)");
List<Row> result = sql("SHOW CREATE TABLE T");
assertThat(result.toString())
.contains(
"CREATE TABLE `PAIMON`.`default`.`T` (\n"
+ " `a` VARCHAR(2147483647) NOT NULL,\n"
- + " `b` CHAR(4),\n"
- + " `c` VARCHAR(6),");
+ + " `b` VARCHAR(2147483647),\n"
+ + " `c` VARCHAR(2147483647),");
sql("INSERT INTO T VALUES('apache', '345', '200')");
result = sql("SELECT * FROM T");
assertThat(result.stream().map(Objects::toString).collect(Collectors.toList()))
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/SortCompactActionForUnawareBucketITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/SortCompactActionForUnawareBucketITCase.java
index a3b327027..dabb8bb2c 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/SortCompactActionForUnawareBucketITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/SortCompactActionForUnawareBucketITCase.java
@@ -139,7 +139,7 @@ public class SortCompactActionForUnawareBucketITCase
extends ActionITCaseBase {
.createReader(dataSplit)
.forEachRemaining(
a -> {
- Integer current = a.getInt(2);
+ int current = a.getShort(2);
Assertions.assertThat(current).isGreaterThanOrEqualTo(i.get());
i.set(current);
});
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java
index 3f1281abc..b2764fc37 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java
@@ -565,7 +565,7 @@ public class CommitterOperatorTest extends
CommitterOperatorTestBase {
table, commit, Committer.createContext("",
metricGroup, true, false, null));
committer.commit(Collections.singletonList(manifestCommittable));
CommitterMetrics metrics = committer.getCommitterMetrics();
- assertThat(metrics.getNumBytesOutCounter().getCount()).isEqualTo(293);
+ assertThat(metrics.getNumBytesOutCounter().getCount()).isEqualTo(529);
assertThat(metrics.getNumRecordsOutCounter().getCount()).isEqualTo(2);
committer.close();
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java
index f6f7a2f76..4cfcffe22 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java
@@ -30,11 +30,6 @@ public class ParquetFormatReadWriteTest extends
FormatReadWriteTest {
super("parquet");
}
- @Override
- protected boolean supportNestedNested() {
- return false;
- }
-
@Override
protected FileFormat fileFormat() {
return new ParquetFileFormat(new FileFormatFactory.FormatContext(new
Options(), 1024));
