voonhous commented on issue #8540:
URL: https://github.com/apache/hudi/issues/8540#issuecomment-1521565619
@easonwood I tried reproducing the issue with the latest version of master
(d3ed4556c8c5cf4c3380ac573903c92abcffbb1d).
Not sure if I am missing anything, can't seem to trigger your error :(.
```sql
test("Test add column + change column type + drop partition") {
withTempDir { tmp =>
val tableName = generateTableName
spark.sql(
s"""
|create table $tableName (
| id string,
| name string,
| price string,
| ts long,
| year string,
| month string,
| day string
|) using hudi
|tblproperties(
| type = 'cow',
| primaryKey = 'id'
|) partitioned by (`year`, `month`, `day`)
|location '${tmp.getCanonicalPath}'
""".stripMargin)
spark.sql(s"insert into $tableName values
(1,'danny','2.22',1000,'2023','04','25')");
checkAnswer(s"select id, name, price, ts, year, month, day from
$tableName")(
Seq("1", "danny", "2.22", 1000, "2023", "04", "25")
)
// enable hudi-full schema evolution
spark.sql(s"set hoodie.schema.on.read.enable=true")
// add a column
spark.sql(s"alter table $tableName add column (new_col bigint)")
spark.sql(s"insert into $tableName values " +
s"(2,'danny','2.22',1001,222222,'2023','04','23'), " +
s"(3,'danny','3.33',1001,333333,'2023','04','24')")
checkAnswer(s"select id, name, price, ts, new_col, year, month, day
from $tableName")(
Seq("1", "danny", "2.22", 1000, null, "2023", "04", "25"),
Seq("2", "danny", "2.22", 1001, 222222, "2023", "04", "23"),
Seq("3", "danny", "3.33", 1001, 333333, "2023", "04", "24")
)
// change column type of ts to string
spark.sql(s"alter table $tableName alter ts type string")
// insert a new record into a different partition
spark.sql(s"insert into $tableName values
(4,'danny','4.44','1002',444444,'2023','04','22')")
checkAnswer(s"select id, name, price, ts, new_col, year, month, day
from $tableName")(
Seq("1", "danny", "2.22", "1000", null, "2023", "04", "25"),
Seq("2", "danny", "2.22", "1001", 222222, "2023", "04", "23"),
Seq("3", "danny", "3.33", "1001", 333333, "2023", "04", "24"),
Seq("4", "danny", "4.44", "1002", 444444, "2023", "04", "22")
)
// drop a partition
spark.sql(s"alter table $tableName drop partition
(year=2023,month=04,day=24)")
checkAnswer(s"select id, name, price, ts, new_col, year, month, day
from $tableName")(
Seq("1", "danny", "2.22", "1000", null, "2023", "04", "25"),
Seq("2", "danny", "2.22", "1001", 222222, "2023", "04", "23"),
Seq("4", "danny", "4.44", "1002", 444444, "2023", "04", "22")
)
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
